diff --git a/.env.example b/.env.example index 002b9d5..47ee018 100644 --- a/.env.example +++ b/.env.example @@ -31,4 +31,4 @@ LARGE_ENTITY_THRESHOLD=100 # The number of times to retry a target if a provider fails. MAX_RETRIES_PER_TARGET=8 # How long cached provider responses are stored (in hours). -CACHE_EXPIRY_HOURS=12 +CACHE_TIMEOUT_HOURS=12 diff --git a/app.py b/app.py index 942277b..b4f3335 100644 --- a/app.py +++ b/app.py @@ -10,6 +10,7 @@ import traceback from flask import Flask, render_template, request, jsonify, send_file, session from datetime import datetime, timezone, timedelta import io +import os from core.session_manager import session_manager from config import config @@ -304,21 +305,6 @@ def export_results(): traceback.print_exc() return jsonify({'success': False, 'error': f'Export failed: {str(e)}'}), 500 - -@app.route('/api/providers', methods=['GET']) -def get_providers(): - """Get information about available providers.""" - try: - user_session_id, scanner = get_user_scanner() - provider_info = scanner.get_provider_info() - - return jsonify({'success': True, 'providers': provider_info, 'user_session_id': user_session_id}) - - except Exception as e: - traceback.print_exc() - return jsonify({'success': False, 'error': f'Internal server error: {str(e)}'}), 500 - - @app.route('/api/config/api-keys', methods=['POST']) def set_api_keys(): """Set API keys for the current session.""" @@ -355,6 +341,106 @@ def set_api_keys(): traceback.print_exc() return jsonify({'success': False, 'error': f'Internal server error: {str(e)}'}), 500 +@app.route('/api/providers', methods=['GET']) +def get_providers(): + """Get enhanced information about available providers including API key sources.""" + try: + user_session_id, scanner = get_user_scanner() + base_provider_info = scanner.get_provider_info() + + # Enhance provider info with API key source information + enhanced_provider_info = {} + + for provider_name, info in base_provider_info.items(): + enhanced_info = dict(info) # Copy base info + + if info['requires_api_key']: + # Determine API key source and configuration status + api_key = scanner.config.get_api_key(provider_name) + backend_api_key = os.getenv(f'{provider_name.upper()}_API_KEY') + + if backend_api_key: + # API key configured via backend/environment + enhanced_info.update({ + 'api_key_configured': True, + 'api_key_source': 'backend', + 'api_key_help': f'API key configured via environment variable {provider_name.upper()}_API_KEY' + }) + elif api_key: + # API key configured via web interface + enhanced_info.update({ + 'api_key_configured': True, + 'api_key_source': 'frontend', + 'api_key_help': f'API key set via web interface (session-only)' + }) + else: + # No API key configured + enhanced_info.update({ + 'api_key_configured': False, + 'api_key_source': None, + 'api_key_help': f'Requires API key to enable {info["display_name"]} integration' + }) + else: + # Provider doesn't require API key + enhanced_info.update({ + 'api_key_configured': True, # Always "configured" for non-API providers + 'api_key_source': None, + 'api_key_help': None + }) + + enhanced_provider_info[provider_name] = enhanced_info + + return jsonify({ + 'success': True, + 'providers': enhanced_provider_info, + 'user_session_id': user_session_id + }) + + except Exception as e: + traceback.print_exc() + return jsonify({'success': False, 'error': f'Internal server error: {str(e)}'}), 500 + + +@app.route('/api/config/providers', methods=['POST']) +def configure_providers(): + """Configure provider settings (enable/disable).""" + try: + data = request.get_json() + if data is None: + return jsonify({'success': False, 'error': 'No provider settings provided'}), 400 + + user_session_id, scanner = get_user_scanner() + session_config = scanner.config + + updated_providers = [] + + for provider_name, settings in data.items(): + provider_name_clean = provider_name.lower().strip() + + if 'enabled' in settings: + # Update the enabled state in session config + session_config.enabled_providers[provider_name_clean] = settings['enabled'] + updated_providers.append(provider_name_clean) + + if updated_providers: + # Reinitialize providers with new settings + scanner._initialize_providers() + session_manager.update_session_scanner(user_session_id, scanner) + + return jsonify({ + 'success': True, + 'message': f'Provider settings updated for: {", ".join(updated_providers)}', + 'user_session_id': user_session_id + }) + else: + return jsonify({'success': False, 'error': 'No valid provider settings were provided.'}), 400 + + except Exception as e: + traceback.print_exc() + return jsonify({'success': False, 'error': f'Internal server error: {str(e)}'}), 500 + + + @app.errorhandler(404) def not_found(error): """Handle 404 errors.""" diff --git a/config.py b/config.py index 2b2ff1c..3333846 100644 --- a/config.py +++ b/config.py @@ -25,7 +25,6 @@ class Config: self.max_concurrent_requests = 1 self.large_entity_threshold = 100 self.max_retries_per_target = 8 - self.cache_expiry_hours = 12 # --- Provider Caching Settings --- self.cache_timeout_hours = 6 # Provider-specific cache timeout @@ -69,7 +68,6 @@ class Config: self.max_concurrent_requests = int(os.getenv('MAX_CONCURRENT_REQUESTS', self.max_concurrent_requests)) self.large_entity_threshold = int(os.getenv('LARGE_ENTITY_THRESHOLD', self.large_entity_threshold)) self.max_retries_per_target = int(os.getenv('MAX_RETRIES_PER_TARGET', self.max_retries_per_target)) - self.cache_expiry_hours = int(os.getenv('CACHE_EXPIRY_HOURS', self.cache_expiry_hours)) self.cache_timeout_hours = int(os.getenv('CACHE_TIMEOUT_HOURS', self.cache_timeout_hours)) # Override Flask and session settings @@ -87,6 +85,60 @@ class Config: self.enabled_providers[provider] = True return True + def set_provider_enabled(self, provider: str, enabled: bool) -> bool: + """ + Set provider enabled status for the session. + + Args: + provider: Provider name + enabled: Whether the provider should be enabled + + Returns: + True if the setting was applied successfully + """ + provider_key = provider.lower() + self.enabled_providers[provider_key] = enabled + return True + + def get_provider_enabled(self, provider: str) -> bool: + """ + Get provider enabled status. + + Args: + provider: Provider name + + Returns: + True if the provider is enabled + """ + provider_key = provider.lower() + return self.enabled_providers.get(provider_key, True) # Default to enabled + + def bulk_set_provider_settings(self, provider_settings: dict) -> dict: + """ + Set multiple provider settings at once. + + Args: + provider_settings: Dict of provider_name -> {'enabled': bool, ...} + + Returns: + Dict with results for each provider + """ + results = {} + + for provider_name, settings in provider_settings.items(): + provider_key = provider_name.lower() + + try: + if 'enabled' in settings: + self.enabled_providers[provider_key] = settings['enabled'] + results[provider_key] = {'success': True, 'enabled': settings['enabled']} + else: + results[provider_key] = {'success': False, 'error': 'No enabled setting provided'} + except Exception as e: + results[provider_key] = {'success': False, 'error': str(e)} + + return results + def get_api_key(self, provider: str) -> Optional[str]: """Get API key for a provider.""" return self.api_keys.get(provider) diff --git a/core/graph_manager.py b/core/graph_manager.py index 3a904f9..5c28c79 100644 --- a/core/graph_manager.py +++ b/core/graph_manager.py @@ -4,7 +4,7 @@ Graph data model for DNSRecon using NetworkX. Manages in-memory graph storage with confidence scoring and forensic metadata. Now fully compatible with the unified ProviderResult data model. -UPDATED: Fixed certificate styling and correlation edge labeling. +UPDATED: Fixed correlation exclusion keys to match actual attribute names. """ import re from datetime import datetime, timezone @@ -41,7 +41,30 @@ class GraphManager: self.correlation_index = {} # Compile regex for date filtering for efficiency self.date_pattern = re.compile(r'^\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}') - self.EXCLUDED_KEYS = ['crtsh_cert_validity_period_days','crtsh_cert_source','crtsh_cert_common_name'] + + # These are the actual attribute names created in providers, WITHOUT provider prefix + self.EXCLUDED_KEYS = [ + # Certificate metadata that creates noise + 'cert_source', # Always 'crtsh' for crtsh provider + 'cert_common_name', + 'cert_validity_period_days', # Numerical, not useful for correlation + #'cert_certificate_id', # Unique per certificate + #'cert_serial_number', # Unique per certificate + 'cert_entry_timestamp', # Timestamp, filtered by date regex anyway + 'cert_not_before', # Date, filtered by date regex anyway + 'cert_not_after', # Date, filtered by date regex anyway + # DNS metadata that creates noise + 'dns_ttl', # TTL values are not meaningful for correlation + # Shodan metadata that might create noise + 'timestamp', # Generic timestamp fields + 'last_update', # Generic timestamp fields + #'org', # Too generic, causes false correlations + #'isp', # Too generic, causes false correlations + # Generic noisy attributes + 'updated_timestamp', # Any timestamp field + 'discovery_timestamp', # Any timestamp field + 'query_timestamp', # Any timestamp field + ] def __getstate__(self): """Prepare GraphManager for pickling, excluding compiled regex.""" @@ -72,14 +95,31 @@ class GraphManager: attr_value = attr.get('value') attr_provider = attr.get('provider', 'unknown') - # Skip excluded attributes and invalid values - if any(excluded_key in attr_name for excluded_key in self.EXCLUDED_KEYS) or not isinstance(attr_value, (str, int, float, bool)) or attr_value is None: - continue + # IMPROVED: More comprehensive exclusion logic + should_exclude = ( + # Check against excluded keys (exact match or substring) + any(excluded_key in attr_name or attr_name == excluded_key for excluded_key in self.EXCLUDED_KEYS) or + # Invalid value types + not isinstance(attr_value, (str, int, float, bool)) or + attr_value is None or + # Boolean values are not useful for correlation + isinstance(attr_value, bool) or + # String values that are too short or are dates + (isinstance(attr_value, str) and ( + len(attr_value) < 4 or + self.date_pattern.match(attr_value) or + # Exclude common generic values that create noise + attr_value.lower() in ['unknown', 'none', 'null', 'n/a', 'true', 'false', '0', '1'] + )) or + # Numerical values that are likely to be unique identifiers + (isinstance(attr_value, (int, float)) and ( + attr_value == 0 or # Zero values are not meaningful + attr_value == 1 or # One values are too common + abs(attr_value) > 1000000 # Very large numbers are likely IDs + )) + ) - if isinstance(attr_value, bool): - continue - - if isinstance(attr_value, str) and (len(attr_value) < 4 or self.date_pattern.match(attr_value)): + if should_exclude: continue # Initialize correlation tracking for this value @@ -149,7 +189,7 @@ class GraphManager: if self.graph.has_node(node_id) and not self.graph.has_edge(node_id, correlation_node_id): # Format relationship label as "corr_provider_attribute" - relationship_label = f"{provider}_{attribute}" + relationship_label = f"corr_{provider}_{attribute}" self.add_edge( source_id=node_id, @@ -170,7 +210,7 @@ class GraphManager: def _has_direct_edge_bidirectional(self, node_a: str, node_b: str) -> bool: """ Check if there's a direct edge between two nodes in either direction. - Returns True if node_a→node_b OR node_b→node_a exists. + Returns True if node_aâ†'node_b OR node_bâ†'node_a exists. """ return (self.graph.has_edge(node_a, node_b) or self.graph.has_edge(node_b, node_a)) @@ -410,12 +450,6 @@ class GraphManager: """Get all nodes of a specific type.""" return [n for n, d in self.graph.nodes(data=True) if d.get('type') == node_type.value] - def get_neighbors(self, node_id: str) -> List[str]: - """Get all unique neighbors (predecessors and successors) for a node.""" - if not self.graph.has_node(node_id): - return [] - return list(set(self.graph.predecessors(node_id)) | set(self.graph.successors(node_id))) - def get_high_confidence_edges(self, min_confidence: float = 0.8) -> List[Tuple[str, str, Dict]]: """Get edges with confidence score above a given threshold.""" return [(u, v, d) for u, v, d in self.graph.edges(data=True) diff --git a/core/provider_result.py b/core/provider_result.py index df2f6f1..7355cf4 100644 --- a/core/provider_result.py +++ b/core/provider_result.py @@ -101,6 +101,7 @@ class ProviderResult: """Get the total number of attributes in this result.""" return len(self.attributes) - def is_large_entity(self, threshold: int) -> bool: - """Check if this result qualifies as a large entity based on relationship count.""" - return self.get_relationship_count() > threshold \ No newline at end of file + ##TODO + #def is_large_entity(self, threshold: int) -> bool: + # """Check if this result qualifies as a large entity based on relationship count.""" + # return self.get_relationship_count() > threshold \ No newline at end of file diff --git a/core/scanner.py b/core/scanner.py index dcd86a2..5e5f8be 100644 --- a/core/scanner.py +++ b/core/scanner.py @@ -370,6 +370,7 @@ class Scanner: task_tuple = (provider_name, target_item) if task_tuple in processed_tasks: self.tasks_skipped += 1 + self.indicators_completed +=1 continue if depth > max_depth: @@ -405,7 +406,7 @@ class Scanner: if self.target_retries[task_tuple] <= self.config.max_retries_per_target: self.task_queue.put((priority, (provider_name, target_item, depth))) self.tasks_re_enqueued += 1 - self.total_tasks_ever_enqueued += 1 + #self.total_tasks_ever_enqueued += 1 else: self.scan_failed_due_to_retries = True self._log_target_processing_error(str(task_tuple), "Max retries exceeded") diff --git a/core/session_manager.py b/core/session_manager.py index 266c356..30a1940 100644 --- a/core/session_manager.py +++ b/core/session_manager.py @@ -108,64 +108,6 @@ class SessionManager: print(f"ERROR: Failed to create session {session_id}: {e}") raise - def clone_session_preserving_config(self, source_session_id: str) -> str: - """ - FIXED: Create a new session that preserves the configuration (including API keys) from an existing session. - This is used when we need a fresh scanner but want to keep user configuration. - """ - with self.creation_lock: - print(f"=== CLONING SESSION {source_session_id} (PRESERVING CONFIG) ===") - - try: - # Get the source session data - source_session_data = self._get_session_data(source_session_id) - if not source_session_data: - print(f"ERROR: Source session {source_session_id} not found for cloning") - return self.create_session() # Fallback to new session - - # Create new session ID - new_session_id = str(uuid.uuid4()) - - # Get the preserved configuration - preserved_config = source_session_data.get('config') - if not preserved_config: - print(f"WARNING: No config found in source session, creating new") - from core.session_config import create_session_config - preserved_config = create_session_config() - - print(f"Preserving config with API keys: {list(preserved_config.api_keys.keys())}") - - # Create new scanner with preserved config - new_scanner = Scanner(session_config=preserved_config) - new_scanner.session_id = new_session_id - - - new_session_data = { - 'scanner': new_scanner, - 'config': preserved_config, - 'created_at': time.time(), - 'last_activity': time.time(), - 'status': 'active', - 'cloned_from': source_session_id - } - - # Store in Redis - serialized_data = pickle.dumps(new_session_data) - session_key = self._get_session_key(new_session_id) - self.redis_client.setex(session_key, self.session_timeout, serialized_data) - - # Initialize stop signal - stop_key = self._get_stop_signal_key(new_session_id) - self.redis_client.setex(stop_key, self.session_timeout, b'0') - - print(f"Cloned session {new_session_id} with preserved configuration") - return new_session_id - - except Exception as e: - print(f"ERROR: Failed to clone session {source_session_id}: {e}") - # Fallback to creating a new session - return self.create_session() - def set_stop_signal(self, session_id: str) -> bool: """ Set the stop signal for a session (cross-process safe). diff --git a/providers/crtsh_provider.py b/providers/crtsh_provider.py index bb712a6..1e623c0 100644 --- a/providers/crtsh_provider.py +++ b/providers/crtsh_provider.py @@ -502,76 +502,6 @@ class CrtShProvider(BaseProvider): return [d for d in final_domains if _is_valid_domain(d)] - def _find_shared_certificates(self, certs1: List[Dict[str, Any]], certs2: List[Dict[str, Any]]) -> List[Dict[str, Any]]: - """Find certificates that are shared between two domain certificate lists.""" - shared = [] - - cert1_ids = set() - for cert in certs1: - cert_id = cert.get('certificate_id') - if cert_id and isinstance(cert_id, (int, str, float, bool, tuple)): - cert1_ids.add(cert_id) - - for cert in certs2: - cert_id = cert.get('certificate_id') - if cert_id and isinstance(cert_id, (int, str, float, bool, tuple)): - if cert_id in cert1_ids: - shared.append(cert) - - return shared - - def _summarize_certificates(self, certificates: List[Dict[str, Any]]) -> Dict[str, Any]: - """Create a summary of certificates for a domain.""" - if not certificates: - return { - 'total_certificates': 0, - 'valid_certificates': 0, - 'expired_certificates': 0, - 'expires_soon_count': 0, - 'unique_issuers': [], - 'latest_certificate': None, - 'has_valid_cert': False, - 'certificate_details': [] - } - - valid_count = sum(1 for cert in certificates if cert.get('is_currently_valid')) - expired_count = len(certificates) - valid_count - expires_soon_count = sum(1 for cert in certificates if cert.get('expires_soon')) - - unique_issuers = list(set(cert.get('issuer_name') for cert in certificates if cert.get('issuer_name'))) - - # Find the most recent certificate - latest_cert = None - latest_date = None - - for cert in certificates: - try: - if cert.get('not_before'): - cert_date = self._parse_certificate_date(cert['not_before']) - if latest_date is None or cert_date > latest_date: - latest_date = cert_date - latest_cert = cert - except Exception: - continue - - # Sort certificates by date for better display (newest first) - sorted_certificates = sorted( - certificates, - key=lambda c: self._get_certificate_sort_date(c), - reverse=True - ) - - return { - 'total_certificates': len(certificates), - 'valid_certificates': valid_count, - 'expired_certificates': expired_count, - 'expires_soon_count': expires_soon_count, - 'unique_issuers': unique_issuers, - 'latest_certificate': latest_cert, - 'has_valid_cert': valid_count > 0, - 'certificate_details': sorted_certificates - } - def _get_certificate_sort_date(self, cert: Dict[str, Any]) -> datetime: """Get a sortable date from certificate data for chronological ordering.""" try: diff --git a/static/css/main.css b/static/css/main.css index 0177c4b..af09f27 100644 --- a/static/css/main.css +++ b/static/css/main.css @@ -2114,4 +2114,185 @@ input[type="text"]:focus, select:focus { flex-direction: column; gap: 0.5rem; } +} + +/* Add these styles to the existing main.css file - they use existing design patterns */ + +/* Settings Modal Enhancements - using existing provider-item styles */ +#settings-modal .modal-section { + margin-bottom: 1.5rem; +} + +#settings-modal .modal-section:last-child { + margin-bottom: 0; +} + +/* Provider configuration section using existing provider-item styling */ +#provider-config-list .provider-item { + margin-bottom: 1rem; + background: linear-gradient(135deg, #2a2a2a 0%, #1e1e1e 100%); + border: 1px solid #333; + border-radius: 8px; + transition: all 0.3s ease; +} + +#provider-config-list .provider-item:last-child { + margin-bottom: 0; +} + +#provider-config-list .provider-item:hover { + border-color: #444; + transform: translateY(-1px); + box-shadow: 0 4px 12px rgba(0,0,0,0.3); +} + +/* Provider toggle styling - extends existing checkbox pattern */ +.provider-toggle { + appearance: none !important; + width: 16px !important; + height: 16px !important; + border: 2px solid #555 !important; + background: #1a1a1a !important; + cursor: pointer !important; + position: relative !important; + border-radius: 3px !important; + transition: all 0.3s ease !important; + margin: 0 !important; +} + +.provider-toggle:checked { + background: #00ff41 !important; + border-color: #00ff41 !important; +} + +.provider-toggle:checked::after { + content: '✓' !important; + position: absolute !important; + top: -3px !important; + left: 1px !important; + color: #1a1a1a !important; + font-size: 12px !important; + font-weight: bold !important; +} + +.provider-toggle:hover { + border-color: #00ff41 !important; + box-shadow: 0 0 5px rgba(0, 255, 65, 0.3) !important; +} + +/* API Key status row styling - uses existing status-row pattern */ +.api-key-status-row { + padding: 0.75rem; + border-radius: 4px; + border: 1px solid; + transition: all 0.3s ease; + margin-top: 0.5rem; +} + +.api-key-status-row.configured { + background: rgba(0, 255, 65, 0.1); + border-color: rgba(0, 255, 65, 0.3); +} + +.api-key-status-row.not-configured { + background: rgba(255, 153, 0, 0.1); + border-color: rgba(255, 153, 0, 0.3); +} + +/* Settings modal button styling - uses existing button classes */ +#settings-modal .button-group { + padding-top: 1.5rem; + border-top: 1px solid #333; + margin-top: 1.5rem; +} + +/* Clear button specific styling - extends btn-secondary */ +.clear-api-key-btn { + font-size: 0.8rem !important; + padding: 0.4rem 0.8rem !important; + min-width: auto !important; +} + +.clear-api-key-btn:disabled { + opacity: 0.5 !important; + cursor: not-allowed !important; +} + +/* Provider configuration checkbox label styling */ +#provider-config-list .status-row label { + color: #e0e0e0; + cursor: pointer; + transition: color 0.3s ease; + font-size: 0.9rem; + font-weight: 500; +} + +#provider-config-list .status-row label:hover { + color: #00ff41; +} + +/* Settings section count badges - uses existing merge-badge styling */ +#provider-count, +#api-key-count { + background: #444; + color: #fff; + padding: 0.25rem 0.5rem; + border-radius: 3px; + font-size: 0.8rem; + font-weight: 600; + min-width: 20px; + text-align: center; +} + +/* API key help text styling - uses existing patterns */ +#settings-modal .apikey-help { + font-size: 0.8rem; + color: #666; + margin-top: 0.5rem; + font-style: italic; + line-height: 1.4; +} + +/* Modal section content spacing */ +#settings-modal .modal-section-content { + padding: 1rem; +} + +#settings-modal .modal-section-content .provider-item { + padding: 1rem; +} + +/* Transition effects for API key clearing */ +.api-key-clearing { + opacity: 0.6; + pointer-events: none; + transition: all 0.3s ease; +} + +.api-key-cleared { + opacity: 0; + transform: translateX(-10px); + transition: all 0.3s ease; +} + +/* Settings modal responsiveness - extends existing responsive patterns */ +@media (max-width: 768px) { + #settings-modal .modal-content { + width: 95%; + margin: 5% auto; + max-height: 90vh; + } + + #provider-config-list .provider-item { + margin-bottom: 0.75rem; + } + + #settings-modal .provider-stats { + grid-template-columns: 1fr; + gap: 0.5rem; + } + + #settings-modal .button-group { + flex-direction: column; + } } \ No newline at end of file diff --git a/static/js/main.js b/static/js/main.js index f1d82c0..948182b 100644 --- a/static/js/main.js +++ b/static/js/main.js @@ -34,8 +34,8 @@ class DNSReconApp { this.updateStatus(); this.loadProviders(); this.initializeEnhancedModals(); + this.addCheckboxStyling(); - // FIXED: Force initial graph update to handle empty sessions properly this.updateGraph(); console.log('DNSRecon application initialized successfully'); @@ -81,9 +81,7 @@ class DNSReconApp { // Settings Modal elements settingsModal: document.getElementById('settings-modal'), settingsModalClose: document.getElementById('settings-modal-close'), - apiKeyInputs: document.getElementById('api-key-inputs'), - saveApiKeys: document.getElementById('save-api-keys'), - resetApiKeys: document.getElementById('reset-api-keys'), + // Other elements sessionId: document.getElementById('session-id'), @@ -182,10 +180,21 @@ class DNSReconApp { }); } if (this.elements.saveApiKeys) { - this.elements.saveApiKeys.addEventListener('click', () => this.saveApiKeys()); + this.elements.saveApiKeys.removeEventListener('click', this.saveApiKeys); } if (this.elements.resetApiKeys) { - this.elements.resetApiKeys.addEventListener('click', () => this.resetApiKeys()); + this.elements.resetApiKeys.removeEventListener('click', this.resetApiKeys); + } + + // Setup new handlers + const saveSettingsBtn = document.getElementById('save-settings'); + const resetSettingsBtn = document.getElementById('reset-settings'); + + if (saveSettingsBtn) { + saveSettingsBtn.addEventListener('click', () => this.saveSettings()); + } + if (resetSettingsBtn) { + resetSettingsBtn.addEventListener('click', () => this.resetSettings()); } // Listen for the custom event from the graph @@ -724,7 +733,7 @@ class DNSReconApp { if (response.success) { this.updateProviderDisplay(response.providers); - this.buildApiKeyModal(response.providers); + this.buildSettingsModal(response.providers); // Updated to use new function console.log('Providers loaded successfully'); } @@ -732,6 +741,411 @@ class DNSReconApp { console.error('Failed to load providers:', error); } } + + /** + * Build the enhanced settings modal with provider configuration and API keys + * @param {Object} providers - Provider information from backend + */ + buildSettingsModal(providers) { + this.buildProviderConfigSection(providers); + this.buildApiKeySection(providers); + this.updateSettingsCounts(providers); + } + + /** + * Build the provider configuration section with enable/disable checkboxes + * @param {Object} providers - Provider information + */ + buildProviderConfigSection(providers) { + const providerConfigList = document.getElementById('provider-config-list'); + if (!providerConfigList) return; + + providerConfigList.innerHTML = ''; + + for (const [name, info] of Object.entries(providers)) { + const providerConfig = document.createElement('div'); + providerConfig.className = 'provider-item'; + + const statusClass = info.enabled ? 'enabled' : 'disabled'; + const statusIcon = info.enabled ? '✓' : '✗'; + + providerConfig.innerHTML = ` +
+
${info.display_name}
+
+ ${statusIcon} ${info.enabled ? 'Enabled' : 'Disabled'} +
+
+
+
+ +
+
+ `; + + providerConfigList.appendChild(providerConfig); + } + + // Add checkbox styling and event handlers + this.setupProviderCheckboxes(); + } + + /** + * Setup provider checkbox styling and event handlers + */ + setupProviderCheckboxes() { + const checkboxes = document.querySelectorAll('.provider-toggle'); + + checkboxes.forEach(checkbox => { + // Apply existing checkbox styling + checkbox.style.cssText = ` + appearance: none; + width: 16px; + height: 16px; + border: 2px solid #555; + background: #1a1a1a; + cursor: pointer; + position: relative; + border-radius: 3px; + transition: all 0.3s ease; + `; + + // Update visual state + this.updateCheckboxAppearance(checkbox); + + // Add change event handler + checkbox.addEventListener('change', (e) => { + this.updateCheckboxAppearance(e.target); + }); + }); + } + + /** + * Add CSS for checkbox styling since we're using existing styles + */ + addCheckboxStyling() { + // Add CSS for the checkboxes to work with existing styles + const style = document.createElement('style'); + style.textContent = ` + .provider-toggle[data-checked="true"]::after { + content: '✓'; + position: absolute; + top: -2px; + left: 2px; + color: #1a1a1a; + font-size: 12px; + font-weight: bold; + } + + .provider-toggle:hover { + border-color: #00ff41; + } + + .api-key-status-row { + transition: all 0.3s ease; + } + + .provider-item { + margin-bottom: 1rem; + } + + .provider-item:last-child { + margin-bottom: 0; + } + `; + document.head.appendChild(style); + } + + + /** + * Update checkbox appearance based on checked state + */ + updateCheckboxAppearance(checkbox) { + if (checkbox.checked) { + checkbox.style.background = '#00ff41'; + checkbox.style.borderColor = '#00ff41'; + checkbox.style.setProperty('content', '"✓"', 'important'); + + // Add checkmark via pseudo-element simulation + checkbox.setAttribute('data-checked', 'true'); + } else { + checkbox.style.background = '#1a1a1a'; + checkbox.style.borderColor = '#555'; + checkbox.removeAttribute('data-checked'); + } + } + + /** + * Enhanced API key section builder - FIXED to always allow API key input + * @param {Object} providers - Provider information + */ + buildApiKeySection(providers) { + const apiKeyInputs = document.getElementById('api-key-inputs'); + if (!apiKeyInputs) return; + + apiKeyInputs.innerHTML = ''; + let hasApiKeyProviders = false; + + for (const [name, info] of Object.entries(providers)) { + if (info.requires_api_key) { + hasApiKeyProviders = true; + + const inputGroup = document.createElement('div'); + inputGroup.className = 'provider-item'; + + // Check if API key is set via backend (not clearable) or frontend (clearable) + const isBackendConfigured = info.api_key_source === 'backend'; + + if (info.api_key_configured && isBackendConfigured) { + // API key is configured via backend - show status only + inputGroup.innerHTML = ` +
+
${info.display_name}
+
✓ Backend Configured
+
+
+
+
+
API Key Active
+
+ Configured via environment variable +
+
+
+
+ `; + } else if (info.api_key_configured && !isBackendConfigured) { + // API key is configured via frontend - show status with clear option + inputGroup.innerHTML = ` +
+
${info.display_name}
+
✓ Web Configured
+
+
+
+
+
API Key Active
+
+ Set via web interface (session-only) +
+
+ +
+
+ `; + } else { + // API key not configured - ALWAYS show input field + const statusClass = info.enabled ? 'enabled' : 'api-key-required'; + const statusText = info.enabled ? '○ Ready for API Key' : '⚠️ API Key Required'; + + inputGroup.innerHTML = ` +
+
${info.display_name}
+
+ ${statusText} +
+
+
+ + +
+ ${info.api_key_help || `Provides enhanced ${info.display_name.toLowerCase()} data and context.`} + ${!info.enabled ? ' Enable the provider above to use this API key.' : ''} +
+
+ `; + } + + apiKeyInputs.appendChild(inputGroup); + } + } + + if (!hasApiKeyProviders) { + apiKeyInputs.innerHTML = ` +
+
No providers require API keys
+
All Active
+
+ `; + } + + // Setup clear button event handlers + this.setupApiKeyClearHandlers(); + } + + /** + * Setup API key clear button handlers + */ + setupApiKeyClearHandlers() { + document.querySelectorAll('.clear-api-key-btn').forEach(button => { + button.addEventListener('click', (e) => { + e.preventDefault(); + const provider = e.currentTarget.dataset.provider; + this.clearSingleApiKey(provider, e.currentTarget); + }); + }); + } + + /** + * Clear a single API key with immediate feedback + */ + async clearSingleApiKey(provider, buttonElement) { + try { + // Show immediate feedback + const originalContent = buttonElement.innerHTML; + buttonElement.innerHTML = '[...]Clearing...'; + buttonElement.disabled = true; + + const response = await this.apiCall('/api/config/api-keys', 'POST', { [provider]: '' }); + + if (response.success) { + // Find the parent container and update it + const providerContainer = buttonElement.closest('.provider-item'); + const statusRow = providerContainer.querySelector('.api-key-status-row'); + + // Animate out the current status + statusRow.style.transition = 'all 0.3s ease'; + statusRow.style.opacity = '0'; + statusRow.style.transform = 'translateX(-10px)'; + + setTimeout(() => { + // Replace with input field + const providerName = buttonElement.dataset.provider; + const apiKeySection = this.elements.apiKeyInputs; + + // Rebuild the API key section to reflect changes + this.loadProviders(); + + this.showSuccess(`API key for ${provider} has been cleared.`); + }, 300); + + } else { + throw new Error(response.error || 'Failed to clear API key'); + } + } catch (error) { + // Restore button on error + buttonElement.innerHTML = originalContent; + buttonElement.disabled = false; + this.showError(`Error clearing API key: ${error.message}`); + } + } + + /** + * Update settings modal counts + */ + updateSettingsCounts(providers) { + const providerCount = Object.keys(providers).length; + const apiKeyCount = Object.values(providers).filter(p => p.requires_api_key).length; + + const providerCountElement = document.getElementById('provider-count'); + const apiKeyCountElement = document.getElementById('api-key-count'); + + if (providerCountElement) providerCountElement.textContent = providerCount; + if (apiKeyCountElement) apiKeyCountElement.textContent = apiKeyCount; + } + + /** + * Enhanced save settings function + */ + async saveSettings() { + try { + const settings = { + apiKeys: {}, + providerSettings: {} + }; + + // Collect API key inputs + const apiKeyInputs = document.querySelectorAll('#api-key-inputs input[type="password"]'); + apiKeyInputs.forEach(input => { + const provider = input.dataset.provider; + const value = input.value.trim(); + if (provider && value) { + settings.apiKeys[provider] = value; + } + }); + + // Collect provider enable/disable settings + const providerCheckboxes = document.querySelectorAll('.provider-toggle'); + providerCheckboxes.forEach(checkbox => { + const provider = checkbox.dataset.provider; + if (provider) { + settings.providerSettings[provider] = { + enabled: checkbox.checked + }; + } + }); + + // Save API keys if any + if (Object.keys(settings.apiKeys).length > 0) { + const apiKeyResponse = await this.apiCall('/api/config/api-keys', 'POST', settings.apiKeys); + if (!apiKeyResponse.success) { + throw new Error(apiKeyResponse.error || 'Failed to save API keys'); + } + } + + // Save provider settings if any + if (Object.keys(settings.providerSettings).length > 0) { + const providerResponse = await this.apiCall('/api/config/providers', 'POST', settings.providerSettings); + if (!providerResponse.success) { + throw new Error(providerResponse.error || 'Failed to save provider settings'); + } + } + + this.showSuccess('Settings saved successfully'); + this.hideSettingsModal(); + + // Reload providers to reflect changes + this.loadProviders(); + + } catch (error) { + this.showError(`Error saving settings: ${error.message}`); + } + } + + /** + * Reset settings to defaults + */ + async resetSettings() { + try { + // Clear all API key inputs + const apiKeyInputs = document.querySelectorAll('#api-key-inputs input[type="password"]'); + apiKeyInputs.forEach(input => { + input.value = ''; + }); + + // Reset all provider checkboxes to enabled (default) + const providerCheckboxes = document.querySelectorAll('.provider-toggle'); + providerCheckboxes.forEach(checkbox => { + checkbox.checked = true; + this.updateCheckboxAppearance(checkbox); + }); + + // Reset recursion depth to default + const depthSelect = document.getElementById('max-depth'); + if (depthSelect) { + depthSelect.value = '2'; + } + + this.showInfo('Settings reset to defaults'); + + } catch (error) { + this.showError(`Error resetting settings: ${error.message}`); + } + } /** * Update provider display @@ -1501,33 +1915,6 @@ class DNSReconApp { } } - /** - * Copy text to clipboard with user feedback - */ - async copyToClipboard(text) { - try { - await navigator.clipboard.writeText(text); - this.showMessage('Copied to clipboard', 'success'); - } catch (err) { - // Fallback for older browsers - const textArea = document.createElement('textarea'); - textArea.value = text; - textArea.style.position = 'fixed'; - textArea.style.opacity = '0'; - document.body.appendChild(textArea); - textArea.select(); - - try { - document.execCommand('copy'); - this.showMessage('Copied to clipboard', 'success'); - } catch (fallbackErr) { - this.showMessage('Failed to copy text', 'error'); - } - - document.body.removeChild(textArea); - } - } - /** * Enhanced keyboard navigation for modals */ @@ -1690,40 +2077,7 @@ class DNSReconApp { }); } - /** - * Check if graph data has changed - * @param {Object} graphData - New graph data - * @returns {boolean} True if data has changed - */ - hasGraphChanged(graphData) { - // Simple check based on node and edge counts and timestamps - const currentStats = this.graphManager.getStatistics(); - const newNodeCount = graphData.nodes ? graphData.nodes.length : 0; - const newEdgeCount = graphData.edges ? graphData.edges.length : 0; - - // FIXED: Always update if we currently have no data (ensures placeholder is handled correctly) - if (currentStats.nodeCount === 0 && currentStats.edgeCount === 0) { - return true; - } - - // Check if counts changed - const countsChanged = currentStats.nodeCount !== newNodeCount || currentStats.edgeCount !== newEdgeCount; - - // Also check if we have new timestamp data - const hasNewTimestamp = graphData.statistics && - graphData.statistics.last_modified && - graphData.statistics.last_modified !== this.lastGraphTimestamp; - - if (hasNewTimestamp) { - this.lastGraphTimestamp = graphData.statistics.last_modified; - } - - const changed = countsChanged || hasNewTimestamp; - - console.log(`Graph change check: Current(${currentStats.nodeCount}n, ${currentStats.edgeCount}e) vs New(${newNodeCount}n, ${newEdgeCount}e) = ${changed}`); - - return changed; - } + /** * Make API call to server @@ -1823,15 +2177,6 @@ class DNSReconApp { return statusMap[status] || status; } - /** - * Format label for display - * @param {string} label - Raw label - * @returns {string} Formatted label - */ - formatLabel(label) { - return label.replace(/_/g, ' ').replace(/\b\w/g, l => l.toUpperCase()); - } - /** * Format value for display * @param {*} value - Raw value @@ -1955,73 +2300,7 @@ class DNSReconApp { return colors[type] || colors.info; } - /** - * Build the API key modal dynamically - * @param {Object} providers - Provider information - */ - buildApiKeyModal(providers) { - if (!this.elements.apiKeyInputs) return; - this.elements.apiKeyInputs.innerHTML = ''; // Clear existing inputs - let hasApiKeyProviders = false; - for (const [name, info] of Object.entries(providers)) { - if (info.requires_api_key) { - hasApiKeyProviders = true; - const inputGroup = document.createElement('div'); - inputGroup.className = 'apikey-section'; - - if (info.enabled) { - // If the API key is set and the provider is enabled - inputGroup.innerHTML = ` - -
- API Key is set - -
-

Provides infrastructure context and service information.

- `; - } else { - // If the API key is not set - inputGroup.innerHTML = ` - - -

Provides infrastructure context and service information.

- `; - } - this.elements.apiKeyInputs.appendChild(inputGroup); - } - } - - // Add event listeners for the new clear buttons - this.elements.apiKeyInputs.querySelectorAll('.clear-api-key-btn').forEach(button => { - button.addEventListener('click', (e) => { - const provider = e.target.dataset.provider; - this.clearApiKey(provider); - }); - }); - - if (!hasApiKeyProviders) { - this.elements.apiKeyInputs.innerHTML = '

No providers require API keys.

'; - } - } - - /** - * Clear an API key for a specific provider - * @param {string} provider The name of the provider to clear the API key for - */ - async clearApiKey(provider) { - try { - const response = await this.apiCall('/api/config/api-keys', 'POST', { [provider]: '' }); - if (response.success) { - this.showSuccess(`API key for ${provider} has been cleared.`); - this.loadProviders(); // This will rebuild the modal with the updated state - } else { - throw new Error(response.error || 'Failed to clear API key'); - } - } catch (error) { - this.showError(`Error clearing API key: ${error.message}`); - } - } } // Add CSS animations for message toasts diff --git a/templates/index.html b/templates/index.html index 730ab0b..1f9fad7 100644 --- a/templates/index.html +++ b/templates/index.html @@ -1,5 +1,6 @@ + @@ -7,8 +8,11 @@ - + +
@@ -29,13 +33,13 @@

Target Configuration

- +
- +
+