From 52ea7acf04480cbbe13a9e0b40e0a3368a96f573 Mon Sep 17 00:00:00 2001 From: overcuriousity Date: Mon, 22 Sep 2025 15:34:35 +0200 Subject: [PATCH] performance optimization --- .env.example | 2 + app.py | 16 +++++ config.py | 6 ++ static/css/main.css | 12 ++++ static/js/graph.js | 36 +++++++++- static/js/main.js | 167 ++++++++++++++++++++++++++++++++++++-------- 6 files changed, 207 insertions(+), 32 deletions(-) diff --git a/.env.example b/.env.example index 47ee018..849ccf2 100644 --- a/.env.example +++ b/.env.example @@ -32,3 +32,5 @@ LARGE_ENTITY_THRESHOLD=100 MAX_RETRIES_PER_TARGET=8 # How long cached provider responses are stored (in hours). CACHE_TIMEOUT_HOURS=12 + +GRAPH_POLLING_NODE_THRESHOLD=100 diff --git a/app.py b/app.py index 7ab2466..92ebacc 100644 --- a/app.py +++ b/app.py @@ -3,6 +3,7 @@ """ Flask application entry point for DNSRecon web interface. Provides REST API endpoints and serves the web interface with user session support. +UPDATED: Added /api/config endpoint for graph polling optimization settings. """ import json @@ -53,6 +54,21 @@ def index(): return render_template('index.html') +@app.route('/api/config', methods=['GET']) +def get_config(): + """Get configuration settings for frontend.""" + try: + return jsonify({ + 'success': True, + 'config': { + 'graph_polling_node_threshold': config.graph_polling_node_threshold + } + }) + except Exception as e: + traceback.print_exc() + return jsonify({'success': False, 'error': f'Internal server error: {str(e)}'}), 500 + + @app.route('/api/scan/start', methods=['POST']) def start_scan(): """ diff --git a/config.py b/config.py index 3f4eb3f..2a2c323 100644 --- a/config.py +++ b/config.py @@ -26,6 +26,9 @@ class Config: self.large_entity_threshold = 100 self.max_retries_per_target = 8 + # --- Graph Polling Performance Settings --- + self.graph_polling_node_threshold = 100 # Stop graph auto-polling above this many nodes + # --- Provider Caching Settings --- self.cache_timeout_hours = 6 # Provider-specific cache timeout @@ -72,6 +75,9 @@ class Config: self.max_retries_per_target = int(os.getenv('MAX_RETRIES_PER_TARGET', self.max_retries_per_target)) self.cache_timeout_hours = int(os.getenv('CACHE_TIMEOUT_HOURS', self.cache_timeout_hours)) + # Override graph polling threshold from environment + self.graph_polling_node_threshold = int(os.getenv('GRAPH_POLLING_NODE_THRESHOLD', self.graph_polling_node_threshold)) + # Override Flask and session settings self.flask_host = os.getenv('FLASK_HOST', self.flask_host) self.flask_port = int(os.getenv('FLASK_PORT', self.flask_port)) diff --git a/static/css/main.css b/static/css/main.css index 7327d87..853ea1b 100644 --- a/static/css/main.css +++ b/static/css/main.css @@ -1324,4 +1324,16 @@ input[type="password"]:focus { .provider-list { grid-template-columns: 1fr; } +} + +.manual-refresh-btn { + background: rgba(92, 76, 44, 0.9) !important; /* Orange/amber background */ + border: 1px solid #7a6a3a !important; + color: #ffcc00 !important; /* Bright yellow text */ +} + +.manual-refresh-btn:hover { + border-color: #ffcc00 !important; + color: #fff !important; + background: rgba(112, 96, 54, 0.9) !important; } \ No newline at end of file diff --git a/static/js/graph.js b/static/js/graph.js index d2b8ae6..c00f182 100644 --- a/static/js/graph.js +++ b/static/js/graph.js @@ -2,7 +2,7 @@ /** * Graph visualization module for DNSRecon * Handles network graph rendering using vis.js with proper large entity node hiding - * UPDATED: Now compatible with a strictly flat, unified data model for attributes. + * UPDATED: Added manual refresh button for polling optimization when graph becomes large */ const contextMenuCSS = ` .graph-context-menu { @@ -71,6 +71,9 @@ class GraphManager { // Track large entity members for proper hiding this.largeEntityMembers = new Set(); this.isScanning = false; + + // Manual refresh button for polling optimization + this.manualRefreshButton = null; this.options = { nodes: { @@ -254,6 +257,7 @@ class GraphManager { /** * Add interactive graph controls + * UPDATED: Added manual refresh button for polling optimization */ addGraphControls() { const controlsContainer = document.createElement('div'); @@ -264,6 +268,9 @@ class GraphManager { + `; this.container.appendChild(controlsContainer); @@ -274,6 +281,29 @@ class GraphManager { document.getElementById('graph-cluster').addEventListener('click', () => this.toggleClustering()); document.getElementById('graph-unhide').addEventListener('click', () => this.unhideAll()); document.getElementById('graph-revert').addEventListener('click', () => this.revertLastAction()); + + // Manual refresh button - handler will be set by main app + this.manualRefreshButton = document.getElementById('graph-manual-refresh'); + } + + /** + * Set the manual refresh button click handler + * @param {Function} handler - Function to call when manual refresh is clicked + */ + setManualRefreshHandler(handler) { + if (this.manualRefreshButton && typeof handler === 'function') { + this.manualRefreshButton.addEventListener('click', handler); + } + } + + /** + * Show or hide the manual refresh button + * @param {boolean} show - Whether to show the button + */ + showManualRefreshButton(show) { + if (this.manualRefreshButton) { + this.manualRefreshButton.style.display = show ? 'inline-block' : 'none'; + } } addFilterPanel() { @@ -607,7 +637,7 @@ class GraphManager { formatEdgeLabel(relationshipType, confidence) { if (!relationshipType) return ''; - const confidenceText = confidence >= 0.8 ? '●' : confidence >= 0.6 ? '◐' : '○'; + const confidenceText = confidence >= 0.8 ? '●' : confidence >= 0.6 ? '●' : '○'; return `${relationshipType} ${confidenceText}`; } @@ -1417,7 +1447,7 @@ class GraphManager { menuItems += `
  • - 👁️‍🗨️ + 👻 Hide Node
  • diff --git a/static/js/main.js b/static/js/main.js index 0721d53..205fa1d 100644 --- a/static/js/main.js +++ b/static/js/main.js @@ -9,7 +9,8 @@ class DNSReconApp { console.log('DNSReconApp constructor called'); this.graphManager = null; this.scanStatus = 'idle'; - this.pollInterval = null; + this.statusPollInterval = null; // Separate status polling + this.graphPollInterval = null; // Separate graph polling this.currentSessionId = null; this.elements = {}; @@ -17,6 +18,10 @@ class DNSReconApp { this.isScanning = false; this.lastGraphUpdate = null; + // Graph polling optimization + this.graphPollingNodeThreshold = 500; // Default, will be loaded from config + this.graphPollingEnabled = true; + this.init(); } @@ -35,6 +40,7 @@ class DNSReconApp { this.loadProviders(); this.initializeEnhancedModals(); this.addCheckboxStyling(); + this.loadConfig(); // Load configuration including threshold this.updateGraph(); @@ -45,6 +51,21 @@ class DNSReconApp { } }); } + + /** + * Load configuration from backend + */ + async loadConfig() { + try { + const response = await this.apiCall('/api/config'); + if (response.success) { + this.graphPollingNodeThreshold = response.config.graph_polling_node_threshold; + console.log(`Graph polling threshold set to: ${this.graphPollingNodeThreshold} nodes`); + } + } catch (error) { + console.warn('Failed to load config, using defaults:', error); + } + } /** * Initialize DOM element references @@ -263,18 +284,53 @@ class DNSReconApp { } /** - * Initialize graph visualization + * Initialize graph visualization with manual refresh button */ initializeGraph() { try { console.log('Initializing graph manager...'); this.graphManager = new GraphManager('network-graph'); + + // Set up manual refresh handler + this.graphManager.setManualRefreshHandler(() => { + console.log('Manual graph refresh requested'); + this.updateGraph(); + }); + console.log('Graph manager initialized successfully'); } catch (error) { console.error('Failed to initialize graph manager:', error); this.showError('Failed to initialize graph visualization'); } } + + /** + * Check if graph polling should be enabled based on node count + */ + shouldEnableGraphPolling() { + if (!this.graphManager || !this.graphManager.nodes) { + return true; + } + + const nodeCount = this.graphManager.nodes.length; + return nodeCount <= this.graphPollingNodeThreshold; + } + + /** + * Update manual refresh button visibility and state. + * The button will be visible whenever auto-polling is disabled, + * and enabled only when a scan is in progress. + */ + updateManualRefreshButton() { + if (!this.graphManager || !this.graphManager.manualRefreshButton) return; + + const shouldShow = !this.graphPollingEnabled; + this.graphManager.showManualRefreshButton(shouldShow); + + if (shouldShow) { + this.graphManager.manualRefreshButton.disabled = false; + } + } /** * Start scan with error handling @@ -324,18 +380,21 @@ class DNSReconApp { if (clearGraph) { this.graphManager.clear(); + this.graphPollingEnabled = true; // Reset polling when starting fresh } console.log(`Scan started for ${target} with depth ${maxDepth}`); - // Start polling immediately with faster interval for responsiveness - this.startPolling(); + // Start optimized polling + this.startOptimizedPolling(); // Force an immediate status update console.log('Forcing immediate status update...'); setTimeout(() => { this.updateStatus(); - this.updateGraph(); + if (this.graphPollingEnabled) { + this.updateGraph(); + } }, 100); } else { @@ -348,6 +407,35 @@ class DNSReconApp { this.setUIState('idle'); } } + + /** + * Start optimized polling with separate status and graph intervals + */ + startOptimizedPolling() { + console.log('=== STARTING OPTIMIZED POLLING ==='); + + this.stopPolling(); // Stop any existing polling + + // Always poll status for progress bar + this.statusPollInterval = setInterval(() => { + this.updateStatus(); + this.loadProviders(); + }, 2000); + + // Only poll graph if enabled + if (this.graphPollingEnabled) { + this.graphPollInterval = setInterval(() => { + this.updateGraph(); + }, 2000); + console.log('Graph polling started'); + } else { + console.log('Graph polling disabled due to node count'); + } + + this.updateManualRefreshButton(); + console.log(`Optimized polling started - Status: enabled, Graph: ${this.graphPollingEnabled ? 'enabled' : 'disabled'}`); + } + /** * Scan stop with immediate UI feedback */ @@ -374,15 +462,8 @@ class DNSReconApp { this.updateStatus(); }, 100); - // Continue polling for a bit to catch the status change - this.startPolling(500); // Fast polling to catch status change - - // Stop fast polling after 10 seconds - setTimeout(() => { - if (this.scanStatus === 'stopped' || this.scanStatus === 'idle') { - this.stopPolling(); - } - }, 10000); + // Continue status polling for a bit to catch the status change + // No need to resume graph polling } else { throw new Error(response.error || 'Failed to stop scan'); @@ -573,12 +654,20 @@ class DNSReconApp { */ stopPolling() { console.log('=== STOPPING POLLING ==='); - if (this.pollInterval) { - clearInterval(this.pollInterval); - this.pollInterval = null; + + if (this.statusPollInterval) { + clearInterval(this.statusPollInterval); + this.statusPollInterval = null; } + + if (this.graphPollInterval) { + clearInterval(this.graphPollInterval); + this.graphPollInterval = null; + } + + this.updateManualRefreshButton(); } - + /** * Status update with better error handling */ @@ -611,7 +700,7 @@ class DNSReconApp { } /** - * Update graph from server + * Update graph from server with polling optimization */ async updateGraph() { try { @@ -626,11 +715,29 @@ class DNSReconApp { console.log('- Nodes:', graphData.nodes ? graphData.nodes.length : 0); console.log('- Edges:', graphData.edges ? graphData.edges.length : 0); - // FIXED: Always update graph, even if empty - let GraphManager handle placeholder + // Always update graph, even if empty - let GraphManager handle placeholder if (this.graphManager) { this.graphManager.updateGraph(graphData); this.lastGraphUpdate = Date.now(); + // Check if we should disable graph polling after this update + const nodeCount = graphData.nodes ? graphData.nodes.length : 0; + const shouldEnablePolling = nodeCount <= this.graphPollingNodeThreshold; + + if (this.graphPollingEnabled && !shouldEnablePolling) { + console.log(`Graph polling disabled: ${nodeCount} nodes exceeds threshold of ${this.graphPollingNodeThreshold}`); + this.graphPollingEnabled = false; + this.showWarning(`Graph auto-refresh disabled: ${nodeCount} nodes exceed threshold of ${this.graphPollingNodeThreshold}. Use manual refresh button.`); + + // Stop graph polling but keep status polling + if (this.graphPollInterval) { + clearInterval(this.graphPollInterval); + this.graphPollInterval = null; + } + + this.updateManualRefreshButton(); + } + // Update relationship count in status const edgeCount = graphData.edges ? graphData.edges.length : 0; if (this.elements.relationshipsDisplay) { @@ -639,7 +746,7 @@ class DNSReconApp { } } else { console.error('Graph update failed:', response); - // FIXED: Show placeholder when graph update fails + // Show placeholder when graph update fails if (this.graphManager && this.graphManager.container) { const placeholder = this.graphManager.container.querySelector('.graph-placeholder'); if (placeholder) { @@ -650,7 +757,7 @@ class DNSReconApp { } catch (error) { console.error('Failed to update graph:', error); - // FIXED: Show placeholder on error + // Show placeholder on error if (this.graphManager && this.graphManager.container) { const placeholder = this.graphManager.container.querySelector('.graph-placeholder'); if (placeholder) { @@ -659,7 +766,6 @@ class DNSReconApp { } } } - /** * Update status display elements @@ -737,8 +843,6 @@ class DNSReconApp { case 'running': this.setUIState('scanning', task_queue_size); this.showSuccess('Scan is running'); - // Increase polling frequency for active scans - this.startPolling(5000); // Poll every 5 second for running scans this.updateConnectionStatus('active'); break; @@ -748,8 +852,9 @@ class DNSReconApp { this.showSuccess('Scan completed successfully'); this.updateConnectionStatus('completed'); this.loadProviders(); - // Force a final graph update - console.log('Scan completed - forcing final graph update'); + + // Do final graph update when scan completes + console.log('Scan completed - performing final graph update'); setTimeout(() => this.updateGraph(), 100); break; @@ -820,6 +925,7 @@ class DNSReconApp { switch (state) { case 'scanning': + case 'running': this.isScanning = true; if (this.graphManager) { this.graphManager.isScanning = true; @@ -852,12 +958,12 @@ class DNSReconApp { this.graphManager.isScanning = false; } if (this.elements.startScan) { - this.elements.startScan.disabled = !isQueueEmpty; + this.elements.startScan.disabled = false; this.elements.startScan.classList.remove('loading'); this.elements.startScan.innerHTML = '[RUN]Start Reconnaissance'; } if (this.elements.addToGraph) { - this.elements.addToGraph.disabled = !isQueueEmpty; + this.elements.addToGraph.disabled = false; this.elements.addToGraph.classList.remove('loading'); } if (this.elements.stopScan) { @@ -867,6 +973,9 @@ class DNSReconApp { if (this.elements.targetInput) this.elements.targetInput.disabled = false; if (this.elements.maxDepth) this.elements.maxDepth.disabled = false; if (this.elements.configureSettings) this.elements.configureSettings.disabled = false; + + // Update manual refresh button visibility + this.updateManualRefreshButton(); break; } }