mstoeck3/dnscope
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

performance optimization

Browse Source
This commit is contained in:
overcuriousity 2025-09-22 15:34:35 +02:00
parent 513eff12ef
commit 52ea7acf04
6 changed files with 207 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.env.example
View File

@ -32,3 +32,5 @@ LARGE_ENTITY_THRESHOLD=100
MAX_RETRIES_PER_TARGET=8 MAX_RETRIES_PER_TARGET=8
# How long cached provider responses are stored (in hours). # How long cached provider responses are stored (in hours).
CACHE_TIMEOUT_HOURS=12 CACHE_TIMEOUT_HOURS=12
GRAPH_POLLING_NODE_THRESHOLD=100

16
app.py
View File

@ -3,6 +3,7 @@
""" """
Flask application entry point for DNSRecon web interface. Flask application entry point for DNSRecon web interface.
Provides REST API endpoints and serves the web interface with user session support. Provides REST API endpoints and serves the web interface with user session support.
UPDATED: Added /api/config endpoint for graph polling optimization settings.
""" """
import json import json
@ -53,6 +54,21 @@ def index():
return render_template('index.html') return render_template('index.html')
@app.route('/api/config', methods=['GET'])
def get_config():
"""Get configuration settings for frontend."""
try:
return jsonify({
'success': True,
'config': {
'graph_polling_node_threshold': config.graph_polling_node_threshold
}
})
except Exception as e:
traceback.print_exc()
return jsonify({'success': False, 'error': f'Internal server error: {str(e)}'}), 500
@app.route('/api/scan/start', methods=['POST']) @app.route('/api/scan/start', methods=['POST'])
def start_scan(): def start_scan():
""" """

6
config.py
View File

@ -26,6 +26,9 @@ class Config:
self.large_entity_threshold = 100 self.large_entity_threshold = 100
self.max_retries_per_target = 8 self.max_retries_per_target = 8
# --- Graph Polling Performance Settings ---
self.graph_polling_node_threshold = 100 # Stop graph auto-polling above this many nodes
# --- Provider Caching Settings --- # --- Provider Caching Settings ---
self.cache_timeout_hours = 6 # Provider-specific cache timeout self.cache_timeout_hours = 6 # Provider-specific cache timeout
@ -72,6 +75,9 @@ class Config:
self.max_retries_per_target = int(os.getenv('MAX_RETRIES_PER_TARGET', self.max_retries_per_target)) self.max_retries_per_target = int(os.getenv('MAX_RETRIES_PER_TARGET', self.max_retries_per_target))
self.cache_timeout_hours = int(os.getenv('CACHE_TIMEOUT_HOURS', self.cache_timeout_hours)) self.cache_timeout_hours = int(os.getenv('CACHE_TIMEOUT_HOURS', self.cache_timeout_hours))
# Override graph polling threshold from environment
self.graph_polling_node_threshold = int(os.getenv('GRAPH_POLLING_NODE_THRESHOLD', self.graph_polling_node_threshold))
# Override Flask and session settings # Override Flask and session settings
self.flask_host = os.getenv('FLASK_HOST', self.flask_host) self.flask_host = os.getenv('FLASK_HOST', self.flask_host)
self.flask_port = int(os.getenv('FLASK_PORT', self.flask_port)) self.flask_port = int(os.getenv('FLASK_PORT', self.flask_port))

12
static/css/main.css
View File

@ -1325,3 +1325,15 @@ input[type="password"]:focus {
grid-template-columns: 1fr; grid-template-columns: 1fr;
} }
} }
.manual-refresh-btn {
background: rgba(92, 76, 44, 0.9) !important; /* Orange/amber background */
border: 1px solid #7a6a3a !important;
color: #ffcc00 !important; /* Bright yellow text */
}
.manual-refresh-btn:hover {
border-color: #ffcc00 !important;
color: #fff !important;
background: rgba(112, 96, 54, 0.9) !important;
}

36
static/js/graph.js
View File

@ -2,7 +2,7 @@
/** /**
* Graph visualization module for DNSRecon * Graph visualization module for DNSRecon
* Handles network graph rendering using vis.js with proper large entity node hiding * Handles network graph rendering using vis.js with proper large entity node hiding
* UPDATED: Now compatible with a strictly flat, unified data model for attributes. * UPDATED: Added manual refresh button for polling optimization when graph becomes large
*/ */
const contextMenuCSS = ` const contextMenuCSS = `
.graph-context-menu { .graph-context-menu {
@ -72,6 +72,9 @@ class GraphManager {
this.largeEntityMembers = new Set(); this.largeEntityMembers = new Set();
this.isScanning = false; this.isScanning = false;
// Manual refresh button for polling optimization
this.manualRefreshButton = null;
this.options = { this.options = {
nodes: { nodes: {
shape: 'dot', shape: 'dot',
@ -254,6 +257,7 @@ class GraphManager {
/** /**
* Add interactive graph controls * Add interactive graph controls
* UPDATED: Added manual refresh button for polling optimization
*/ */
addGraphControls() { addGraphControls() {
const controlsContainer = document.createElement('div'); const controlsContainer = document.createElement('div');
@ -264,6 +268,9 @@ class GraphManager {
<button class="graph-control-btn" id="graph-cluster" title="Cluster Nodes">[CLUSTER]</button> <button class="graph-control-btn" id="graph-cluster" title="Cluster Nodes">[CLUSTER]</button>
<button class="graph-control-btn" id="graph-unhide" title="Unhide All">[UNHIDE]</button> <button class="graph-control-btn" id="graph-unhide" title="Unhide All">[UNHIDE]</button>
<button class="graph-control-btn" id="graph-revert" title="Revert Last Action">[REVERT]</button> <button class="graph-control-btn" id="graph-revert" title="Revert Last Action">[REVERT]</button>
<button class="graph-control-btn manual-refresh-btn" id="graph-manual-refresh"
title="Manual Refresh - Auto-refresh disabled due to large graph"
style="display: none;">[REFRESH]</button>
`; `;
this.container.appendChild(controlsContainer); this.container.appendChild(controlsContainer);
@ -274,6 +281,29 @@ class GraphManager {
document.getElementById('graph-cluster').addEventListener('click', () => this.toggleClustering()); document.getElementById('graph-cluster').addEventListener('click', () => this.toggleClustering());
document.getElementById('graph-unhide').addEventListener('click', () => this.unhideAll()); document.getElementById('graph-unhide').addEventListener('click', () => this.unhideAll());
document.getElementById('graph-revert').addEventListener('click', () => this.revertLastAction()); document.getElementById('graph-revert').addEventListener('click', () => this.revertLastAction());
// Manual refresh button - handler will be set by main app
this.manualRefreshButton = document.getElementById('graph-manual-refresh');
}
/**
* Set the manual refresh button click handler
* @param {Function} handler - Function to call when manual refresh is clicked
*/
setManualRefreshHandler(handler) {
if (this.manualRefreshButton && typeof handler === 'function') {
this.manualRefreshButton.addEventListener('click', handler);
}
}
/**
* Show or hide the manual refresh button
* @param {boolean} show - Whether to show the button
*/
showManualRefreshButton(show) {
if (this.manualRefreshButton) {
this.manualRefreshButton.style.display = show ? 'inline-block' : 'none';
}
} }
addFilterPanel() { addFilterPanel() {
@ -607,7 +637,7 @@ class GraphManager {
formatEdgeLabel(relationshipType, confidence) { formatEdgeLabel(relationshipType, confidence) {
if (!relationshipType) return ''; if (!relationshipType) return '';
const confidenceText = confidence >= 0.8 ? '●' : confidence >= 0.6 ? '◐' : '○'; const confidenceText = confidence >= 0.8 ? '●' : confidence >= 0.6 ? '●' : '○';
return `${relationshipType} ${confidenceText}`; return `${relationshipType} ${confidenceText}`;
} }
@ -1417,7 +1447,7 @@ class GraphManager {
menuItems += ` menuItems += `
<li data-action="hide" data-node-id="${nodeId}"> <li data-action="hide" data-node-id="${nodeId}">
<span class="menu-icon">👁️‍🗨️</span> <span class="menu-icon">👻</span>
<span>Hide Node</span> <span>Hide Node</span>
</li> </li>
<li data-action="delete" data-node-id="${nodeId}"> <li data-action="delete" data-node-id="${nodeId}">

165
static/js/main.js
View File

@ -9,7 +9,8 @@ class DNSReconApp {
console.log('DNSReconApp constructor called'); console.log('DNSReconApp constructor called');
this.graphManager = null; this.graphManager = null;
this.scanStatus = 'idle'; this.scanStatus = 'idle';
this.pollInterval = null; this.statusPollInterval = null; // Separate status polling
this.graphPollInterval = null; // Separate graph polling
this.currentSessionId = null; this.currentSessionId = null;
this.elements = {}; this.elements = {};
@ -17,6 +18,10 @@ class DNSReconApp {
this.isScanning = false; this.isScanning = false;
this.lastGraphUpdate = null; this.lastGraphUpdate = null;
// Graph polling optimization
this.graphPollingNodeThreshold = 500; // Default, will be loaded from config
this.graphPollingEnabled = true;
this.init(); this.init();
} }
@ -35,6 +40,7 @@ class DNSReconApp {
this.loadProviders(); this.loadProviders();
this.initializeEnhancedModals(); this.initializeEnhancedModals();
this.addCheckboxStyling(); this.addCheckboxStyling();
this.loadConfig(); // Load configuration including threshold
this.updateGraph(); this.updateGraph();
@ -46,6 +52,21 @@ class DNSReconApp {
}); });
} }
/**
* Load configuration from backend
*/
async loadConfig() {
try {
const response = await this.apiCall('/api/config');
if (response.success) {
this.graphPollingNodeThreshold = response.config.graph_polling_node_threshold;
console.log(`Graph polling threshold set to: ${this.graphPollingNodeThreshold} nodes`);
}
} catch (error) {
console.warn('Failed to load config, using defaults:', error);
}
}
/** /**
* Initialize DOM element references * Initialize DOM element references
*/ */
@ -263,12 +284,19 @@ class DNSReconApp {
} }
/** /**
* Initialize graph visualization * Initialize graph visualization with manual refresh button
*/ */
initializeGraph() { initializeGraph() {
try { try {
console.log('Initializing graph manager...'); console.log('Initializing graph manager...');
this.graphManager = new GraphManager('network-graph'); this.graphManager = new GraphManager('network-graph');
// Set up manual refresh handler
this.graphManager.setManualRefreshHandler(() => {
console.log('Manual graph refresh requested');
this.updateGraph();
});
console.log('Graph manager initialized successfully'); console.log('Graph manager initialized successfully');
} catch (error) { } catch (error) {
console.error('Failed to initialize graph manager:', error); console.error('Failed to initialize graph manager:', error);
@ -276,6 +304,34 @@ class DNSReconApp {
} }
} }
/**
* Check if graph polling should be enabled based on node count
*/
shouldEnableGraphPolling() {
if (!this.graphManager || !this.graphManager.nodes) {
return true;
}
const nodeCount = this.graphManager.nodes.length;
return nodeCount <= this.graphPollingNodeThreshold;
}
/**
* Update manual refresh button visibility and state.
* The button will be visible whenever auto-polling is disabled,
* and enabled only when a scan is in progress.
*/
updateManualRefreshButton() {
if (!this.graphManager || !this.graphManager.manualRefreshButton) return;
const shouldShow = !this.graphPollingEnabled;
this.graphManager.showManualRefreshButton(shouldShow);
if (shouldShow) {
this.graphManager.manualRefreshButton.disabled = false;
}
}
/** /**
* Start scan with error handling * Start scan with error handling
*/ */
@ -324,18 +380,21 @@ class DNSReconApp {
if (clearGraph) { if (clearGraph) {
this.graphManager.clear(); this.graphManager.clear();
this.graphPollingEnabled = true; // Reset polling when starting fresh
} }
console.log(`Scan started for ${target} with depth ${maxDepth}`); console.log(`Scan started for ${target} with depth ${maxDepth}`);
// Start polling immediately with faster interval for responsiveness // Start optimized polling
this.startPolling(); this.startOptimizedPolling();
// Force an immediate status update // Force an immediate status update
console.log('Forcing immediate status update...'); console.log('Forcing immediate status update...');
setTimeout(() => { setTimeout(() => {
this.updateStatus(); this.updateStatus();
this.updateGraph(); if (this.graphPollingEnabled) {
this.updateGraph();
}
}, 100); }, 100);
} else { } else {
@ -348,6 +407,35 @@ class DNSReconApp {
this.setUIState('idle'); this.setUIState('idle');
} }
} }
/**
* Start optimized polling with separate status and graph intervals
*/
startOptimizedPolling() {
console.log('=== STARTING OPTIMIZED POLLING ===');
this.stopPolling(); // Stop any existing polling
// Always poll status for progress bar
this.statusPollInterval = setInterval(() => {
this.updateStatus();
this.loadProviders();
}, 2000);
// Only poll graph if enabled
if (this.graphPollingEnabled) {
this.graphPollInterval = setInterval(() => {
this.updateGraph();
}, 2000);
console.log('Graph polling started');
} else {
console.log('Graph polling disabled due to node count');
}
this.updateManualRefreshButton();
console.log(`Optimized polling started - Status: enabled, Graph: ${this.graphPollingEnabled ? 'enabled' : 'disabled'}`);
}
/** /**
* Scan stop with immediate UI feedback * Scan stop with immediate UI feedback
*/ */
@ -374,15 +462,8 @@ class DNSReconApp {
this.updateStatus(); this.updateStatus();
}, 100); }, 100);
// Continue polling for a bit to catch the status change // Continue status polling for a bit to catch the status change
this.startPolling(500); // Fast polling to catch status change // No need to resume graph polling
// Stop fast polling after 10 seconds
setTimeout(() => {
if (this.scanStatus === 'stopped' || this.scanStatus === 'idle') {
this.stopPolling();
}
}, 10000);
} else { } else {
throw new Error(response.error || 'Failed to stop scan'); throw new Error(response.error || 'Failed to stop scan');
@ -573,10 +654,18 @@ class DNSReconApp {
*/ */
stopPolling() { stopPolling() {
console.log('=== STOPPING POLLING ==='); console.log('=== STOPPING POLLING ===');
if (this.pollInterval) {
clearInterval(this.pollInterval); if (this.statusPollInterval) {
this.pollInterval = null; clearInterval(this.statusPollInterval);
this.statusPollInterval = null;
} }
if (this.graphPollInterval) {
clearInterval(this.graphPollInterval);
this.graphPollInterval = null;
}
this.updateManualRefreshButton();
} }
/** /**
@ -611,7 +700,7 @@ class DNSReconApp {
} }
/** /**
* Update graph from server * Update graph from server with polling optimization
*/ */
async updateGraph() { async updateGraph() {
try { try {
@ -626,11 +715,29 @@ class DNSReconApp {
console.log('- Nodes:', graphData.nodes ? graphData.nodes.length : 0); console.log('- Nodes:', graphData.nodes ? graphData.nodes.length : 0);
console.log('- Edges:', graphData.edges ? graphData.edges.length : 0); console.log('- Edges:', graphData.edges ? graphData.edges.length : 0);
// FIXED: Always update graph, even if empty - let GraphManager handle placeholder // Always update graph, even if empty - let GraphManager handle placeholder
if (this.graphManager) { if (this.graphManager) {
this.graphManager.updateGraph(graphData); this.graphManager.updateGraph(graphData);
this.lastGraphUpdate = Date.now(); this.lastGraphUpdate = Date.now();
// Check if we should disable graph polling after this update
const nodeCount = graphData.nodes ? graphData.nodes.length : 0;
const shouldEnablePolling = nodeCount <= this.graphPollingNodeThreshold;
if (this.graphPollingEnabled && !shouldEnablePolling) {
console.log(`Graph polling disabled: ${nodeCount} nodes exceeds threshold of ${this.graphPollingNodeThreshold}`);
this.graphPollingEnabled = false;
this.showWarning(`Graph auto-refresh disabled: ${nodeCount} nodes exceed threshold of ${this.graphPollingNodeThreshold}. Use manual refresh button.`);
// Stop graph polling but keep status polling
if (this.graphPollInterval) {
clearInterval(this.graphPollInterval);
this.graphPollInterval = null;
}
this.updateManualRefreshButton();
}
// Update relationship count in status // Update relationship count in status
const edgeCount = graphData.edges ? graphData.edges.length : 0; const edgeCount = graphData.edges ? graphData.edges.length : 0;
if (this.elements.relationshipsDisplay) { if (this.elements.relationshipsDisplay) {
@ -639,7 +746,7 @@ class DNSReconApp {
} }
} else { } else {
console.error('Graph update failed:', response); console.error('Graph update failed:', response);
// FIXED: Show placeholder when graph update fails // Show placeholder when graph update fails
if (this.graphManager && this.graphManager.container) { if (this.graphManager && this.graphManager.container) {
const placeholder = this.graphManager.container.querySelector('.graph-placeholder'); const placeholder = this.graphManager.container.querySelector('.graph-placeholder');
if (placeholder) { if (placeholder) {
@ -650,7 +757,7 @@ class DNSReconApp {
} catch (error) { } catch (error) {
console.error('Failed to update graph:', error); console.error('Failed to update graph:', error);
// FIXED: Show placeholder on error // Show placeholder on error
if (this.graphManager && this.graphManager.container) { if (this.graphManager && this.graphManager.container) {
const placeholder = this.graphManager.container.querySelector('.graph-placeholder'); const placeholder = this.graphManager.container.querySelector('.graph-placeholder');
if (placeholder) { if (placeholder) {
@ -660,7 +767,6 @@ class DNSReconApp {
} }
} }
/** /**
* Update status display elements * Update status display elements
* @param {Object} status - Status object from server * @param {Object} status - Status object from server
@ -737,8 +843,6 @@ class DNSReconApp {
case 'running': case 'running':
this.setUIState('scanning', task_queue_size); this.setUIState('scanning', task_queue_size);
this.showSuccess('Scan is running'); this.showSuccess('Scan is running');
// Increase polling frequency for active scans
this.startPolling(5000); // Poll every 5 second for running scans
this.updateConnectionStatus('active'); this.updateConnectionStatus('active');
break; break;
@ -748,8 +852,9 @@ class DNSReconApp {
this.showSuccess('Scan completed successfully'); this.showSuccess('Scan completed successfully');
this.updateConnectionStatus('completed'); this.updateConnectionStatus('completed');
this.loadProviders(); this.loadProviders();
// Force a final graph update
console.log('Scan completed - forcing final graph update'); // Do final graph update when scan completes
console.log('Scan completed - performing final graph update');
setTimeout(() => this.updateGraph(), 100); setTimeout(() => this.updateGraph(), 100);
break; break;
@ -820,6 +925,7 @@ class DNSReconApp {
switch (state) { switch (state) {
case 'scanning': case 'scanning':
case 'running':
this.isScanning = true; this.isScanning = true;
if (this.graphManager) { if (this.graphManager) {
this.graphManager.isScanning = true; this.graphManager.isScanning = true;
@ -852,12 +958,12 @@ class DNSReconApp {
this.graphManager.isScanning = false; this.graphManager.isScanning = false;
} }
if (this.elements.startScan) { if (this.elements.startScan) {
this.elements.startScan.disabled = !isQueueEmpty; this.elements.startScan.disabled = false;
this.elements.startScan.classList.remove('loading'); this.elements.startScan.classList.remove('loading');
this.elements.startScan.innerHTML = '<span class="btn-icon">[RUN]</span><span>Start Reconnaissance</span>'; this.elements.startScan.innerHTML = '<span class="btn-icon">[RUN]</span><span>Start Reconnaissance</span>';
} }
if (this.elements.addToGraph) { if (this.elements.addToGraph) {
this.elements.addToGraph.disabled = !isQueueEmpty; this.elements.addToGraph.disabled = false;
this.elements.addToGraph.classList.remove('loading'); this.elements.addToGraph.classList.remove('loading');
} }
if (this.elements.stopScan) { if (this.elements.stopScan) {
@ -867,6 +973,9 @@ class DNSReconApp {
if (this.elements.targetInput) this.elements.targetInput.disabled = false; if (this.elements.targetInput) this.elements.targetInput.disabled = false;
if (this.elements.maxDepth) this.elements.maxDepth.disabled = false; if (this.elements.maxDepth) this.elements.maxDepth.disabled = false;
if (this.elements.configureSettings) this.elements.configureSettings.disabled = false; if (this.elements.configureSettings) this.elements.configureSettings.disabled = false;
// Update manual refresh button visibility
this.updateManualRefreshButton();
break; break;
} }
} }