mstoeck3/dnscope
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

performance optimization

Browse Source
This commit is contained in:
overcuriousity 2025-09-22 15:34:35 +02:00
parent 513eff12ef
commit 52ea7acf04
6 changed files with 207 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.env.example
View File

@ -32,3 +32,5 @@ LARGE_ENTITY_THRESHOLD=100
MAX_RETRIES_PER_TARGET=8
# How long cached provider responses are stored (in hours).
CACHE_TIMEOUT_HOURS=12
GRAPH_POLLING_NODE_THRESHOLD=100

16
app.py
View File

@ -3,6 +3,7 @@
"""
Flask application entry point for DNSRecon web interface.
Provides REST API endpoints and serves the web interface with user session support.
UPDATED: Added /api/config endpoint for graph polling optimization settings.
"""
import json
@ -53,6 +54,21 @@ def index():
return render_template('index.html')
@app.route('/api/config', methods=['GET'])
def get_config():
"""Get configuration settings for frontend."""
try:
return jsonify({
'success': True,
'config': {
'graph_polling_node_threshold': config.graph_polling_node_threshold
}
})
except Exception as e:
traceback.print_exc()
return jsonify({'success': False, 'error': f'Internal server error: {str(e)}'}), 500
@app.route('/api/scan/start', methods=['POST'])
def start_scan():
"""

6
config.py
View File

@ -26,6 +26,9 @@ class Config:
self.large_entity_threshold = 100
self.max_retries_per_target = 8
# --- Graph Polling Performance Settings ---
self.graph_polling_node_threshold = 100 # Stop graph auto-polling above this many nodes
# --- Provider Caching Settings ---
self.cache_timeout_hours = 6 # Provider-specific cache timeout
@ -72,6 +75,9 @@ class Config:
self.max_retries_per_target = int(os.getenv('MAX_RETRIES_PER_TARGET', self.max_retries_per_target))
self.cache_timeout_hours = int(os.getenv('CACHE_TIMEOUT_HOURS', self.cache_timeout_hours))
# Override graph polling threshold from environment
self.graph_polling_node_threshold = int(os.getenv('GRAPH_POLLING_NODE_THRESHOLD', self.graph_polling_node_threshold))
# Override Flask and session settings
self.flask_host = os.getenv('FLASK_HOST', self.flask_host)
self.flask_port = int(os.getenv('FLASK_PORT', self.flask_port))

12
static/css/main.css
View File

@ -1325,3 +1325,15 @@ input[type="password"]:focus {
grid-template-columns: 1fr;
}
}
.manual-refresh-btn {
background: rgba(92, 76, 44, 0.9) !important; /* Orange/amber background */
border: 1px solid #7a6a3a !important;
color: #ffcc00 !important; /* Bright yellow text */
}
.manual-refresh-btn:hover {
border-color: #ffcc00 !important;
color: #fff !important;
background: rgba(112, 96, 54, 0.9) !important;
}

36
static/js/graph.js
View File

@ -2,7 +2,7 @@
/**
* Graph visualization module for DNSRecon
* Handles network graph rendering using vis.js with proper large entity node hiding
* UPDATED: Now compatible with a strictly flat, unified data model for attributes.
* UPDATED: Added manual refresh button for polling optimization when graph becomes large
*/
const contextMenuCSS = `
.graph-context-menu {
@ -72,6 +72,9 @@ class GraphManager {
this.largeEntityMembers = new Set();
this.isScanning = false;
// Manual refresh button for polling optimization
this.manualRefreshButton = null;
this.options = {
nodes: {
shape: 'dot',
@ -254,6 +257,7 @@ class GraphManager {
/**
* Add interactive graph controls
* UPDATED: Added manual refresh button for polling optimization
*/
addGraphControls() {
const controlsContainer = document.createElement('div');
@ -264,6 +268,9 @@ class GraphManager {
<button class="graph-control-btn" id="graph-cluster" title="Cluster Nodes">[CLUSTER]</button>
<button class="graph-control-btn" id="graph-unhide" title="Unhide All">[UNHIDE]</button>
<button class="graph-control-btn" id="graph-revert" title="Revert Last Action">[REVERT]</button>
<button class="graph-control-btn manual-refresh-btn" id="graph-manual-refresh"
title="Manual Refresh - Auto-refresh disabled due to large graph"
style="display: none;">[REFRESH]</button>
`;
this.container.appendChild(controlsContainer);
@ -274,6 +281,29 @@ class GraphManager {
document.getElementById('graph-cluster').addEventListener('click', () => this.toggleClustering());
document.getElementById('graph-unhide').addEventListener('click', () => this.unhideAll());
document.getElementById('graph-revert').addEventListener('click', () => this.revertLastAction());
// Manual refresh button - handler will be set by main app
this.manualRefreshButton = document.getElementById('graph-manual-refresh');
}
/**
* Set the manual refresh button click handler
* @param {Function} handler - Function to call when manual refresh is clicked
*/
setManualRefreshHandler(handler) {
if (this.manualRefreshButton && typeof handler === 'function') {
this.manualRefreshButton.addEventListener('click', handler);
}
}
/**
* Show or hide the manual refresh button
* @param {boolean} show - Whether to show the button
*/
showManualRefreshButton(show) {
if (this.manualRefreshButton) {
this.manualRefreshButton.style.display = show ? 'inline-block' : 'none';
}
}
addFilterPanel() {
@ -607,7 +637,7 @@ class GraphManager {
formatEdgeLabel(relationshipType, confidence) {
if (!relationshipType) return '';
const confidenceText = confidence >= 0.8 ? '●' : confidence >= 0.6 ? '◐' : '○';
const confidenceText = confidence >= 0.8 ? '●' : confidence >= 0.6 ? '●' : '○';
return `${relationshipType} ${confidenceText}`;
}
@ -1417,7 +1447,7 @@ class GraphManager {
menuItems += `
<li data-action="hide" data-node-id="${nodeId}">
<span class="menu-icon">👁️‍🗨️</span>
<span class="menu-icon">👻</span>
<span>Hide Node</span>
</li>
<li data-action="delete" data-node-id="${nodeId}">

163
static/js/main.js
View File

@ -9,7 +9,8 @@ class DNSReconApp {
console.log('DNSReconApp constructor called');
this.graphManager = null;
this.scanStatus = 'idle';
this.pollInterval = null;
this.statusPollInterval = null; // Separate status polling
this.graphPollInterval = null; // Separate graph polling
this.currentSessionId = null;
this.elements = {};
@ -17,6 +18,10 @@ class DNSReconApp {
this.isScanning = false;
this.lastGraphUpdate = null;
// Graph polling optimization
this.graphPollingNodeThreshold = 500; // Default, will be loaded from config
this.graphPollingEnabled = true;
this.init();
}
@ -35,6 +40,7 @@ class DNSReconApp {
this.loadProviders();
this.initializeEnhancedModals();
this.addCheckboxStyling();
this.loadConfig(); // Load configuration including threshold
this.updateGraph();
@ -46,6 +52,21 @@ class DNSReconApp {
});
}
/**
* Load configuration from backend
*/
async loadConfig() {
try {
const response = await this.apiCall('/api/config');
if (response.success) {
this.graphPollingNodeThreshold = response.config.graph_polling_node_threshold;
console.log(`Graph polling threshold set to: ${this.graphPollingNodeThreshold} nodes`);
}
} catch (error) {
console.warn('Failed to load config, using defaults:', error);
}
}
/**
* Initialize DOM element references
*/
@ -263,12 +284,19 @@ class DNSReconApp {
}
/**
* Initialize graph visualization
* Initialize graph visualization with manual refresh button
*/
initializeGraph() {
try {
console.log('Initializing graph manager...');
this.graphManager = new GraphManager('network-graph');
// Set up manual refresh handler
this.graphManager.setManualRefreshHandler(() => {
console.log('Manual graph refresh requested');
this.updateGraph();
});
console.log('Graph manager initialized successfully');
} catch (error) {
console.error('Failed to initialize graph manager:', error);
@ -276,6 +304,34 @@ class DNSReconApp {
}
}
/**
* Check if graph polling should be enabled based on node count
*/
shouldEnableGraphPolling() {
if (!this.graphManager || !this.graphManager.nodes) {
return true;
}
const nodeCount = this.graphManager.nodes.length;
return nodeCount <= this.graphPollingNodeThreshold;
}
/**
* Update manual refresh button visibility and state.
* The button will be visible whenever auto-polling is disabled,
* and enabled only when a scan is in progress.
*/
updateManualRefreshButton() {
if (!this.graphManager || !this.graphManager.manualRefreshButton) return;
const shouldShow = !this.graphPollingEnabled;
this.graphManager.showManualRefreshButton(shouldShow);
if (shouldShow) {
this.graphManager.manualRefreshButton.disabled = false;
}
}
/**
* Start scan with error handling
*/
@ -324,18 +380,21 @@ class DNSReconApp {
if (clearGraph) {
this.graphManager.clear();
this.graphPollingEnabled = true; // Reset polling when starting fresh
}
console.log(`Scan started for ${target} with depth ${maxDepth}`);
// Start polling immediately with faster interval for responsiveness
this.startPolling();
// Start optimized polling
this.startOptimizedPolling();
// Force an immediate status update
console.log('Forcing immediate status update...');
setTimeout(() => {
this.updateStatus();
if (this.graphPollingEnabled) {
this.updateGraph();
}
}, 100);
} else {
@ -348,6 +407,35 @@ class DNSReconApp {
this.setUIState('idle');
}
}
/**
* Start optimized polling with separate status and graph intervals
*/
startOptimizedPolling() {
console.log('=== STARTING OPTIMIZED POLLING ===');
this.stopPolling(); // Stop any existing polling
// Always poll status for progress bar
this.statusPollInterval = setInterval(() => {
this.updateStatus();
this.loadProviders();
}, 2000);
// Only poll graph if enabled
if (this.graphPollingEnabled) {
this.graphPollInterval = setInterval(() => {
this.updateGraph();
}, 2000);
console.log('Graph polling started');
} else {
console.log('Graph polling disabled due to node count');
}
this.updateManualRefreshButton();
console.log(`Optimized polling started - Status: enabled, Graph: ${this.graphPollingEnabled ? 'enabled' : 'disabled'}`);
}
/**
* Scan stop with immediate UI feedback
*/
@ -374,15 +462,8 @@ class DNSReconApp {
this.updateStatus();
}, 100);
// Continue polling for a bit to catch the status change
this.startPolling(500); // Fast polling to catch status change
// Stop fast polling after 10 seconds
setTimeout(() => {
if (this.scanStatus === 'stopped' || this.scanStatus === 'idle') {
this.stopPolling();
}
}, 10000);
// Continue status polling for a bit to catch the status change
// No need to resume graph polling
} else {
throw new Error(response.error || 'Failed to stop scan');
@ -573,10 +654,18 @@ class DNSReconApp {
*/
stopPolling() {
console.log('=== STOPPING POLLING ===');
if (this.pollInterval) {
clearInterval(this.pollInterval);
this.pollInterval = null;
if (this.statusPollInterval) {
clearInterval(this.statusPollInterval);
this.statusPollInterval = null;
}
if (this.graphPollInterval) {
clearInterval(this.graphPollInterval);
this.graphPollInterval = null;
}
this.updateManualRefreshButton();
}
/**
@ -611,7 +700,7 @@ class DNSReconApp {
}
/**
* Update graph from server
* Update graph from server with polling optimization
*/
async updateGraph() {
try {
@ -626,11 +715,29 @@ class DNSReconApp {
console.log('- Nodes:', graphData.nodes ? graphData.nodes.length : 0);
console.log('- Edges:', graphData.edges ? graphData.edges.length : 0);
// FIXED: Always update graph, even if empty - let GraphManager handle placeholder
// Always update graph, even if empty - let GraphManager handle placeholder
if (this.graphManager) {
this.graphManager.updateGraph(graphData);
this.lastGraphUpdate = Date.now();
// Check if we should disable graph polling after this update
const nodeCount = graphData.nodes ? graphData.nodes.length : 0;
const shouldEnablePolling = nodeCount <= this.graphPollingNodeThreshold;
if (this.graphPollingEnabled && !shouldEnablePolling) {
console.log(`Graph polling disabled: ${nodeCount} nodes exceeds threshold of ${this.graphPollingNodeThreshold}`);
this.graphPollingEnabled = false;
this.showWarning(`Graph auto-refresh disabled: ${nodeCount} nodes exceed threshold of ${this.graphPollingNodeThreshold}. Use manual refresh button.`);
// Stop graph polling but keep status polling
if (this.graphPollInterval) {
clearInterval(this.graphPollInterval);
this.graphPollInterval = null;
}
this.updateManualRefreshButton();
}
// Update relationship count in status
const edgeCount = graphData.edges ? graphData.edges.length : 0;
if (this.elements.relationshipsDisplay) {
@ -639,7 +746,7 @@ class DNSReconApp {
}
} else {
console.error('Graph update failed:', response);
// FIXED: Show placeholder when graph update fails
// Show placeholder when graph update fails
if (this.graphManager && this.graphManager.container) {
const placeholder = this.graphManager.container.querySelector('.graph-placeholder');
if (placeholder) {
@ -650,7 +757,7 @@ class DNSReconApp {
} catch (error) {
console.error('Failed to update graph:', error);
// FIXED: Show placeholder on error
// Show placeholder on error
if (this.graphManager && this.graphManager.container) {
const placeholder = this.graphManager.container.querySelector('.graph-placeholder');
if (placeholder) {
@ -660,7 +767,6 @@ class DNSReconApp {
}
}
/**
* Update status display elements
* @param {Object} status - Status object from server
@ -737,8 +843,6 @@ class DNSReconApp {
case 'running':
this.setUIState('scanning', task_queue_size);
this.showSuccess('Scan is running');
// Increase polling frequency for active scans
this.startPolling(5000); // Poll every 5 second for running scans
this.updateConnectionStatus('active');
break;
@ -748,8 +852,9 @@ class DNSReconApp {
this.showSuccess('Scan completed successfully');
this.updateConnectionStatus('completed');
this.loadProviders();
// Force a final graph update
console.log('Scan completed - forcing final graph update');
// Do final graph update when scan completes
console.log('Scan completed - performing final graph update');
setTimeout(() => this.updateGraph(), 100);
break;
@ -820,6 +925,7 @@ class DNSReconApp {
switch (state) {
case 'scanning':
case 'running':
this.isScanning = true;
if (this.graphManager) {
this.graphManager.isScanning = true;
@ -852,12 +958,12 @@ class DNSReconApp {
this.graphManager.isScanning = false;
}
if (this.elements.startScan) {
this.elements.startScan.disabled = !isQueueEmpty;
this.elements.startScan.disabled = false;
this.elements.startScan.classList.remove('loading');
this.elements.startScan.innerHTML = '<span class="btn-icon">[RUN]</span><span>Start Reconnaissance</span>';
}
if (this.elements.addToGraph) {
this.elements.addToGraph.disabled = !isQueueEmpty;
this.elements.addToGraph.disabled = false;
this.elements.addToGraph.classList.remove('loading');
}
if (this.elements.stopScan) {
@ -867,6 +973,9 @@ class DNSReconApp {
if (this.elements.targetInput) this.elements.targetInput.disabled = false;
if (this.elements.maxDepth) this.elements.maxDepth.disabled = false;
if (this.elements.configureSettings) this.elements.configureSettings.disabled = false;
// Update manual refresh button visibility
this.updateManualRefreshButton();
break;
}
}