diff --git a/.env.example b/.env.example index 6c05aa7..002b9d5 100644 --- a/.env.example +++ b/.env.example @@ -25,7 +25,7 @@ DEFAULT_RECURSION_DEPTH=2 # Default timeout for provider API requests in seconds. DEFAULT_TIMEOUT=30 # The number of concurrent provider requests to make. -MAX_CONCURRENT_REQUESTS=5 +MAX_CONCURRENT_REQUESTS=1 # The number of results from a provider that triggers the "large entity" grouping. LARGE_ENTITY_THRESHOLD=100 # The number of times to retry a target if a provider fails. diff --git a/app.py b/app.py index 450337c..fceb61a 100644 --- a/app.py +++ b/app.py @@ -32,7 +32,7 @@ def get_user_scanner(): if current_flask_session_id: existing_scanner = session_manager.get_session(current_flask_session_id) if existing_scanner: - print(f"Reusing existing session: {current_flask_session_id}") + #print(f"Reusing existing session: {current_flask_session_id}") return current_flask_session_id, existing_scanner else: print(f"Session {current_flask_session_id} expired, will create new one") @@ -93,7 +93,6 @@ def start_scan(): return jsonify({'success': False, 'error': 'Failed to get scanner instance.'}), 500 print(f"Using scanner {id(scanner)} in session {user_session_id}") - print(f"Scanner has {len(scanner.providers)} providers: {[p.get_name() for p in scanner.providers]}") # FIXED: Pass clear_graph flag to scanner, let it handle graph clearing internally success = scanner.start_scan(target, max_depth, clear_graph=clear_graph, force_rescan_target=force_rescan_target) @@ -469,7 +468,6 @@ def get_providers(): completed_tasks = scanner.indicators_completed total_tasks = scanner.total_tasks_ever_enqueued print(f"DEBUG: Task Progress - Completed: {completed_tasks}, Total Enqueued: {total_tasks}") - print(f"DEBUG: Scanner has {len(scanner.providers)} providers: {[p.get_name() for p in scanner.providers]}") else: print("DEBUG: No active scanner session found.") diff --git a/config.py b/config.py index a01eb24..2b2ff1c 100644 --- a/config.py +++ b/config.py @@ -21,8 +21,8 @@ class Config: # --- General Settings --- self.default_recursion_depth = 2 - self.default_timeout = 30 - self.max_concurrent_requests = 5 + self.default_timeout = 60 + self.max_concurrent_requests = 1 self.large_entity_threshold = 100 self.max_retries_per_target = 8 self.cache_expiry_hours = 12 diff --git a/core/graph_manager.py b/core/graph_manager.py index f0629cb..0d21d2d 100644 --- a/core/graph_manager.py +++ b/core/graph_manager.py @@ -344,13 +344,19 @@ class GraphManager: return False node_data = self.graph.nodes[large_entity_id] - attributes = node_data.get('attributes', {}) + attributes = node_data.get('attributes', []) + + # Find the 'nodes' attribute dictionary in the list + nodes_attr = next((attr for attr in attributes if attr.get('name') == 'nodes'), None) # Remove from the list of member nodes - if 'nodes' in attributes and node_id_to_extract in attributes['nodes']: - attributes['nodes'].remove(node_id_to_extract) - # Update the count - attributes['count'] = len(attributes['nodes']) + if nodes_attr and 'value' in nodes_attr and isinstance(nodes_attr['value'], list) and node_id_to_extract in nodes_attr['value']: + nodes_attr['value'].remove(node_id_to_extract) + + # Find the 'count' attribute and update it + count_attr = next((attr for attr in attributes if attr.get('name') == 'count'), None) + if count_attr: + count_attr['value'] = len(nodes_attr['value']) else: # This can happen if the node was already extracted, which is not an error. print(f"Warning: Node {node_id_to_extract} not found in the 'nodes' list of {large_entity_id}.") diff --git a/core/logger.py b/core/logger.py index f391030..371cd7e 100644 --- a/core/logger.py +++ b/core/logger.py @@ -152,7 +152,7 @@ class ForensicLogger: # Log to standard logger if error: - self.logger.error(f"API Request Failed - {provider}: {url} - {error}") + self.logger.error(f"API Request Failed.") else: self.logger.info(f"API Request - {provider}: {url} - Status: {status_code}") diff --git a/core/scanner.py b/core/scanner.py index 735ddf3..55d6b1c 100644 --- a/core/scanner.py +++ b/core/scanner.py @@ -432,7 +432,7 @@ class Scanner: self.indicators_completed += 1 if not self._is_stop_requested(): - all_new_targets = new_targets.union(large_entity_members) + all_new_targets = new_targets for new_target in all_new_targets: is_ip_new = _is_valid_ip(new_target) eligible_providers_new = self._get_eligible_providers(new_target, is_ip_new, False) @@ -576,6 +576,33 @@ class Scanner: print(f"Stop requested before processing results from {provider_name} for {target}") return discovered_targets, False + attributes_by_node = defaultdict(list) + for attribute in provider_result.attributes: + # Convert the StandardAttribute object to a dictionary that the frontend can use + attr_dict = { + "name": attribute.name, + "value": attribute.value, + "type": attribute.type, + "provider": attribute.provider, + "confidence": attribute.confidence, + "metadata": attribute.metadata + } + attributes_by_node[attribute.target_node].append(attr_dict) + + # Add attributes to nodes + for node_id, node_attributes_list in attributes_by_node.items(): + if self.graph.graph.has_node(node_id): + # Determine node type + if _is_valid_ip(node_id): + node_type = NodeType.IP + elif node_id.startswith('AS') and node_id[2:].isdigit(): + node_type = NodeType.ASN + else: + node_type = NodeType.DOMAIN + + # Add node with the list of attributes + self.graph.add_node(node_id, node_type, attributes=node_attributes_list) + # Check for large entity based on relationship count if provider_result.get_relationship_count() > self.config.large_entity_threshold: print(f"Large entity detected: {provider_name} returned {provider_result.get_relationship_count()} relationships for {target}") @@ -619,34 +646,6 @@ class Scanner: if _is_valid_domain(target_node) or _is_valid_ip(target_node): discovered_targets.add(target_node) - # Process attributes, preserving them as a list of objects - attributes_by_node = defaultdict(list) - for attribute in provider_result.attributes: - # Convert the StandardAttribute object to a dictionary that the frontend can use - attr_dict = { - "name": attribute.name, - "value": attribute.value, - "type": attribute.type, - "provider": attribute.provider, - "confidence": attribute.confidence, - "metadata": attribute.metadata - } - attributes_by_node[attribute.target_node].append(attr_dict) - - # Add attributes to nodes - for node_id, node_attributes_list in attributes_by_node.items(): - if self.graph.graph.has_node(node_id): - # Determine node type - if _is_valid_ip(node_id): - node_type = NodeType.IP - elif node_id.startswith('AS') and node_id[2:].isdigit(): - node_type = NodeType.ASN - else: - node_type = NodeType.DOMAIN - - # Add node with the list of attributes - self.graph.add_node(node_id, node_type, attributes=node_attributes_list) - return discovered_targets, False def _create_large_entity_from_provider_result(self, source: str, provider_name: str, @@ -793,7 +792,11 @@ class Scanner: # 4. Re-queue the extracted node for full processing by all eligible providers print(f"Re-queueing extracted node {node_id_to_extract} for full reconnaissance...") is_ip = _is_valid_ip(node_id_to_extract) - current_depth = self.graph.graph.nodes[large_entity_id].get('attributes', {}).get('discovery_depth', 0) + + # FIX: Correctly retrieve discovery_depth from the list of attributes + large_entity_attributes = self.graph.graph.nodes[large_entity_id].get('attributes', []) + discovery_depth_attr = next((attr for attr in large_entity_attributes if attr.get('name') == 'discovery_depth'), None) + current_depth = discovery_depth_attr['value'] if discovery_depth_attr else 0 eligible_providers = self._get_eligible_providers(node_id_to_extract, is_ip, False) for provider in eligible_providers: diff --git a/core/session_manager.py b/core/session_manager.py index d44c9e8..266c356 100644 --- a/core/session_manager.py +++ b/core/session_manager.py @@ -50,7 +50,6 @@ class SessionManager: """Restore SessionManager after unpickling.""" self.__dict__.update(state) # Re-initialize unpickleable attributes - import redis self.redis_client = redis.StrictRedis(db=0, decode_responses=False) self.lock = threading.Lock() self.creation_lock = threading.Lock() @@ -140,7 +139,6 @@ class SessionManager: new_scanner = Scanner(session_config=preserved_config) new_scanner.session_id = new_session_id - print(f"New scanner has {len(new_scanner.providers)} providers: {[p.get_name() for p in new_scanner.providers]}") new_session_data = { 'scanner': new_scanner, diff --git a/static/js/graph.js b/static/js/graph.js index 9273ddf..a6693f0 100644 --- a/static/js/graph.js +++ b/static/js/graph.js @@ -214,7 +214,6 @@ class GraphManager { }); document.body.appendChild(this.contextMenu); - console.log('Context menu created and added to body'); } /** @@ -291,7 +290,6 @@ class GraphManager { // FIXED: Right-click context menu this.container.addEventListener('contextmenu', (event) => { event.preventDefault(); - console.log('Right-click detected at:', event.offsetX, event.offsetY); // Get coordinates relative to the canvas const pointer = { @@ -300,7 +298,6 @@ class GraphManager { }; const nodeId = this.network.getNodeAt(pointer); - console.log('Node at pointer:', nodeId); if (nodeId) { // Pass the original client event for positioning @@ -341,19 +338,12 @@ class GraphManager { // Stabilization events with progress this.network.on('stabilizationProgress', (params) => { const progress = params.iterations / params.total; - this.updateStabilizationProgress(progress); }); this.network.on('stabilizationIterationsDone', () => { this.onStabilizationComplete(); }); - // Selection events - this.network.on('select', (params) => { - console.log('Selected nodes:', params.nodes); - console.log('Selected edges:', params.edges); - }); - // Click away to hide context menu document.addEventListener('click', (e) => { if (!this.contextMenu.contains(e.target)) { @@ -979,15 +969,6 @@ class GraphManager { }, 2000); } - /** - * Update stabilization progress - * @param {number} progress - Progress value (0-1) - */ - updateStabilizationProgress(progress) { - // Could show a progress indicator if needed - console.log(`Graph stabilization: ${(progress * 100).toFixed(1)}%`); - } - /** * Handle stabilization completion */ @@ -1175,11 +1156,11 @@ class GraphManager { adjacencyList ); - console.log(`Reachability analysis complete:`, { + /*console.log(`Reachability analysis complete:`, { reachable: analysis.reachableNodes.size, unreachable: analysis.unreachableNodes.size, clusters: analysis.isolatedClusters.length - }); + });*/ return analysis; } @@ -1294,7 +1275,6 @@ class GraphManager { * Replaces the existing applyAllFilters() method */ applyAllFilters() { - console.log("Applying filters with enhanced reachability analysis..."); if (this.nodes.length === 0) return; // Get filter criteria from UI @@ -1350,23 +1330,10 @@ class GraphManager { operation: 'hide_with_reachability', timestamp: Date.now() }; - - // Apply hiding with forensic documentation - const updates = nodesToHide.map(id => ({ - id: id, - hidden: true, - forensicNote: `Hidden due to reachability analysis from ${nodeId}` - })); this.nodes.update(updates); this.addToHistory('hide', historyData); - console.log(`Forensic hide operation: ${nodesToHide.length} nodes hidden`, { - originalTarget: nodeId, - cascadeNodes: nodesToHide.length - 1, - isolatedClusters: analysis.isolatedClusters.length - }); - return { hiddenNodes: nodesToHide, isolatedClusters: analysis.isolatedClusters @@ -1449,9 +1416,7 @@ class GraphManager { // Handle operation results if (!operationFailed) { - this.addToHistory('delete', historyData); - console.log(`Forensic delete operation completed:`, historyData.forensicAnalysis); - + this.addToHistory('delete', historyData); return { success: true, deletedNodes: nodesToDelete, @@ -1542,7 +1507,6 @@ class GraphManager { e.stopPropagation(); const action = e.currentTarget.dataset.action; const nodeId = e.currentTarget.dataset.nodeId; - console.log('Context menu action:', action, 'for node:', nodeId); this.performContextMenuAction(action, nodeId); this.hideContextMenu(); }); @@ -1562,9 +1526,7 @@ class GraphManager { * UPDATED: Enhanced context menu actions using new methods * Updates the existing performContextMenuAction() method */ - performContextMenuAction(action, nodeId) { - console.log('Performing enhanced action:', action, 'on node:', nodeId); - + performContextMenuAction(action, nodeId) { switch (action) { case 'focus': this.focusOnNode(nodeId); diff --git a/static/js/main.js b/static/js/main.js index ea96e9e..f1d82c0 100644 --- a/static/js/main.js +++ b/static/js/main.js @@ -120,7 +120,6 @@ class DNSReconApp { max-width: 400px; `; document.body.appendChild(messageContainer); - console.log('Message container created'); } } @@ -135,7 +134,6 @@ class DNSReconApp { this.initializeModalFunctionality(); this.elements.startScan.addEventListener('click', (e) => { - console.log('Start scan button clicked'); e.preventDefault(); this.startScan(); }); @@ -146,13 +144,11 @@ class DNSReconApp { }); this.elements.stopScan.addEventListener('click', (e) => { - console.log('Stop scan button clicked'); e.preventDefault(); this.stopScan(); }); this.elements.exportResults.addEventListener('click', (e) => { - console.log('Export results button clicked'); e.preventDefault(); this.exportResults(); }); @@ -162,7 +158,6 @@ class DNSReconApp { // Enter key support for target domain input this.elements.targetInput.addEventListener('keypress', (e) => { if (e.key === 'Enter' && !this.isScanning) { - console.log('Enter key pressed in domain input'); this.startScan(); } }); @@ -287,12 +282,8 @@ class DNSReconApp { force_rescan_target: forceRescanTarget }; - console.log('Request data:', requestData); - const response = await this.apiCall('/api/scan/start', 'POST', requestData); - - console.log('API response received:', response); - + if (response.success) { this.currentSessionId = response.scan_id; this.showSuccess('Reconnaissance scan started successfully'); @@ -345,7 +336,6 @@ class DNSReconApp { if (response.success) { this.showSuccess('Scan stop requested'); - console.log('Scan stop requested successfully'); // Force immediate status update setTimeout(() => { @@ -414,7 +404,6 @@ class DNSReconApp { } this.pollInterval = setInterval(() => { - console.log('--- Polling tick ---'); this.updateStatus(); this.updateGraph(); this.loadProviders(); @@ -439,16 +428,11 @@ class DNSReconApp { */ async updateStatus() { try { - console.log('Updating status...'); const response = await this.apiCall('/api/scan/status'); - console.log('Status response:', response); if (response.success && response.status) { const status = response.status; - console.log('Current scan status:', status.status); - console.log('Current progress:', status.progress_percentage + '%'); - console.log('Graph stats:', status.graph_statistics); this.updateStatusDisplay(status); @@ -478,7 +462,6 @@ class DNSReconApp { console.log('Updating graph...'); const response = await this.apiCall('/api/graph'); - console.log('Graph response:', response); if (response.success) { const graphData = response.graph; @@ -527,9 +510,7 @@ class DNSReconApp { * @param {Object} status - Status object from server */ updateStatusDisplay(status) { - try { - console.log('Updating status display...'); - + try { // Update status text with animation if (this.elements.scanStatus) { const formattedStatus = this.formatStatus(status.status); @@ -584,8 +565,6 @@ class DNSReconApp { } this.setUIState(status.status, status.task_queue_size); - - console.log('Status display updated successfully'); } catch (error) { console.error('Error updating status display:', error); } @@ -680,9 +659,7 @@ class DNSReconApp { /** * UI state management with immediate button updates */ - setUIState(state, task_queue_size) { - console.log(`Setting UI state to: ${state}`); - + setUIState(state, task_queue_size) { const isQueueEmpty = task_queue_size === 0; switch (state) { @@ -743,7 +720,6 @@ class DNSReconApp { */ async loadProviders() { try { - console.log('Loading providers...'); const response = await this.apiCall('/api/providers'); if (response.success) { @@ -1401,7 +1377,6 @@ class DNSReconApp { e.stopPropagation(); const nodeId = e.target.dataset.nodeId || e.target.getAttribute('data-node-id'); - console.log('Node link clicked:', nodeId); if (nodeId && this.graphManager && this.graphManager.nodes) { const nextNode = this.graphManager.nodes.get(nodeId); @@ -1758,7 +1733,6 @@ class DNSReconApp { * @returns {Promise} Response data */ async apiCall(endpoint, method = 'GET', data = null) { - console.log(`Making API call: ${method} ${endpoint}`, data ? data : '(no data)'); try { const options = { @@ -1773,17 +1747,14 @@ class DNSReconApp { console.log('Request body:', options.body); } - console.log('Fetch options:', options); const response = await fetch(endpoint, options); - console.log('Response status:', response.status, response.statusText); if (!response.ok) { throw new Error(`HTTP ${response.status}: ${response.statusText}`); } const result = await response.json(); - console.log('Response data:', result); return result; @@ -1917,9 +1888,7 @@ class DNSReconApp { * @param {string} message - Message text * @param {string} type - Message type (success, error, warning, info) */ - showMessage(message, type = 'info') { - console.log(`${type.toUpperCase()}: ${message}`); - + showMessage(message, type = 'info') { // Create message element const messageElement = document.createElement('div'); messageElement.className = `message-toast message-${type}`;