From 97aa18f788d699d2458f8e3fe5cc36b7390fdd1b Mon Sep 17 00:00:00 2001 From: overcuriousity Date: Tue, 16 Sep 2025 20:21:08 +0200 Subject: [PATCH 01/20] implement new data api --- core/graph_manager.py | 52 ++- core/provider_result.py | 106 +++++ core/rate_limiter.py | 1 - core/scanner.py | 529 +++++++++++----------- core/session_manager.py | 6 +- providers/base_provider.py | 13 +- providers/crtsh_provider.py | 845 ++++++++++++++++------------------- providers/dns_provider.py | 139 ++++-- providers/shodan_provider.py | 224 ++++++++-- static/js/graph.js | 39 +- static/js/main.js | 372 +++++---------- 11 files changed, 1206 insertions(+), 1120 deletions(-) create mode 100644 core/provider_result.py diff --git a/core/graph_manager.py b/core/graph_manager.py index 6086ea8..69dbda8 100644 --- a/core/graph_manager.py +++ b/core/graph_manager.py @@ -1,8 +1,9 @@ -# core/graph_manager.py +# dnsrecon-reduced/core/graph_manager.py """ Graph data model for DNSRecon using NetworkX. Manages in-memory graph storage with confidence scoring and forensic metadata. +Now fully compatible with the unified ProviderResult data model. """ import re from datetime import datetime, timezone @@ -28,6 +29,7 @@ class GraphManager: """ Thread-safe graph manager for DNSRecon infrastructure mapping. Uses NetworkX for in-memory graph storage with confidence scoring. + Compatible with unified ProviderResult data model. """ def __init__(self): @@ -192,21 +194,36 @@ class GraphManager: }) return all_correlations - def add_node(self, node_id: str, node_type: NodeType, attributes: Optional[Dict[str, Any]] = None, + def add_node(self, node_id: str, node_type: NodeType, attributes: Optional[List[Dict[str, Any]]] = None, description: str = "", metadata: Optional[Dict[str, Any]] = None) -> bool: - """Add a node to the graph, update attributes, and process correlations.""" + """ + Add a node to the graph, update attributes, and process correlations. + Now compatible with unified data model - attributes are dictionaries from converted StandardAttribute objects. + """ is_new_node = not self.graph.has_node(node_id) if is_new_node: self.graph.add_node(node_id, type=node_type.value, added_timestamp=datetime.now(timezone.utc).isoformat(), - attributes=attributes or {}, + attributes=attributes or [], # Store as a list from the start description=description, metadata=metadata or {}) else: - # Safely merge new attributes into existing attributes + # Safely merge new attributes into the existing list of attributes if attributes: - existing_attributes = self.graph.nodes[node_id].get('attributes', {}) - existing_attributes.update(attributes) + existing_attributes = self.graph.nodes[node_id].get('attributes', []) + + # Handle cases where old data might still be in dictionary format + if not isinstance(existing_attributes, list): + existing_attributes = [] + + # Create a set of existing attribute names for efficient duplicate checking + existing_attr_names = {attr['name'] for attr in existing_attributes} + + for new_attr in attributes: + if new_attr['name'] not in existing_attr_names: + existing_attributes.append(new_attr) + existing_attr_names.add(new_attr['name']) + self.graph.nodes[node_id]['attributes'] = existing_attributes if description: self.graph.nodes[node_id]['description'] = description @@ -485,19 +502,28 @@ class GraphManager: if d.get('confidence_score', 0) >= min_confidence] def get_graph_data(self) -> Dict[str, Any]: - """Export graph data formatted for frontend visualization.""" + """ + Export graph data formatted for frontend visualization. + Compatible with unified data model - preserves all attribute information for frontend display. + """ nodes = [] for node_id, attrs in self.graph.nodes(data=True): node_data = {'id': node_id, 'label': node_id, 'type': attrs.get('type', 'unknown'), - 'attributes': attrs.get('attributes', {}), + 'attributes': attrs.get('attributes', []), # Ensure attributes is a list 'description': attrs.get('description', ''), 'metadata': attrs.get('metadata', {}), 'added_timestamp': attrs.get('added_timestamp')} + # Customize node appearance based on type and attributes node_type = node_data['type'] - attributes = node_data['attributes'] - if node_type == 'domain' and attributes.get('certificates', {}).get('has_valid_cert') is False: - node_data['color'] = {'background': '#c7c7c7', 'border': '#999'} # Gray for invalid cert + attributes_list = node_data['attributes'] + + # CORRECTED LOGIC: Handle certificate validity styling + if node_type == 'domain' and isinstance(attributes_list, list): + # Find the certificates attribute in the list + cert_attr = next((attr for attr in attributes_list if attr.get('name') == 'certificates'), None) + if cert_attr and cert_attr.get('value', {}).get('has_valid_cert') is False: + node_data['color'] = {'background': '#c7c7c7', 'border': '#999'} # Gray for invalid cert # Add incoming and outgoing edges to node data if self.graph.has_node(node_id): @@ -528,7 +554,7 @@ class GraphManager: 'last_modified': self.last_modified, 'total_nodes': self.get_node_count(), 'total_edges': self.get_edge_count(), - 'graph_format': 'dnsrecon_v1_nodeling' + 'graph_format': 'dnsrecon_v1_unified_model' }, 'graph': graph_data, 'statistics': self.get_statistics() diff --git a/core/provider_result.py b/core/provider_result.py new file mode 100644 index 0000000..df2f6f1 --- /dev/null +++ b/core/provider_result.py @@ -0,0 +1,106 @@ +# dnsrecon-reduced/core/provider_result.py + +""" +Unified data model for DNSRecon passive reconnaissance. +Standardizes the data structure across all providers to ensure consistent processing. +""" + +from typing import Any, Optional, List, Dict +from dataclasses import dataclass, field +from datetime import datetime, timezone + + +@dataclass +class StandardAttribute: + """A unified data structure for a single piece of information about a node.""" + target_node: str + name: str + value: Any + type: str + provider: str + confidence: float + timestamp: datetime = field(default_factory=lambda: datetime.now(timezone.utc)) + metadata: Optional[Dict[str, Any]] = field(default_factory=dict) + + def __post_init__(self): + """Validate the attribute after initialization.""" + if not isinstance(self.confidence, (int, float)) or not 0.0 <= self.confidence <= 1.0: + raise ValueError(f"Confidence must be between 0.0 and 1.0, got {self.confidence}") + + +@dataclass +class Relationship: + """A unified data structure for a directional link between two nodes.""" + source_node: str + target_node: str + relationship_type: str + confidence: float + provider: str + timestamp: datetime = field(default_factory=lambda: datetime.now(timezone.utc)) + raw_data: Optional[Dict[str, Any]] = field(default_factory=dict) + + def __post_init__(self): + """Validate the relationship after initialization.""" + if not isinstance(self.confidence, (int, float)) or not 0.0 <= self.confidence <= 1.0: + raise ValueError(f"Confidence must be between 0.0 and 1.0, got {self.confidence}") + + +@dataclass +class ProviderResult: + """A container for all data returned by a provider from a single query.""" + attributes: List[StandardAttribute] = field(default_factory=list) + relationships: List[Relationship] = field(default_factory=list) + + def add_attribute(self, target_node: str, name: str, value: Any, attr_type: str, + provider: str, confidence: float = 0.8, + metadata: Optional[Dict[str, Any]] = None) -> None: + """Helper method to add an attribute to the result.""" + self.attributes.append(StandardAttribute( + target_node=target_node, + name=name, + value=value, + type=attr_type, + provider=provider, + confidence=confidence, + metadata=metadata or {} + )) + + def add_relationship(self, source_node: str, target_node: str, relationship_type: str, + provider: str, confidence: float = 0.8, + raw_data: Optional[Dict[str, Any]] = None) -> None: + """Helper method to add a relationship to the result.""" + self.relationships.append(Relationship( + source_node=source_node, + target_node=target_node, + relationship_type=relationship_type, + confidence=confidence, + provider=provider, + raw_data=raw_data or {} + )) + + def get_discovered_nodes(self) -> set: + """Get all unique node identifiers discovered in this result.""" + nodes = set() + + # Add nodes from relationships + for rel in self.relationships: + nodes.add(rel.source_node) + nodes.add(rel.target_node) + + # Add nodes from attributes + for attr in self.attributes: + nodes.add(attr.target_node) + + return nodes + + def get_relationship_count(self) -> int: + """Get the total number of relationships in this result.""" + return len(self.relationships) + + def get_attribute_count(self) -> int: + """Get the total number of attributes in this result.""" + return len(self.attributes) + + def is_large_entity(self, threshold: int) -> bool: + """Check if this result qualifies as a large entity based on relationship count.""" + return self.get_relationship_count() > threshold \ No newline at end of file diff --git a/core/rate_limiter.py b/core/rate_limiter.py index 7fadff4..d5a11d6 100644 --- a/core/rate_limiter.py +++ b/core/rate_limiter.py @@ -1,7 +1,6 @@ # dnsrecon-reduced/core/rate_limiter.py import time -import redis class GlobalRateLimiter: def __init__(self, redis_client): diff --git a/core/scanner.py b/core/scanner.py index 6ce05f3..11b5493 100644 --- a/core/scanner.py +++ b/core/scanner.py @@ -2,18 +2,18 @@ import threading import traceback -import time import os import importlib import redis from typing import List, Set, Dict, Any, Tuple, Optional -from concurrent.futures import ThreadPoolExecutor, as_completed, CancelledError, Future +from concurrent.futures import ThreadPoolExecutor from collections import defaultdict from queue import PriorityQueue from datetime import datetime, timezone from core.graph_manager import GraphManager, NodeType from core.logger import get_forensic_logger, new_session +from core.provider_result import ProviderResult from utils.helpers import _is_valid_ip, _is_valid_domain from providers.base_provider import BaseProvider from core.rate_limiter import GlobalRateLimiter @@ -30,6 +30,7 @@ class ScanStatus: class Scanner: """ Main scanning orchestrator for DNSRecon passive reconnaissance. + Now provider-agnostic, consuming standardized ProviderResult objects. """ def __init__(self, session_config=None): @@ -470,6 +471,10 @@ class Scanner: print(f" - Tasks processed: {len(processed_tasks)}") def _query_single_provider_for_target(self, provider: BaseProvider, target: str, depth: int) -> Tuple[Set[str], Set[str], bool]: + """ + Query a single provider and process the unified ProviderResult. + Now provider-agnostic - handles any provider that returns ProviderResult. + """ if self._is_stop_requested(): print(f"Stop requested before querying {provider.get_name()} for {target}") return set(), set(), False @@ -478,21 +483,24 @@ class Scanner: target_type = NodeType.IP if is_ip else NodeType.DOMAIN print(f"Querying {provider.get_name()} for {target_type.value}: {target} at depth {depth}") + # Ensure target node exists in graph self.graph.add_node(target, target_type) self._initialize_provider_states(target) new_targets = set() large_entity_members = set() - node_attributes = defaultdict(lambda: defaultdict(list)) provider_successful = True try: - provider_results = self._query_single_provider_forensic(provider, target, is_ip, depth) - if provider_results is None: + # Query provider - now returns unified ProviderResult + provider_result = self._query_single_provider_unified(provider, target, is_ip, depth) + + if provider_result is None: provider_successful = False elif not self._is_stop_requested(): - discovered, is_large_entity = self._process_provider_results( - target, provider, provider_results, node_attributes, depth + # Process the unified result + discovered, is_large_entity = self._process_provider_result_unified( + target, provider, provider_result, depth ) if is_large_entity: large_entity_members.update(discovered) @@ -504,15 +512,177 @@ class Scanner: provider_successful = False self._log_provider_error(target, provider.get_name(), str(e)) - if not self._is_stop_requested(): - for node_id, attributes in node_attributes.items(): - if self.graph.graph.has_node(node_id): - node_is_ip = _is_valid_ip(node_id) - node_type_to_add = NodeType.IP if node_is_ip else NodeType.DOMAIN - self.graph.add_node(node_id, node_type_to_add, attributes=attributes) - return new_targets, large_entity_members, provider_successful + def _query_single_provider_unified(self, provider: BaseProvider, target: str, is_ip: bool, current_depth: int) -> Optional[ProviderResult]: + """ + Query a single provider with stop signal checking, now returns ProviderResult. + """ + provider_name = provider.get_name() + start_time = datetime.now(timezone.utc) + + if self._is_stop_requested(): + print(f"Stop requested before querying {provider_name} for {target}") + return None + + print(f"Querying {provider_name} for {target}") + + self.logger.logger.info(f"Attempting {provider_name} query for {target} at depth {current_depth}") + + try: + # Query the provider - returns unified ProviderResult + if is_ip: + result = provider.query_ip(target) + else: + result = provider.query_domain(target) + + if self._is_stop_requested(): + print(f"Stop requested after querying {provider_name} for {target}") + return None + + # Update provider state with relationship count (more meaningful than raw result count) + relationship_count = result.get_relationship_count() if result else 0 + self._update_provider_state(target, provider_name, 'success', relationship_count, None, start_time) + + print(f"✓ {provider_name} returned {relationship_count} relationships for {target}") + return result + + except Exception as e: + self._update_provider_state(target, provider_name, 'failed', 0, str(e), start_time) + print(f"✗ {provider_name} failed for {target}: {e}") + return None + + def _process_provider_result_unified(self, target: str, provider: BaseProvider, + provider_result: ProviderResult, current_depth: int) -> Tuple[Set[str], bool]: + """ + Process a unified ProviderResult object to update the graph. + Returns (discovered_targets, is_large_entity). + """ + provider_name = provider.get_name() + discovered_targets = set() + + if self._is_stop_requested(): + print(f"Stop requested before processing results from {provider_name} for {target}") + return discovered_targets, False + + # Check for large entity based on relationship count + if provider_result.get_relationship_count() > self.config.large_entity_threshold: + print(f"Large entity detected: {provider_name} returned {provider_result.get_relationship_count()} relationships for {target}") + members = self._create_large_entity_from_provider_result(target, provider_name, provider_result, current_depth) + return members, True + + # Process relationships + for i, relationship in enumerate(provider_result.relationships): + if i % 5 == 0 and self._is_stop_requested(): # Check periodically for stop + print(f"Stop requested while processing relationships from {provider_name} for {target}") + break + + # Add nodes for relationship endpoints + source_node = relationship.source_node + target_node = relationship.target_node + + # Determine node types + source_type = NodeType.IP if _is_valid_ip(source_node) else NodeType.DOMAIN + if target_node.startswith('AS') and target_node[2:].isdigit(): + target_type = NodeType.ASN + elif _is_valid_ip(target_node): + target_type = NodeType.IP + else: + target_type = NodeType.DOMAIN + + # Add nodes to graph + self.graph.add_node(source_node, source_type) + self.graph.add_node(target_node, target_type) + + # Add edge to graph + if self.graph.add_edge( + source_node, target_node, + relationship.relationship_type, + relationship.confidence, + provider_name, + relationship.raw_data + ): + print(f"Added relationship: {source_node} -> {target_node} ({relationship.relationship_type})") + + # Track discovered targets for further processing + if _is_valid_domain(target_node) or _is_valid_ip(target_node): + discovered_targets.add(target_node) + + # Process attributes, preserving them as a list of objects + attributes_by_node = defaultdict(list) + for attribute in provider_result.attributes: + # Convert the StandardAttribute object to a dictionary that the frontend can use + attr_dict = { + "name": attribute.name, + "value": attribute.value, + "type": attribute.type, + "provider": attribute.provider, + "confidence": attribute.confidence, + "metadata": attribute.metadata + } + attributes_by_node[attribute.target_node].append(attr_dict) + + # Add attributes to nodes + for node_id, node_attributes_list in attributes_by_node.items(): + if self.graph.graph.has_node(node_id): + # Determine node type + if _is_valid_ip(node_id): + node_type = NodeType.IP + elif node_id.startswith('AS') and node_id[2:].isdigit(): + node_type = NodeType.ASN + else: + node_type = NodeType.DOMAIN + + # Add node with the list of attributes + self.graph.add_node(node_id, node_type, attributes=node_attributes_list) + + return discovered_targets, False + + def _create_large_entity_from_provider_result(self, source: str, provider_name: str, + provider_result: ProviderResult, current_depth: int) -> Set[str]: + """ + Create a large entity node from a ProviderResult and return the members for DNS processing. + """ + entity_id = f"large_entity_{provider_name}_{hash(source) & 0x7FFFFFFF}" + + # Extract target nodes from relationships + targets = [rel.target_node for rel in provider_result.relationships] + node_type = 'unknown' + + if targets: + if _is_valid_domain(targets[0]): + node_type = 'domain' + elif _is_valid_ip(targets[0]): + node_type = 'ip' + + # Create nodes in graph (they exist but are grouped) + for target in targets: + target_node_type = NodeType.DOMAIN if node_type == 'domain' else NodeType.IP + self.graph.add_node(target, target_node_type) + + attributes = { + 'count': len(targets), + 'nodes': targets, + 'node_type': node_type, + 'source_provider': provider_name, + 'discovery_depth': current_depth, + 'threshold_exceeded': self.config.large_entity_threshold, + } + description = f'Large entity created due to {len(targets)} relationships from {provider_name}' + + self.graph.add_node(entity_id, NodeType.LARGE_ENTITY, attributes=attributes, description=description) + + # Create edge from source to large entity + if provider_result.relationships: + rel_type = provider_result.relationships[0].relationship_type + self.graph.add_edge(source, entity_id, rel_type, 0.9, provider_name, + {'large_entity_info': f'Contains {len(targets)} {node_type}s'}) + + self.logger.logger.warning(f"Large entity created: {entity_id} contains {len(targets)} targets from {provider_name}") + print(f"Created large entity {entity_id} for {len(targets)} {node_type}s from {provider_name}") + + return set(targets) + def stop_scan(self) -> bool: """Request immediate scan termination with proper cleanup.""" try: @@ -558,6 +728,73 @@ class Scanner: traceback.print_exc() return False + def extract_node_from_large_entity(self, large_entity_id: str, node_id_to_extract: str) -> bool: + """ + Extracts a node from a large entity, re-creates its original edge, and + re-queues it for full scanning. + """ + if not self.graph.graph.has_node(large_entity_id): + print(f"ERROR: Large entity {large_entity_id} not found.") + return False + + # 1. Get the original source node that discovered the large entity + predecessors = list(self.graph.graph.predecessors(large_entity_id)) + if not predecessors: + print(f"ERROR: No source node found for large entity {large_entity_id}.") + return False + source_node_id = predecessors[0] + + # Get the original edge data to replicate it for the extracted node + original_edge_data = self.graph.graph.get_edge_data(source_node_id, large_entity_id) + if not original_edge_data: + print(f"ERROR: Could not find original edge data from {source_node_id} to {large_entity_id}.") + return False + + # 2. Modify the graph data structure first + success = self.graph.extract_node_from_large_entity(large_entity_id, node_id_to_extract) + if not success: + print(f"ERROR: Node {node_id_to_extract} could not be removed from {large_entity_id}'s attributes.") + return False + + # 3. Create the direct edge from the original source to the newly extracted node + print(f"Re-creating direct edge from {source_node_id} to extracted node {node_id_to_extract}") + self.graph.add_edge( + source_id=source_node_id, + target_id=node_id_to_extract, + relationship_type=original_edge_data.get('relationship_type', 'extracted_from_large_entity'), + confidence_score=original_edge_data.get('confidence_score', 0.85), # Slightly lower confidence + source_provider=original_edge_data.get('source_provider', 'unknown'), + raw_data={'context': f'Extracted from large entity {large_entity_id}'} + ) + + # 4. Re-queue the extracted node for full processing by all eligible providers + print(f"Re-queueing extracted node {node_id_to_extract} for full reconnaissance...") + is_ip = _is_valid_ip(node_id_to_extract) + current_depth = self.graph.graph.nodes[large_entity_id].get('attributes', {}).get('discovery_depth', 0) + + eligible_providers = self._get_eligible_providers(node_id_to_extract, is_ip, False) + for provider in eligible_providers: + provider_name = provider.get_name() + self.task_queue.put((self._get_priority(provider_name), (provider_name, node_id_to_extract, current_depth))) + self.total_tasks_ever_enqueued += 1 + + # 5. If the scanner is not running, we need to kickstart it to process this one item. + if self.status != ScanStatus.RUNNING: + print("Scanner is idle. Starting a mini-scan to process the extracted node.") + self.status = ScanStatus.RUNNING + self._update_session_state() + + if not self.scan_thread or not self.scan_thread.is_alive(): + self.scan_thread = threading.Thread( + target=self._execute_scan, + args=(self.current_target, self.max_depth), + daemon=True + ) + self.scan_thread.start() + + print(f"Successfully extracted and re-queued {node_id_to_extract} from {large_entity_id}.") + return True + def _update_session_state(self) -> None: """ Update the scanner state in Redis for GUI updates. @@ -656,39 +893,6 @@ class Scanner: provider_state = provider_states.get(provider_name) return provider_state is not None and provider_state.get('status') == 'success' - def _query_single_provider_forensic(self, provider, target: str, is_ip: bool, current_depth: int) -> Optional[List]: - """Query a single provider with stop signal checking.""" - provider_name = provider.get_name() - start_time = datetime.now(timezone.utc) - - if self._is_stop_requested(): - print(f"Stop requested before querying {provider_name} for {target}") - return None - - print(f"Querying {provider_name} for {target}") - - self.logger.logger.info(f"Attempting {provider_name} query for {target} at depth {current_depth}") - - try: - if is_ip: - results = provider.query_ip(target) - else: - results = provider.query_domain(target) - - if self._is_stop_requested(): - print(f"Stop requested after querying {provider_name} for {target}") - return None - - self._update_provider_state(target, provider_name, 'success', len(results), None, start_time) - - print(f"✓ {provider_name} returned {len(results)} results for {target}") - return results - - except Exception as e: - self._update_provider_state(target, provider_name, 'failed', 0, str(e), start_time) - print(f"✗ {provider_name} failed for {target}: {e}") - return None - def _update_provider_state(self, target: str, provider_name: str, status: str, results_count: int, error: Optional[str], start_time: datetime) -> None: """Update provider state in node metadata for forensic tracking.""" @@ -711,237 +915,6 @@ class Scanner: self.logger.logger.info(f"Provider state updated: {target} -> {provider_name} -> {status} ({results_count} results)") - def _process_provider_results(self, target: str, provider, results: List, - node_attributes: Dict, current_depth: int) -> Tuple[Set[str], bool]: - """Process provider results, returns (discovered_targets, is_large_entity).""" - provider_name = provider.get_name() - discovered_targets = set() - - if self._is_stop_requested(): - print(f"Stop requested before processing results from {provider_name} for {target}") - return discovered_targets, False - - if len(results) > self.config.large_entity_threshold: - print(f"Large entity detected: {provider_name} returned {len(results)} results for {target}") - members = self._create_large_entity(target, provider_name, results, current_depth) - return members, True - - for i, (source, rel_target, rel_type, confidence, raw_data) in enumerate(results): - if i % 5 == 0 and self._is_stop_requested(): # Check more frequently - print(f"Stop requested while processing results from {provider_name} for {target}") - break - - self.logger.log_relationship_discovery( - source_node=source, - target_node=rel_target, - relationship_type=rel_type, - confidence_score=confidence, - provider=provider_name, - raw_data=raw_data, - discovery_method=f"{provider_name}_query_depth_{current_depth}" - ) - - # Collect attributes for the source node - self._collect_node_attributes(source, provider_name, rel_type, rel_target, raw_data, node_attributes[source]) - - # If the relationship is asn_membership, collect attributes for the target ASN node - if rel_type == 'asn_membership': - self._collect_node_attributes(rel_target, provider_name, rel_type, source, raw_data, node_attributes[rel_target]) - - - if isinstance(rel_target, list): - # If the target is a list, iterate and process each item - for single_target in rel_target: - if _is_valid_ip(single_target): - self.graph.add_node(single_target, NodeType.IP) - if self.graph.add_edge(source, single_target, rel_type, confidence, provider_name, raw_data): - print(f"Added IP relationship: {source} -> {single_target} ({rel_type})") - discovered_targets.add(single_target) - elif _is_valid_domain(single_target): - self.graph.add_node(single_target, NodeType.DOMAIN) - if self.graph.add_edge(source, single_target, rel_type, confidence, provider_name, raw_data): - print(f"Added domain relationship: {source} -> {single_target} ({rel_type})") - discovered_targets.add(single_target) - self._collect_node_attributes(single_target, provider_name, rel_type, source, raw_data, node_attributes[single_target]) - - elif _is_valid_ip(rel_target): - self.graph.add_node(rel_target, NodeType.IP) - if self.graph.add_edge(source, rel_target, rel_type, confidence, provider_name, raw_data): - print(f"Added IP relationship: {source} -> {rel_target} ({rel_type})") - discovered_targets.add(rel_target) - - elif rel_target.startswith('AS') and rel_target[2:].isdigit(): - self.graph.add_node(rel_target, NodeType.ASN) - if self.graph.add_edge(source, rel_target, rel_type, confidence, provider_name, raw_data): - print(f"Added ASN relationship: {source} -> {rel_target} ({rel_type})") - - elif _is_valid_domain(rel_target): - self.graph.add_node(rel_target, NodeType.DOMAIN) - if self.graph.add_edge(source, rel_target, rel_type, confidence, provider_name, raw_data): - print(f"Added domain relationship: {source} -> {rel_target} ({rel_type})") - discovered_targets.add(rel_target) - self._collect_node_attributes(rel_target, provider_name, rel_type, source, raw_data, node_attributes[rel_target]) - - else: - self._collect_node_attributes(source, provider_name, rel_type, rel_target, raw_data, node_attributes[source]) - - return discovered_targets, False - - def _create_large_entity(self, source: str, provider_name: str, results: List, current_depth: int) -> Set[str]: - """Create a large entity node and returns the members for DNS processing.""" - entity_id = f"large_entity_{provider_name}_{hash(source) & 0x7FFFFFFF}" - - targets = [rel[1] for rel in results if len(rel) > 1] - node_type = 'unknown' - - if targets: - if _is_valid_domain(targets[0]): - node_type = 'domain' - elif _is_valid_ip(targets[0]): - node_type = 'ip' - - # We still create the nodes so they exist in the graph, they are just not processed for edges yet. - for target in targets: - self.graph.add_node(target, NodeType.DOMAIN if node_type == 'domain' else NodeType.IP) - - attributes = { - 'count': len(targets), - 'nodes': targets, - 'node_type': node_type, - 'source_provider': provider_name, - 'discovery_depth': current_depth, - 'threshold_exceeded': self.config.large_entity_threshold, - } - description = f'Large entity created due to {len(targets)} results from {provider_name}' - - self.graph.add_node(entity_id, NodeType.LARGE_ENTITY, attributes=attributes, description=description) - - if results: - rel_type = results[0][2] - self.graph.add_edge(source, entity_id, rel_type, 0.9, provider_name, - {'large_entity_info': f'Contains {len(targets)} {node_type}s'}) - - self.logger.logger.warning(f"Large entity created: {entity_id} contains {len(targets)} targets from {provider_name}") - print(f"Created large entity {entity_id} for {len(targets)} {node_type}s from {provider_name}") - - return set(targets) - - def extract_node_from_large_entity(self, large_entity_id: str, node_id_to_extract: str) -> bool: - """ - Extracts a node from a large entity, re-creates its original edge, and - re-queues it for full scanning. - """ - if not self.graph.graph.has_node(large_entity_id): - print(f"ERROR: Large entity {large_entity_id} not found.") - return False - - # 1. Get the original source node that discovered the large entity - predecessors = list(self.graph.graph.predecessors(large_entity_id)) - if not predecessors: - print(f"ERROR: No source node found for large entity {large_entity_id}.") - return False - source_node_id = predecessors[0] - - # Get the original edge data to replicate it for the extracted node - original_edge_data = self.graph.graph.get_edge_data(source_node_id, large_entity_id) - if not original_edge_data: - print(f"ERROR: Could not find original edge data from {source_node_id} to {large_entity_id}.") - return False - - # 2. Modify the graph data structure first - success = self.graph.extract_node_from_large_entity(large_entity_id, node_id_to_extract) - if not success: - print(f"ERROR: Node {node_id_to_extract} could not be removed from {large_entity_id}'s attributes.") - return False - - # 3. Create the direct edge from the original source to the newly extracted node - print(f"Re-creating direct edge from {source_node_id} to extracted node {node_id_to_extract}") - self.graph.add_edge( - source_id=source_node_id, - target_id=node_id_to_extract, - relationship_type=original_edge_data.get('relationship_type', 'extracted_from_large_entity'), - confidence_score=original_edge_data.get('confidence_score', 0.85), # Slightly lower confidence - source_provider=original_edge_data.get('source_provider', 'unknown'), - raw_data={'context': f'Extracted from large entity {large_entity_id}'} - ) - - # 4. Re-queue the extracted node for full processing by all eligible providers - print(f"Re-queueing extracted node {node_id_to_extract} for full reconnaissance...") - is_ip = _is_valid_ip(node_id_to_extract) - current_depth = self.graph.graph.nodes[large_entity_id].get('attributes', {}).get('discovery_depth', 0) - - eligible_providers = self._get_eligible_providers(node_id_to_extract, is_ip, False) - for provider in eligible_providers: - provider_name = provider.get_name() - self.task_queue.put((self._get_priority(provider_name), (provider_name, node_id_to_extract, current_depth))) - self.total_tasks_ever_enqueued += 1 - - # 5. If the scanner is not running, we need to kickstart it to process this one item. - if self.status != ScanStatus.RUNNING: - print("Scanner is idle. Starting a mini-scan to process the extracted node.") - self.status = ScanStatus.RUNNING - self._update_session_state() - - if not self.scan_thread or not self.scan_thread.is_alive(): - self.scan_thread = threading.Thread( - target=self._execute_scan, - args=(self.current_target, self.max_depth), - daemon=True - ) - self.scan_thread.start() - - print(f"Successfully extracted and re-queued {node_id_to_extract} from {large_entity_id}.") - return True - - def _collect_node_attributes(self, node_id: str, provider_name: str, rel_type: str, - target: str, raw_data: Dict[str, Any], attributes: Dict[str, Any]) -> None: - """Collect and organize attributes for a node.""" - self.logger.logger.debug(f"Collecting attributes for {node_id} from {provider_name}: {rel_type}") - - if provider_name == 'dns': - record_type = raw_data.get('query_type', 'UNKNOWN') - value = raw_data.get('value', target) - dns_entry = f"{record_type}: {value}" - if dns_entry not in attributes.get('dns_records', []): - attributes.setdefault('dns_records', []).append(dns_entry) - - elif provider_name == 'crtsh': - if rel_type == "san_certificate": - domain_certs = raw_data.get('domain_certificates', {}) - if node_id in domain_certs: - cert_summary = domain_certs[node_id] - attributes['certificates'] = cert_summary - if target not in attributes.get('related_domains_san', []): - attributes.setdefault('related_domains_san', []).append(target) - - elif provider_name == 'shodan': - # This logic will now apply to the correct node (ASN or IP) - shodan_attributes = attributes.setdefault('shodan', {}) - for key, value in raw_data.items(): - if key not in shodan_attributes or not shodan_attributes.get(key): - shodan_attributes[key] = value - - if _is_valid_ip(node_id): - if 'ports' in raw_data: - attributes['ports'] = raw_data['ports'] - if 'os' in raw_data and raw_data['os']: - attributes['os'] = raw_data['os'] - - if rel_type == "asn_membership": - # This is the key change: these attributes are for the target (the ASN), - # not the source (the IP). We will add them to the ASN node later. - pass - - record_type_name = rel_type - if record_type_name not in attributes: - attributes[record_type_name] = [] - - if isinstance(target, list): - attributes[record_type_name].extend(target) - else: - if target not in attributes[record_type_name]: - attributes[record_type_name].append(target) - def _log_target_processing_error(self, target: str, error: str) -> None: """Log target processing errors for forensic trail.""" self.logger.logger.error(f"Target processing failed for {target}: {error}") diff --git a/core/session_manager.py b/core/session_manager.py index 7631db9..4662c65 100644 --- a/core/session_manager.py +++ b/core/session_manager.py @@ -5,15 +5,11 @@ import time import uuid import redis import pickle -from typing import Dict, Optional, Any, List +from typing import Dict, Optional, Any from core.scanner import Scanner from config import config -# WARNING: Using pickle can be a security risk if the data source is not trusted. -# In this case, we are only serializing/deserializing our own trusted Scanner objects, -# which is generally safe. Do not unpickle data from untrusted sources. - class SessionManager: """ Manages multiple scanner instances for concurrent user sessions using Redis. diff --git a/providers/base_provider.py b/providers/base_provider.py index 7941fb6..9337658 100644 --- a/providers/base_provider.py +++ b/providers/base_provider.py @@ -4,16 +4,17 @@ import time import requests import threading from abc import ABC, abstractmethod -from typing import List, Dict, Any, Optional, Tuple +from typing import Dict, Any, Optional from core.logger import get_forensic_logger from core.rate_limiter import GlobalRateLimiter +from core.provider_result import ProviderResult class BaseProvider(ABC): """ Abstract base class for all DNSRecon data providers. - Now supports session-specific configuration. + Now supports session-specific configuration and returns standardized ProviderResult objects. """ def __init__(self, name: str, rate_limit: int = 60, timeout: int = 30, session_config=None): @@ -101,7 +102,7 @@ class BaseProvider(ABC): pass @abstractmethod - def query_domain(self, domain: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]: + def query_domain(self, domain: str) -> ProviderResult: """ Query the provider for information about a domain. @@ -109,12 +110,12 @@ class BaseProvider(ABC): domain: Domain to investigate Returns: - List of tuples: (source_node, target_node, relationship_type, confidence, raw_data) + ProviderResult containing standardized attributes and relationships """ pass @abstractmethod - def query_ip(self, ip: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]: + def query_ip(self, ip: str) -> ProviderResult: """ Query the provider for information about an IP address. @@ -122,7 +123,7 @@ class BaseProvider(ABC): ip: IP address to investigate Returns: - List of tuples: (source_node, target_node, relationship_type, confidence, raw_data) + ProviderResult containing standardized attributes and relationships """ pass diff --git a/providers/crtsh_provider.py b/providers/crtsh_provider.py index bfa2c51..4ccce97 100644 --- a/providers/crtsh_provider.py +++ b/providers/crtsh_provider.py @@ -2,21 +2,21 @@ import json import re -import os from pathlib import Path -from typing import List, Dict, Any, Tuple, Set +from typing import List, Dict, Any, Set from urllib.parse import quote from datetime import datetime, timezone import requests from .base_provider import BaseProvider +from core.provider_result import ProviderResult from utils.helpers import _is_valid_domain class CrtShProvider(BaseProvider): """ Provider for querying crt.sh certificate transparency database. - Now uses session-specific configuration and caching with accumulative behavior. + Now returns standardized ProviderResult objects with caching support. """ def __init__(self, name=None, session_config=None): @@ -33,6 +33,9 @@ class CrtShProvider(BaseProvider): # Initialize cache directory self.cache_dir = Path('cache') / 'crtsh' self.cache_dir.mkdir(parents=True, exist_ok=True) + + # Compile regex for date filtering for efficiency + self.date_pattern = re.compile(r'^\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}') def get_name(self) -> str: """Return the provider name.""" @@ -51,16 +54,11 @@ class CrtShProvider(BaseProvider): return {'domains': True, 'ips': False} def is_available(self) -> bool: - """ - Check if the provider is configured to be used. - This method is intentionally simple and does not perform a network request - to avoid blocking application startup. - """ + """Check if the provider is configured to be used.""" return True def _get_cache_file_path(self, domain: str) -> Path: """Generate cache file path for a domain.""" - # Sanitize domain for filename safety safe_domain = domain.replace('.', '_').replace('/', '_').replace('\\', '_') return self.cache_dir / f"{safe_domain}.json" @@ -78,7 +76,7 @@ class CrtShProvider(BaseProvider): last_query_str = cache_data.get("last_upstream_query") if not last_query_str: - return "stale" # Invalid cache format + return "stale" last_query = datetime.fromisoformat(last_query_str.replace('Z', '+00:00')) hours_since_query = (datetime.now(timezone.utc) - last_query).total_seconds() / 3600 @@ -92,27 +90,175 @@ class CrtShProvider(BaseProvider): except (json.JSONDecodeError, ValueError, KeyError) as e: self.logger.logger.warning(f"Invalid cache file format for {cache_file_path}: {e}") return "stale" - - def _load_cached_certificates(self, cache_file_path: Path) -> List[Dict[str, Any]]: - """Load certificates from cache file.""" + + def query_domain(self, domain: str) -> ProviderResult: + """ + Query crt.sh for certificates containing the domain with caching support. + + Args: + domain: Domain to investigate + + Returns: + ProviderResult containing discovered relationships and attributes + """ + if not _is_valid_domain(domain): + return ProviderResult() + + if self._stop_event and self._stop_event.is_set(): + return ProviderResult() + + cache_file = self._get_cache_file_path(domain) + cache_status = self._get_cache_status(cache_file) + + processed_certificates = [] + result = ProviderResult() + + try: + if cache_status == "fresh": + result = self._load_from_cache(cache_file) + self.logger.logger.info(f"Using cached crt.sh data for {domain}") + + else: # "stale" or "not_found" + raw_certificates = self._query_crtsh_api(domain) + + if self._stop_event and self._stop_event.is_set(): + return ProviderResult() + + # Process raw data into the application's expected format + current_processed_certs = [self._extract_certificate_metadata(cert) for cert in raw_certificates] + + if cache_status == "stale": + # Load existing and append new processed certs + existing_result = self._load_from_cache(cache_file) + result = self._merge_results(existing_result, current_processed_certs, domain) + self.logger.logger.info(f"Refreshed and merged cache for {domain}") + else: # "not_found" + # Create new result from processed certs + result = self._process_certificates_to_result(domain, current_processed_certs) + self.logger.logger.info(f"Created fresh result for {domain} ({result.get_relationship_count()} relationships)") + + # Save the result to cache + self._save_result_to_cache(cache_file, result, domain) + + except requests.exceptions.RequestException as e: + self.logger.logger.error(f"API query failed for {domain}: {e}") + if cache_status != "not_found": + result = self._load_from_cache(cache_file) + self.logger.logger.warning(f"Using stale cache for {domain} due to API failure.") + else: + raise e # Re-raise if there's no cache to fall back on + + return result + + def query_ip(self, ip: str) -> ProviderResult: + """ + Query crt.sh for certificates containing the IP address. + Note: crt.sh doesn't typically index by IP, so this returns empty results. + + Args: + ip: IP address to investigate + + Returns: + Empty ProviderResult (crt.sh doesn't support IP-based certificate queries effectively) + """ + return ProviderResult() + + def _load_from_cache(self, cache_file_path: Path) -> ProviderResult: + """Load processed crt.sh data from a cache file.""" try: with open(cache_file_path, 'r') as f: - cache_data = json.load(f) - return cache_data.get('certificates', []) + cache_content = json.load(f) + + result = ProviderResult() + + # Reconstruct relationships + for rel_data in cache_content.get("relationships", []): + result.add_relationship( + source_node=rel_data["source_node"], + target_node=rel_data["target_node"], + relationship_type=rel_data["relationship_type"], + provider=rel_data["provider"], + confidence=rel_data["confidence"], + raw_data=rel_data.get("raw_data", {}) + ) + + # Reconstruct attributes + for attr_data in cache_content.get("attributes", []): + result.add_attribute( + target_node=attr_data["target_node"], + name=attr_data["name"], + value=attr_data["value"], + attr_type=attr_data["type"], + provider=attr_data["provider"], + confidence=attr_data["confidence"], + metadata=attr_data.get("metadata", {}) + ) + + return result + except (json.JSONDecodeError, FileNotFoundError, KeyError) as e: self.logger.logger.error(f"Failed to load cached certificates from {cache_file_path}: {e}") - return [] - + return ProviderResult() + + def _save_result_to_cache(self, cache_file_path: Path, result: ProviderResult, domain: str) -> None: + """Save processed crt.sh result to a cache file.""" + try: + cache_data = { + "domain": domain, + "last_upstream_query": datetime.now(timezone.utc).isoformat(), + "relationships": [ + { + "source_node": rel.source_node, + "target_node": rel.target_node, + "relationship_type": rel.relationship_type, + "confidence": rel.confidence, + "provider": rel.provider, + "raw_data": rel.raw_data + } for rel in result.relationships + ], + "attributes": [ + { + "target_node": attr.target_node, + "name": attr.name, + "value": attr.value, + "type": attr.type, + "provider": attr.provider, + "confidence": attr.confidence, + "metadata": attr.metadata + } for attr in result.attributes + ] + } + cache_file_path.parent.mkdir(parents=True, exist_ok=True) + with open(cache_file_path, 'w') as f: + json.dump(cache_data, f, separators=(',', ':'), default=str) + except Exception as e: + self.logger.logger.warning(f"Failed to save cache file for {domain}: {e}") + + def _merge_results(self, existing_result: ProviderResult, new_certificates: List[Dict[str, Any]], domain: str) -> ProviderResult: + """Merge new certificate data with existing cached result.""" + # Create a fresh result from the new certificates + new_result = self._process_certificates_to_result(domain, new_certificates) + + # Simple merge strategy: combine all relationships and attributes + # In practice, you might want more sophisticated deduplication + merged_result = ProviderResult() + + # Add existing relationships and attributes + merged_result.relationships.extend(existing_result.relationships) + merged_result.attributes.extend(existing_result.attributes) + + # Add new relationships and attributes + merged_result.relationships.extend(new_result.relationships) + merged_result.attributes.extend(new_result.attributes) + + return merged_result + def _query_crtsh_api(self, domain: str) -> List[Dict[str, Any]]: - """ - Query crt.sh API for raw certificate data. - Raises exceptions for network errors to allow core logic to retry. - """ + """Query crt.sh API for raw certificate data.""" url = f"{self.base_url}?q={quote(domain)}&output=json" response = self.make_request(url, target_indicator=domain) if not response or response.status_code != 200: - # This could be a temporary error - raise exception so core can retry raise requests.exceptions.RequestException(f"crt.sh API returned status {response.status_code if response else 'None'}") certificates = response.json() @@ -120,126 +266,126 @@ class CrtShProvider(BaseProvider): return [] return certificates - - def _parse_issuer_organization(self, issuer_dn: str) -> str: + + def _process_certificates_to_result(self, domain: str, certificates: List[Dict[str, Any]]) -> ProviderResult: """ - Parse the issuer Distinguished Name to extract just the organization name. + Process certificates to create ProviderResult with relationships and attributes. + """ + result = ProviderResult() - Args: - issuer_dn: Full issuer DN string (e.g., "C=US, O=Let's Encrypt, CN=R11") - - Returns: - Organization name (e.g., "Let's Encrypt") or original string if parsing fails - """ - if not issuer_dn: - return issuer_dn + if self._stop_event and self._stop_event.is_set(): + print(f"CrtSh processing cancelled before processing for domain: {domain}") + return result + + # Aggregate certificate data by domain + domain_certificates = {} + all_discovered_domains = set() - try: - # Split by comma and look for O= component - components = [comp.strip() for comp in issuer_dn.split(',')] + # Process certificates with cancellation checking + for i, cert_data in enumerate(certificates): + if i % 5 == 0 and self._stop_event and self._stop_event.is_set(): + print(f"CrtSh processing cancelled at certificate {i} for domain: {domain}") + break + + cert_metadata = self._extract_certificate_metadata(cert_data) + cert_domains = self._extract_domains_from_certificate(cert_data) - for component in components: - if component.startswith('O='): - # Extract the value after O= - org_name = component[2:].strip() - # Remove quotes if present - if org_name.startswith('"') and org_name.endswith('"'): - org_name = org_name[1:-1] - return org_name + all_discovered_domains.update(cert_domains) + for cert_domain in cert_domains: + if not _is_valid_domain(cert_domain): + continue + + if cert_domain not in domain_certificates: + domain_certificates[cert_domain] = [] + + domain_certificates[cert_domain].append(cert_metadata) + + if self._stop_event and self._stop_event.is_set(): + print(f"CrtSh query cancelled before relationship creation for domain: {domain}") + return result + + # Create relationships from query domain to ALL discovered domains + for i, discovered_domain in enumerate(all_discovered_domains): + if discovered_domain == domain: + continue # Skip self-relationships - # If no O= component found, return the original string - return issuer_dn + if i % 10 == 0 and self._stop_event and self._stop_event.is_set(): + print(f"CrtSh relationship creation cancelled for domain: {domain}") + break + + if not _is_valid_domain(discovered_domain): + continue - except Exception as e: - self.logger.logger.debug(f"Failed to parse issuer DN '{issuer_dn}': {e}") - return issuer_dn - - def _parse_certificate_date(self, date_string: str) -> datetime: - """ - Parse certificate date from crt.sh format. + # Get certificates for both domains + query_domain_certs = domain_certificates.get(domain, []) + discovered_domain_certs = domain_certificates.get(discovered_domain, []) + + # Find shared certificates + shared_certificates = self._find_shared_certificates(query_domain_certs, discovered_domain_certs) + + # Calculate confidence + confidence = self._calculate_domain_relationship_confidence( + domain, discovered_domain, shared_certificates, all_discovered_domains + ) + + # Create comprehensive raw data for the relationship + relationship_raw_data = { + 'relationship_type': 'certificate_discovery', + 'shared_certificates': shared_certificates, + 'total_shared_certs': len(shared_certificates), + 'discovery_context': self._determine_relationship_context(discovered_domain, domain), + 'domain_certificates': { + domain: self._summarize_certificates(query_domain_certs), + discovered_domain: self._summarize_certificates(discovered_domain_certs) + } + } + + # Add relationship + result.add_relationship( + source_node=domain, + target_node=discovered_domain, + relationship_type='san_certificate', + provider=self.name, + confidence=confidence, + raw_data=relationship_raw_data + ) + + # Log the relationship discovery + self.log_relationship_discovery( + source_node=domain, + target_node=discovered_domain, + relationship_type='san_certificate', + confidence_score=confidence, + raw_data=relationship_raw_data, + discovery_method="certificate_transparency_analysis" + ) - Args: - date_string: Date string from crt.sh API + # Add certificate summary as attributes for all domains that have certificates + for cert_domain, cert_list in domain_certificates.items(): + if cert_list: + cert_summary = self._summarize_certificates(cert_list) + + result.add_attribute( + target_node=cert_domain, + name='certificates', + value=cert_summary, + attr_type='certificate_data', + provider=self.name, + confidence=0.9, + metadata={'total_certificates': len(cert_list)} + ) - Returns: - Parsed datetime object in UTC - """ - if not date_string: - raise ValueError("Empty date string") - - try: - # Handle various possible formats from crt.sh - if date_string.endswith('Z'): - return datetime.fromisoformat(date_string[:-1]).replace(tzinfo=timezone.utc) - elif '+' in date_string or date_string.endswith('UTC'): - # Handle timezone-aware strings - date_string = date_string.replace('UTC', '').strip() - if '+' in date_string: - date_string = date_string.split('+')[0] - return datetime.fromisoformat(date_string).replace(tzinfo=timezone.utc) - else: - # Assume UTC if no timezone specified - return datetime.fromisoformat(date_string).replace(tzinfo=timezone.utc) - except Exception as e: - # Fallback: try parsing without timezone info and assume UTC - try: - return datetime.strptime(date_string[:19], "%Y-%m-%dT%H:%M:%S").replace(tzinfo=timezone.utc) - except Exception: - raise ValueError(f"Unable to parse date: {date_string}") from e - - def _is_cert_valid(self, cert_data: Dict[str, Any]) -> bool: - """ - Check if a certificate is currently valid based on its expiry date. - - Args: - cert_data: Certificate data from crt.sh - - Returns: - True if certificate is currently valid (not expired) - """ - try: - not_after_str = cert_data.get('not_after') - if not not_after_str: - return False - - not_after_date = self._parse_certificate_date(not_after_str) - not_before_str = cert_data.get('not_before') - - now = datetime.now(timezone.utc) - - # Check if certificate is within valid date range - is_not_expired = not_after_date > now - - if not_before_str: - not_before_date = self._parse_certificate_date(not_before_str) - is_not_before_valid = not_before_date <= now - return is_not_expired and is_not_before_valid - - return is_not_expired - - except Exception as e: - self.logger.logger.debug(f"Certificate validity check failed: {e}") - return False + return result def _extract_certificate_metadata(self, cert_data: Dict[str, Any]) -> Dict[str, Any]: - """ - Extract comprehensive metadata from certificate data. - - Args: - cert_data: Raw certificate data from crt.sh - - Returns: - Comprehensive certificate metadata dictionary - """ - # Parse the issuer name to get just the organization + """Extract comprehensive metadata from certificate data.""" raw_issuer_name = cert_data.get('issuer_name', '') parsed_issuer_name = self._parse_issuer_organization(raw_issuer_name) metadata = { 'certificate_id': cert_data.get('id'), 'serial_number': cert_data.get('serial_number'), - 'issuer_name': parsed_issuer_name, # Use parsed organization name - #'issuer_name_full': raw_issuer_name, # deliberately left out, because its not useful in most cases + 'issuer_name': parsed_issuer_name, 'issuer_ca_id': cert_data.get('issuer_ca_id'), 'common_name': cert_data.get('common_name'), 'not_before': cert_data.get('not_before'), @@ -257,7 +403,6 @@ class CrtShProvider(BaseProvider): metadata['is_currently_valid'] = self._is_cert_valid(cert_data) metadata['expires_soon'] = (not_after - datetime.now(timezone.utc)).days <= 30 - # Add human-readable dates metadata['not_before'] = not_before.strftime('%Y-%m-%d %H:%M:%S UTC') metadata['not_after'] = not_after.strftime('%Y-%m-%d %H:%M:%S UTC') @@ -268,220 +413,134 @@ class CrtShProvider(BaseProvider): return metadata - def query_domain(self, domain: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]: - """ - Query crt.sh for certificates containing the domain with caching support. - Properly raises exceptions for network errors to allow core logic retries. - """ - if not _is_valid_domain(domain): - return [] + def _parse_issuer_organization(self, issuer_dn: str) -> str: + """Parse the issuer Distinguished Name to extract just the organization name.""" + if not issuer_dn: + return issuer_dn - if self._stop_event and self._stop_event.is_set(): - return [] + try: + components = [comp.strip() for comp in issuer_dn.split(',')] + + for component in components: + if component.startswith('O='): + org_name = component[2:].strip() + if org_name.startswith('"') and org_name.endswith('"'): + org_name = org_name[1:-1] + return org_name + + return issuer_dn + + except Exception as e: + self.logger.logger.debug(f"Failed to parse issuer DN '{issuer_dn}': {e}") + return issuer_dn - cache_file = self._get_cache_file_path(domain) - cache_status = self._get_cache_status(cache_file) - - processed_certificates = [] + def _parse_certificate_date(self, date_string: str) -> datetime: + """Parse certificate date from crt.sh format.""" + if not date_string: + raise ValueError("Empty date string") try: - if cache_status == "fresh": - processed_certificates = self._load_cached_certificates(cache_file) - self.logger.logger.info(f"Using cached processed data for {domain} ({len(processed_certificates)} certificates)") - - else: # "stale" or "not_found" - raw_certificates = self._query_crtsh_api(domain) - - if self._stop_event and self._stop_event.is_set(): - return [] - - # Process raw data into the application's expected format - current_processed_certs = [self._extract_certificate_metadata(cert) for cert in raw_certificates] - - if cache_status == "stale": - # Append new processed certs to existing ones - processed_certificates = self._append_to_cache(cache_file, current_processed_certs) - self.logger.logger.info(f"Refreshed and appended cache for {domain}") - else: # "not_found" - # Create a new cache file with the processed certs, even if empty - self._create_cache_file(cache_file, domain, current_processed_certs) - processed_certificates = current_processed_certs - self.logger.logger.info(f"Cached fresh data for {domain} ({len(processed_certificates)} certificates)") - - - except requests.exceptions.RequestException as e: - self.logger.logger.error(f"API query failed for {domain}: {e}") - if cache_status != "not_found": - processed_certificates = self._load_cached_certificates(cache_file) - self.logger.logger.warning(f"Using stale cache for {domain} due to API failure.") + if date_string.endswith('Z'): + return datetime.fromisoformat(date_string[:-1]).replace(tzinfo=timezone.utc) + elif '+' in date_string or date_string.endswith('UTC'): + date_string = date_string.replace('UTC', '').strip() + if '+' in date_string: + date_string = date_string.split('+')[0] + return datetime.fromisoformat(date_string).replace(tzinfo=timezone.utc) else: - raise e # Re-raise if there's no cache to fall back on - - if not processed_certificates: - return [] - - return self._process_certificates_to_relationships(domain, processed_certificates) - - def _create_cache_file(self, cache_file_path: Path, domain: str, processed_certificates: List[Dict[str, Any]]) -> None: - """Create new cache file with processed certificates.""" - try: - cache_data = { - "domain": domain, - "last_upstream_query": datetime.now(timezone.utc).isoformat(), - "certificates": processed_certificates # Store processed data - } - cache_file_path.parent.mkdir(parents=True, exist_ok=True) - with open(cache_file_path, 'w') as f: - json.dump(cache_data, f, separators=(',', ':')) + return datetime.fromisoformat(date_string).replace(tzinfo=timezone.utc) except Exception as e: - self.logger.logger.warning(f"Failed to create cache file for {domain}: {e}") + try: + return datetime.strptime(date_string[:19], "%Y-%m-%dT%H:%M:%S").replace(tzinfo=timezone.utc) + except Exception: + raise ValueError(f"Unable to parse date: {date_string}") from e - def _append_to_cache(self, cache_file_path: Path, new_processed_certificates: List[Dict[str, Any]]) -> List[Dict[str, Any]]: - """Append new processed certificates to existing cache and return all certificates.""" + def _is_cert_valid(self, cert_data: Dict[str, Any]) -> bool: + """Check if a certificate is currently valid based on its expiry date.""" try: - with open(cache_file_path, 'r') as f: - cache_data = json.load(f) - - existing_ids = {cert.get('certificate_id') for cert in cache_data.get('certificates', [])} - - for cert in new_processed_certificates: - if cert.get('certificate_id') not in existing_ids: - cache_data['certificates'].append(cert) - - cache_data['last_upstream_query'] = datetime.now(timezone.utc).isoformat() - - with open(cache_file_path, 'w') as f: - json.dump(cache_data, f, separators=(',', ':')) - - return cache_data['certificates'] + not_after_str = cert_data.get('not_after') + if not not_after_str: + return False + + not_after_date = self._parse_certificate_date(not_after_str) + not_before_str = cert_data.get('not_before') + + now = datetime.now(timezone.utc) + is_not_expired = not_after_date > now + + if not_before_str: + not_before_date = self._parse_certificate_date(not_before_str) + is_not_before_valid = not_before_date <= now + return is_not_expired and is_not_before_valid + + return is_not_expired + except Exception as e: - self.logger.logger.warning(f"Failed to append to cache: {e}") - return new_processed_certificates - - def _process_certificates_to_relationships(self, domain: str, certificates: List[Dict[str, Any]]) -> List[Tuple[str, str, str, float, Dict[str, Any]]]: - """ - Process certificates to relationships using existing logic. - This method contains the original processing logic from query_domain. - """ - relationships = [] + self.logger.logger.debug(f"Certificate validity check failed: {e}") + return False + + def _extract_domains_from_certificate(self, cert_data: Dict[str, Any]) -> Set[str]: + """Extract all domains from certificate data.""" + domains = set() - # Check for cancellation before processing - if self._stop_event and self._stop_event.is_set(): - print(f"CrtSh processing cancelled before processing for domain: {domain}") + # Extract from common name + common_name = cert_data.get('common_name', '') + if common_name: + cleaned_cn = self._clean_domain_name(common_name) + if cleaned_cn: + domains.update(cleaned_cn) + + # Extract from name_value field (contains SANs) + name_value = cert_data.get('name_value', '') + if name_value: + for line in name_value.split('\n'): + cleaned_domains = self._clean_domain_name(line.strip()) + if cleaned_domains: + domains.update(cleaned_domains) + + return domains + + def _clean_domain_name(self, domain_name: str) -> List[str]: + """Clean and normalize domain name from certificate data.""" + if not domain_name: return [] - # Aggregate certificate data by domain - domain_certificates = {} - all_discovered_domains = set() - - # Process certificates with cancellation checking - for i, cert_data in enumerate(certificates): - # Check for cancellation every 5 certificates for faster response - if i % 5 == 0 and self._stop_event and self._stop_event.is_set(): - print(f"CrtSh processing cancelled at certificate {i} for domain: {domain}") - break - - cert_metadata = self._extract_certificate_metadata(cert_data) - cert_domains = self._extract_domains_from_certificate(cert_data) - - # Add all domains from this certificate to our tracking - all_discovered_domains.update(cert_domains) - for cert_domain in cert_domains: - if not _is_valid_domain(cert_domain): - continue - - # Initialize domain certificate list if needed - if cert_domain not in domain_certificates: - domain_certificates[cert_domain] = [] - - # Add this certificate to the domain's certificate list - domain_certificates[cert_domain].append(cert_metadata) - - # Final cancellation check before creating relationships - if self._stop_event and self._stop_event.is_set(): - print(f"CrtSh query cancelled before relationship creation for domain: {domain}") - return [] + domain = domain_name.strip().lower() - # Create relationships from query domain to ALL discovered domains with stop checking - for i, discovered_domain in enumerate(all_discovered_domains): - if discovered_domain == domain: - continue # Skip self-relationships - - # Check for cancellation every 10 relationships - if i % 10 == 0 and self._stop_event and self._stop_event.is_set(): - print(f"CrtSh relationship creation cancelled for domain: {domain}") - break + if domain.startswith(('http://', 'https://')): + domain = domain.split('://', 1)[1] - if not _is_valid_domain(discovered_domain): - continue - - # Get certificates for both domains - query_domain_certs = domain_certificates.get(domain, []) - discovered_domain_certs = domain_certificates.get(discovered_domain, []) - - # Find shared certificates (for metadata purposes) - shared_certificates = self._find_shared_certificates(query_domain_certs, discovered_domain_certs) - - # Calculate confidence based on relationship type and shared certificates - confidence = self._calculate_domain_relationship_confidence( - domain, discovered_domain, shared_certificates, all_discovered_domains - ) - - # Create comprehensive raw data for the relationship - relationship_raw_data = { - 'relationship_type': 'certificate_discovery', - 'shared_certificates': shared_certificates, - 'total_shared_certs': len(shared_certificates), - 'discovery_context': self._determine_relationship_context(discovered_domain, domain), - 'domain_certificates': { - domain: self._summarize_certificates(query_domain_certs), - discovered_domain: self._summarize_certificates(discovered_domain_certs) - } - } - - # Create domain -> domain relationship - relationships.append(( - domain, - discovered_domain, - 'san_certificate', - confidence, - relationship_raw_data - )) - - # Log the relationship discovery - self.log_relationship_discovery( - source_node=domain, - target_node=discovered_domain, - relationship_type='san_certificate', - confidence_score=confidence, - raw_data=relationship_raw_data, - discovery_method="certificate_transparency_analysis" - ) + if '/' in domain: + domain = domain.split('/', 1)[0] - return relationships + if ':' in domain and not domain.count(':') > 1: + domain = domain.split(':', 1)[0] + + cleaned_domains = [] + if domain.startswith('*.'): + cleaned_domains.append(domain) + cleaned_domains.append(domain[2:]) + else: + cleaned_domains.append(domain) + + final_domains = [] + for d in cleaned_domains: + d = re.sub(r'[^\w\-\.]', '', d) + if d and not d.startswith(('.', '-')) and not d.endswith(('.', '-')): + final_domains.append(d) + + return [d for d in final_domains if _is_valid_domain(d)] def _find_shared_certificates(self, certs1: List[Dict[str, Any]], certs2: List[Dict[str, Any]]) -> List[Dict[str, Any]]: - """ - Find certificates that are shared between two domain certificate lists. - - Args: - certs1: First domain's certificates - certs2: Second domain's certificates - - Returns: - List of shared certificate metadata - """ + """Find certificates that are shared between two domain certificate lists.""" shared = [] - # Create a set of certificate IDs from the first list for quick lookup cert1_ids = set() for cert in certs1: cert_id = cert.get('certificate_id') - # Ensure the ID is not None and is a hashable type before adding to the set if cert_id and isinstance(cert_id, (int, str, float, bool, tuple)): cert1_ids.add(cert_id) - # Find certificates in the second list that match for cert in certs2: cert_id = cert.get('certificate_id') if cert_id and isinstance(cert_id, (int, str, float, bool, tuple)): @@ -491,15 +550,7 @@ class CrtShProvider(BaseProvider): return shared def _summarize_certificates(self, certificates: List[Dict[str, Any]]) -> Dict[str, Any]: - """ - Create a summary of certificates for a domain. - - Args: - certificates: List of certificate metadata - - Returns: - Summary dictionary with aggregate statistics - """ + """Create a summary of certificates for a domain.""" if not certificates: return { 'total_certificates': 0, @@ -509,14 +560,13 @@ class CrtShProvider(BaseProvider): 'unique_issuers': [], 'latest_certificate': None, 'has_valid_cert': False, - 'certificate_details': [] # Always include empty list + 'certificate_details': [] } valid_count = sum(1 for cert in certificates if cert.get('is_currently_valid')) expired_count = len(certificates) - valid_count expires_soon_count = sum(1 for cert in certificates if cert.get('expires_soon')) - # Get unique issuers (using parsed organization names) unique_issuers = list(set(cert.get('issuer_name') for cert in certificates if cert.get('issuer_name'))) # Find the most recent certificate @@ -548,63 +598,40 @@ class CrtShProvider(BaseProvider): 'unique_issuers': unique_issuers, 'latest_certificate': latest_cert, 'has_valid_cert': valid_count > 0, - 'certificate_details': sorted_certificates # Include full certificate details + 'certificate_details': sorted_certificates } def _get_certificate_sort_date(self, cert: Dict[str, Any]) -> datetime: - """ - Get a sortable date from certificate data for chronological ordering. - - Args: - cert: Certificate metadata dictionary - - Returns: - Datetime object for sorting (falls back to epoch if parsing fails) - """ + """Get a sortable date from certificate data for chronological ordering.""" try: - # Try not_before first (issue date) if cert.get('not_before'): return self._parse_certificate_date(cert['not_before']) - # Fall back to entry_timestamp if available if cert.get('entry_timestamp'): return self._parse_certificate_date(cert['entry_timestamp']) - # Last resort - return a very old date for certificates without dates return datetime(1970, 1, 1, tzinfo=timezone.utc) except Exception: - # If all parsing fails, return epoch return datetime(1970, 1, 1, tzinfo=timezone.utc) def _calculate_domain_relationship_confidence(self, domain1: str, domain2: str, shared_certificates: List[Dict[str, Any]], all_discovered_domains: Set[str]) -> float: - """ - Calculate confidence score for domain relationship based on various factors. - - Args: - domain1: Source domain (query domain) - domain2: Target domain (discovered domain) - shared_certificates: List of shared certificate metadata - all_discovered_domains: All domains discovered in this query - - Returns: - Confidence score between 0.0 and 1.0 - """ + """Calculate confidence score for domain relationship based on various factors.""" base_confidence = 0.9 # Adjust confidence based on domain relationship context relationship_context = self._determine_relationship_context(domain2, domain1) if relationship_context == 'exact_match': - context_bonus = 0.0 # This shouldn't happen, but just in case + context_bonus = 0.0 elif relationship_context == 'subdomain': - context_bonus = 0.1 # High confidence for subdomains + context_bonus = 0.1 elif relationship_context == 'parent_domain': - context_bonus = 0.05 # Medium confidence for parent domains + context_bonus = 0.05 else: - context_bonus = 0.0 # Related domains get base confidence + context_bonus = 0.0 # Adjust confidence based on shared certificates if shared_certificates: @@ -616,18 +643,16 @@ class CrtShProvider(BaseProvider): else: shared_bonus = 0.02 - # Additional bonus for valid shared certificates valid_shared = sum(1 for cert in shared_certificates if cert.get('is_currently_valid')) if valid_shared > 0: validity_bonus = 0.05 else: validity_bonus = 0.0 else: - # Even without shared certificates, domains found in the same query have some relationship shared_bonus = 0.0 validity_bonus = 0.0 - # Adjust confidence based on certificate issuer reputation (if shared certificates exist) + # Adjust confidence based on certificate issuer reputation issuer_bonus = 0.0 if shared_certificates: for cert in shared_certificates: @@ -636,21 +661,11 @@ class CrtShProvider(BaseProvider): issuer_bonus = max(issuer_bonus, 0.03) break - # Calculate final confidence final_confidence = base_confidence + context_bonus + shared_bonus + validity_bonus + issuer_bonus - return max(0.1, min(1.0, final_confidence)) # Clamp between 0.1 and 1.0 + return max(0.1, min(1.0, final_confidence)) def _determine_relationship_context(self, cert_domain: str, query_domain: str) -> str: - """ - Determine the context of the relationship between certificate domain and query domain. - - Args: - cert_domain: Domain found in certificate - query_domain: Original query domain - - Returns: - String describing the relationship context - """ + """Determine the context of the relationship between certificate domain and query domain.""" if cert_domain == query_domain: return 'exact_match' elif cert_domain.endswith(f'.{query_domain}'): @@ -658,88 +673,4 @@ class CrtShProvider(BaseProvider): elif query_domain.endswith(f'.{cert_domain}'): return 'parent_domain' else: - return 'related_domain' - - def query_ip(self, ip: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]: - """ - Query crt.sh for certificates containing the IP address. - Note: crt.sh doesn't typically index by IP, so this returns empty results. - - Args: - ip: IP address to investigate - - Returns: - Empty list (crt.sh doesn't support IP-based certificate queries effectively) - """ - # crt.sh doesn't effectively support IP-based certificate queries - return [] - - def _extract_domains_from_certificate(self, cert_data: Dict[str, Any]) -> Set[str]: - """ - Extract all domains from certificate data. - - Args: - cert_data: Certificate data from crt.sh API - - Returns: - Set of unique domain names found in the certificate - """ - domains = set() - - # Extract from common name - common_name = cert_data.get('common_name', '') - if common_name: - cleaned_cn = self._clean_domain_name(common_name) - if cleaned_cn: - domains.update(cleaned_cn) - - # Extract from name_value field (contains SANs) - name_value = cert_data.get('name_value', '') - if name_value: - # Split by newlines and clean each domain - for line in name_value.split('\n'): - cleaned_domains = self._clean_domain_name(line.strip()) - if cleaned_domains: - domains.update(cleaned_domains) - - return domains - - def _clean_domain_name(self, domain_name: str) -> List[str]: - """ - Clean and normalize domain name from certificate data. - Now returns a list to handle wildcards correctly. - """ - if not domain_name: - return [] - - domain = domain_name.strip().lower() - - # Remove protocol if present - if domain.startswith(('http://', 'https://')): - domain = domain.split('://', 1)[1] - - # Remove path if present - if '/' in domain: - domain = domain.split('/', 1)[0] - - # Remove port if present - if ':' in domain and not domain.count(':') > 1: # Avoid breaking IPv6 - domain = domain.split(':', 1)[0] - - # Handle wildcard domains - cleaned_domains = [] - if domain.startswith('*.'): - # Add both the wildcard and the base domain - cleaned_domains.append(domain) - cleaned_domains.append(domain[2:]) - else: - cleaned_domains.append(domain) - - # Remove any remaining invalid characters and validate - final_domains = [] - for d in cleaned_domains: - d = re.sub(r'[^\w\-\.]', '', d) - if d and not d.startswith(('.', '-')) and not d.endswith(('.', '-')): - final_domains.append(d) - - return [d for d in final_domains if _is_valid_domain(d)] \ No newline at end of file + return 'related_domain' \ No newline at end of file diff --git a/providers/dns_provider.py b/providers/dns_provider.py index d73ef6c..5d972d1 100644 --- a/providers/dns_provider.py +++ b/providers/dns_provider.py @@ -1,15 +1,16 @@ # dnsrecon/providers/dns_provider.py from dns import resolver, reversename -from typing import List, Dict, Any, Tuple +from typing import Dict from .base_provider import BaseProvider +from core.provider_result import ProviderResult from utils.helpers import _is_valid_ip, _is_valid_domain class DNSProvider(BaseProvider): """ Provider for standard DNS resolution and reverse DNS lookups. - Now uses session-specific configuration. + Now returns standardized ProviderResult objects. """ def __init__(self, name=None, session_config=None): @@ -25,7 +26,6 @@ class DNSProvider(BaseProvider): self.resolver = resolver.Resolver() self.resolver.timeout = 5 self.resolver.lifetime = 10 - #self.resolver.nameservers = ['127.0.0.1'] def get_name(self) -> str: """Return the provider name.""" @@ -47,31 +47,35 @@ class DNSProvider(BaseProvider): """DNS is always available - no API key required.""" return True - def query_domain(self, domain: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]: + def query_domain(self, domain: str) -> ProviderResult: """ - Query DNS records for the domain to discover relationships. - ... + Query DNS records for the domain to discover relationships and attributes. + + Args: + domain: Domain to investigate + + Returns: + ProviderResult containing discovered relationships and attributes """ if not _is_valid_domain(domain): - return [] + return ProviderResult() - relationships = [] + result = ProviderResult() # Query all record types for record_type in ['A', 'AAAA', 'CNAME', 'MX', 'NS', 'SOA', 'TXT', 'SRV', 'CAA']: try: - relationships.extend(self._query_record(domain, record_type)) + self._query_record(domain, record_type, result) except resolver.NoAnswer: # This is not an error, just a confirmation that the record doesn't exist. self.logger.logger.debug(f"No {record_type} record found for {domain}") except Exception as e: self.failed_requests += 1 self.logger.logger.debug(f"{record_type} record query failed for {domain}: {e}") - # Optionally, you might want to re-raise other, more serious exceptions. - return relationships + return result - def query_ip(self, ip: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]: + def query_ip(self, ip: str) -> ProviderResult: """ Query reverse DNS for the IP address. @@ -79,12 +83,12 @@ class DNSProvider(BaseProvider): ip: IP address to investigate Returns: - List of relationships discovered from reverse DNS + ProviderResult containing discovered relationships and attributes """ if not _is_valid_ip(ip): - return [] + return ProviderResult() - relationships = [] + result = ProviderResult() try: # Perform reverse DNS lookup @@ -97,27 +101,44 @@ class DNSProvider(BaseProvider): hostname = str(ptr_record).rstrip('.') if _is_valid_domain(hostname): - raw_data = { - 'query_type': 'PTR', - 'ip_address': ip, - 'hostname': hostname, - 'ttl': response.ttl - } + # Add the relationship + result.add_relationship( + source_node=ip, + target_node=hostname, + relationship_type='ptr_record', + provider=self.name, + confidence=0.8, + raw_data={ + 'query_type': 'PTR', + 'ip_address': ip, + 'hostname': hostname, + 'ttl': response.ttl + } + ) - relationships.append(( - ip, - hostname, - 'ptr_record', - 0.8, - raw_data - )) + # Add PTR record as attribute to the IP + result.add_attribute( + target_node=ip, + name='ptr_record', + value=hostname, + attr_type='dns_record', + provider=self.name, + confidence=0.8, + metadata={'ttl': response.ttl} + ) + # Log the relationship discovery self.log_relationship_discovery( source_node=ip, target_node=hostname, relationship_type='ptr_record', confidence_score=0.8, - raw_data=raw_data, + raw_data={ + 'query_type': 'PTR', + 'ip_address': ip, + 'hostname': hostname, + 'ttl': response.ttl + }, discovery_method="reverse_dns_lookup" ) @@ -130,18 +151,24 @@ class DNSProvider(BaseProvider): # Re-raise the exception so the scanner can handle the failure raise e - return relationships + return result - def _query_record(self, domain: str, record_type: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]: + def _query_record(self, domain: str, record_type: str, result: ProviderResult) -> None: """ - Query a specific type of DNS record for the domain. + Query a specific type of DNS record for the domain and add results to ProviderResult. + + Args: + domain: Domain to query + record_type: DNS record type (A, AAAA, CNAME, etc.) + result: ProviderResult to populate """ - relationships = [] try: self.total_requests += 1 response = self.resolver.resolve(domain, record_type) self.successful_requests += 1 + dns_records = [] + for record in response: target = "" if record_type in ['A', 'AAAA']: @@ -153,12 +180,16 @@ class DNSProvider(BaseProvider): elif record_type == 'SOA': target = str(record.mname).rstrip('.') elif record_type in ['TXT']: - # TXT records are treated as metadata, not relationships. + # TXT records are treated as attributes, not relationships + txt_value = str(record).strip('"') + dns_records.append(f"TXT: {txt_value}") continue elif record_type == 'SRV': target = str(record.target).rstrip('.') elif record_type == 'CAA': - target = f"{record.flags} {record.tag.decode('utf-8')} \"{record.value.decode('utf-8')}\"" + caa_value = f"{record.flags} {record.tag.decode('utf-8')} \"{record.value.decode('utf-8')}\"" + dns_records.append(f"CAA: {caa_value}") + continue else: target = str(record) @@ -170,16 +201,22 @@ class DNSProvider(BaseProvider): 'ttl': response.ttl } relationship_type = f"{record_type.lower()}_record" - confidence = 0.8 # Default confidence for DNS records + confidence = 0.8 # Standard confidence for DNS records - relationships.append(( - domain, - target, - relationship_type, - confidence, - raw_data - )) + # Add relationship + result.add_relationship( + source_node=domain, + target_node=target, + relationship_type=relationship_type, + provider=self.name, + confidence=confidence, + raw_data=raw_data + ) + # Add DNS record as attribute to the source domain + dns_records.append(f"{record_type}: {target}") + + # Log relationship discovery self.log_relationship_discovery( source_node=domain, target_node=target, @@ -189,10 +226,20 @@ class DNSProvider(BaseProvider): discovery_method=f"dns_{record_type.lower()}_record" ) + # Add DNS records as a consolidated attribute + if dns_records: + result.add_attribute( + target_node=domain, + name='dns_records', + value=dns_records, + attr_type='dns_record_list', + provider=self.name, + confidence=0.8, + metadata={'record_types': [record_type]} + ) + except Exception as e: self.failed_requests += 1 self.logger.logger.debug(f"{record_type} record query failed for {domain}: {e}") # Re-raise the exception so the scanner can handle it - raise e - - return relationships \ No newline at end of file + raise e \ No newline at end of file diff --git a/providers/shodan_provider.py b/providers/shodan_provider.py index 30c48f5..21b530c 100644 --- a/providers/shodan_provider.py +++ b/providers/shodan_provider.py @@ -1,20 +1,20 @@ # dnsrecon/providers/shodan_provider.py import json -import os from pathlib import Path -from typing import List, Dict, Any, Tuple +from typing import Dict, Any from datetime import datetime, timezone import requests from .base_provider import BaseProvider +from core.provider_result import ProviderResult from utils.helpers import _is_valid_ip, _is_valid_domain class ShodanProvider(BaseProvider): """ Provider for querying Shodan API for IP address information. - Now uses session-specific API keys, is limited to IP-only queries, and includes caching. + Now returns standardized ProviderResult objects with caching support. """ def __init__(self, name=None, session_config=None): @@ -85,88 +85,156 @@ class ShodanProvider(BaseProvider): except (json.JSONDecodeError, ValueError, KeyError): return "stale" - def query_domain(self, domain: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]: + def query_domain(self, domain: str) -> ProviderResult: """ Domain queries are no longer supported for the Shodan provider. + + Args: + domain: Domain to investigate + + Returns: + Empty ProviderResult """ - return [] + return ProviderResult() - def query_ip(self, ip: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]: + def query_ip(self, ip: str) -> ProviderResult: """ - Query Shodan for information about an IP address, with caching of processed relationships. + Query Shodan for information about an IP address, with caching of processed data. + + Args: + ip: IP address to investigate + + Returns: + ProviderResult containing discovered relationships and attributes """ if not _is_valid_ip(ip) or not self.is_available(): - return [] + return ProviderResult() cache_file = self._get_cache_file_path(ip) cache_status = self._get_cache_status(cache_file) - relationships = [] + result = ProviderResult() try: if cache_status == "fresh": - relationships = self._load_from_cache(cache_file) - self.logger.logger.info(f"Using cached Shodan relationships for {ip}") - else: # "stale" or "not_found" + result = self._load_from_cache(cache_file) + self.logger.logger.info(f"Using cached Shodan data for {ip}") + else: # "stale" or "not_found" url = f"{self.base_url}/shodan/host/{ip}" params = {'key': self.api_key} response = self.make_request(url, method="GET", params=params, target_indicator=ip) if response and response.status_code == 200: data = response.json() - # Process the data into relationships BEFORE caching - relationships = self._process_shodan_data(ip, data) - self._save_to_cache(cache_file, relationships) # Save the processed relationships + # Process the data into ProviderResult BEFORE caching + result = self._process_shodan_data(ip, data) + self._save_to_cache(cache_file, result, data) # Save both result and raw data elif cache_status == "stale": # If API fails on a stale cache, use the old data - relationships = self._load_from_cache(cache_file) + result = self._load_from_cache(cache_file) except requests.exceptions.RequestException as e: self.logger.logger.error(f"Shodan API query failed for {ip}: {e}") if cache_status == "stale": - relationships = self._load_from_cache(cache_file) + result = self._load_from_cache(cache_file) - return relationships + return result - def _load_from_cache(self, cache_file_path: Path) -> List[Tuple[str, str, str, float, Dict[str, Any]]]: - """Load processed Shodan relationships from a cache file.""" + def _load_from_cache(self, cache_file_path: Path) -> ProviderResult: + """Load processed Shodan data from a cache file.""" try: with open(cache_file_path, 'r') as f: cache_content = json.load(f) - # The entire file content is the list of relationships - return cache_content.get("relationships", []) + + result = ProviderResult() + + # Reconstruct relationships + for rel_data in cache_content.get("relationships", []): + result.add_relationship( + source_node=rel_data["source_node"], + target_node=rel_data["target_node"], + relationship_type=rel_data["relationship_type"], + provider=rel_data["provider"], + confidence=rel_data["confidence"], + raw_data=rel_data.get("raw_data", {}) + ) + + # Reconstruct attributes + for attr_data in cache_content.get("attributes", []): + result.add_attribute( + target_node=attr_data["target_node"], + name=attr_data["name"], + value=attr_data["value"], + attr_type=attr_data["type"], + provider=attr_data["provider"], + confidence=attr_data["confidence"], + metadata=attr_data.get("metadata", {}) + ) + + return result + except (json.JSONDecodeError, FileNotFoundError, KeyError): - return [] + return ProviderResult() - def _save_to_cache(self, cache_file_path: Path, relationships: List[Tuple[str, str, str, float, Dict[str, Any]]]) -> None: - """Save processed Shodan relationships to a cache file.""" + def _save_to_cache(self, cache_file_path: Path, result: ProviderResult, raw_data: Dict[str, Any]) -> None: + """Save processed Shodan data to a cache file.""" try: cache_data = { "last_upstream_query": datetime.now(timezone.utc).isoformat(), - "relationships": relationships + "raw_data": raw_data, # Preserve original for forensic purposes + "relationships": [ + { + "source_node": rel.source_node, + "target_node": rel.target_node, + "relationship_type": rel.relationship_type, + "confidence": rel.confidence, + "provider": rel.provider, + "raw_data": rel.raw_data + } for rel in result.relationships + ], + "attributes": [ + { + "target_node": attr.target_node, + "name": attr.name, + "value": attr.value, + "type": attr.type, + "provider": attr.provider, + "confidence": attr.confidence, + "metadata": attr.metadata + } for attr in result.attributes + ] } with open(cache_file_path, 'w') as f: - json.dump(cache_data, f, separators=(',', ':')) + json.dump(cache_data, f, separators=(',', ':'), default=str) except Exception as e: self.logger.logger.warning(f"Failed to save Shodan cache for {cache_file_path.name}: {e}") - def _process_shodan_data(self, ip: str, data: Dict[str, Any]) -> List[Tuple[str, str, str, float, Dict[str, Any]]]: + def _process_shodan_data(self, ip: str, data: Dict[str, Any]) -> ProviderResult: """ - Process Shodan data to extract relationships. + Process Shodan data to extract relationships and attributes. + + Args: + ip: IP address queried + data: Raw Shodan response data + + Returns: + ProviderResult with relationships and attributes """ - relationships = [] + result = ProviderResult() # Extract hostname relationships hostnames = data.get('hostnames', []) for hostname in hostnames: if _is_valid_domain(hostname): - relationships.append(( - ip, - hostname, - 'a_record', - 0.8, - data - )) + result.add_relationship( + source_node=ip, + target_node=hostname, + relationship_type='a_record', + provider=self.name, + confidence=0.8, + raw_data=data + ) + self.log_relationship_discovery( source_node=ip, target_node=hostname, @@ -180,13 +248,15 @@ class ShodanProvider(BaseProvider): asn = data.get('asn') if asn: asn_name = f"AS{asn[2:]}" if isinstance(asn, str) and asn.startswith('AS') else f"AS{asn}" - relationships.append(( - ip, - asn_name, - 'asn_membership', - 0.7, - data - )) + result.add_relationship( + source_node=ip, + target_node=asn_name, + relationship_type='asn_membership', + provider=self.name, + confidence=0.7, + raw_data=data + ) + self.log_relationship_discovery( source_node=ip, target_node=asn_name, @@ -195,5 +265,67 @@ class ShodanProvider(BaseProvider): raw_data=data, discovery_method="shodan_asn_lookup" ) - - return relationships \ No newline at end of file + + # Add comprehensive Shodan host information as attributes + if 'ports' in data: + result.add_attribute( + target_node=ip, + name='ports', + value=data['ports'], + attr_type='network_info', + provider=self.name, + confidence=0.9 + ) + + if 'os' in data and data['os']: + result.add_attribute( + target_node=ip, + name='operating_system', + value=data['os'], + attr_type='system_info', + provider=self.name, + confidence=0.8 + ) + + if 'org' in data: + result.add_attribute( + target_node=ip, + name='organization', + value=data['org'], + attr_type='network_info', + provider=self.name, + confidence=0.8 + ) + + if 'country_name' in data: + result.add_attribute( + target_node=ip, + name='country', + value=data['country_name'], + attr_type='location_info', + provider=self.name, + confidence=0.9 + ) + + if 'city' in data: + result.add_attribute( + target_node=ip, + name='city', + value=data['city'], + attr_type='location_info', + provider=self.name, + confidence=0.8 + ) + + # Store complete Shodan data as a comprehensive attribute + result.add_attribute( + target_node=ip, + name='shodan_host_info', + value=data, # Complete Shodan response for full forensic detail + attr_type='comprehensive_data', + provider=self.name, + confidence=0.9, + metadata={'data_source': 'shodan_api', 'query_type': 'host_lookup'} + ) + + return result \ No newline at end of file diff --git a/static/js/graph.js b/static/js/graph.js index f1c703f..6f8d6f3 100644 --- a/static/js/graph.js +++ b/static/js/graph.js @@ -1,6 +1,7 @@ /** * Graph visualization module for DNSRecon * Handles network graph rendering using vis.js with proper large entity node hiding + * UPDATED: Now compatible with unified data model (StandardAttribute objects) */ const contextMenuCSS = ` .graph-context-menu { @@ -380,11 +381,15 @@ class GraphManager { const largeEntityMap = new Map(); graphData.nodes.forEach(node => { - if (node.type === 'large_entity' && node.attributes && Array.isArray(node.attributes.nodes)) { - node.attributes.nodes.forEach(nodeId => { - largeEntityMap.set(nodeId, node.id); - this.largeEntityMembers.add(nodeId); - }); + if (node.type === 'large_entity' && node.attributes) { + // UPDATED: Handle unified data model - look for 'nodes' attribute in the attributes list + const nodesAttribute = this.findAttributeByName(node.attributes, 'nodes'); + if (nodesAttribute && Array.isArray(nodesAttribute.value)) { + nodesAttribute.value.forEach(nodeId => { + largeEntityMap.set(nodeId, node.id); + this.largeEntityMembers.add(nodeId); + }); + } } }); @@ -466,8 +471,21 @@ class GraphManager { } /** - * Process node data with styling and metadata - * @param {Object} node - Raw node data + * UPDATED: Helper method to find an attribute by name in the standardized attributes list + * @param {Array} attributes - List of StandardAttribute objects + * @param {string} name - Attribute name to find + * @returns {Object|null} The attribute object if found, null otherwise + */ + findAttributeByName(attributes, name) { + if (!Array.isArray(attributes)) { + return null; + } + return attributes.find(attr => attr.name === name) || null; + } + + /** + * UPDATED: Process node data with styling and metadata for unified data model + * @param {Object} node - Raw node data with standardized attributes * @returns {Object} Processed node data */ processNode(node) { @@ -478,7 +496,7 @@ class GraphManager { size: this.getNodeSize(node.type), borderColor: this.getNodeBorderColor(node.type), shape: this.getNodeShape(node.type), - attributes: node.attributes || {}, + attributes: node.attributes || [], // Keep as standardized attributes list description: node.description || '', metadata: node.metadata || {}, type: node.type, @@ -491,9 +509,10 @@ class GraphManager { processedNode.borderWidth = Math.max(2, Math.floor(node.confidence * 5)); } - // Style based on certificate validity + // UPDATED: Style based on certificate validity using unified data model if (node.type === 'domain') { - if (node.attributes && node.attributes.certificates && node.attributes.certificates.has_valid_cert === false) { + const certificatesAttr = this.findAttributeByName(node.attributes, 'certificates'); + if (certificatesAttr && certificatesAttr.value && certificatesAttr.value.has_valid_cert === false) { processedNode.color = { background: '#888888', border: '#666666' }; } } diff --git a/static/js/main.js b/static/js/main.js index 7c7b920..5ce2477 100644 --- a/static/js/main.js +++ b/static/js/main.js @@ -1,6 +1,7 @@ /** * Main application logic for DNSRecon web interface * Handles UI interactions, API communication, and data flow + * UPDATED: Now compatible with unified data model (StandardAttribute objects) */ class DNSReconApp { @@ -808,10 +809,22 @@ class DNSReconApp { } /** - * Enhanced node details HTML generation with better visual hierarchy - * File: static/js/main.js (replace generateNodeDetailsHtml method) + * UPDATED: Helper method to find an attribute by name in the standardized attributes list + * @param {Array} attributes - List of StandardAttribute objects + * @param {string} name - Attribute name to find + * @returns {Object|null} The attribute object if found, null otherwise */ + findAttributeByName(attributes, name) { + if (!Array.isArray(attributes)) { + return null; + } + return attributes.find(attr => attr.name === name) || null; + } + /** + * UPDATED: Enhanced node details HTML generation for unified data model + * Now processes StandardAttribute objects instead of simple key-value pairs + */ generateNodeDetailsHtml(node) { if (!node) return '
Details not available.
'; @@ -857,23 +870,28 @@ class DNSReconApp { return detailsHtml; } + /** + * UPDATED: Generate details for standard nodes using unified data model + */ generateStandardNodeDetails(node) { let html = ''; // Relationships sections html += this.generateRelationshipsSection(node); - // Enhanced attributes section with special certificate handling - if (node.attributes && Object.keys(node.attributes).length > 0) { - const { certificates, ...otherAttributes } = node.attributes; + // UPDATED: Enhanced attributes section with special certificate handling for unified model + if (node.attributes && Array.isArray(node.attributes) && node.attributes.length > 0) { + // Find certificate attribute separately + const certificatesAttr = this.findAttributeByName(node.attributes, 'certificates'); // Handle certificates separately with enhanced display - if (certificates) { - html += this.generateCertificateSection({ certificates }); + if (certificatesAttr) { + html += this.generateCertificateSection(certificatesAttr); } - // Handle other attributes normally - if (Object.keys(otherAttributes).length > 0) { + // Handle other attributes normally (excluding certificates to avoid duplication) + const otherAttributes = node.attributes.filter(attr => attr.name !== 'certificates'); + if (otherAttributes.length > 0) { html += this.generateAttributesSection(otherAttributes); } } @@ -888,10 +906,10 @@ class DNSReconApp { } /** - * Enhanced certificate section generation using existing styles + * UPDATED: Enhanced certificate section generation for unified data model */ - generateCertificateSection(attributes) { - const certificates = attributes.certificates; + generateCertificateSection(certificatesAttr) { + const certificates = certificatesAttr.value; if (!certificates || typeof certificates !== 'object') { return ''; } @@ -1094,10 +1112,22 @@ class DNSReconApp { return html; } + /** + * UPDATED: Generate large entity details using unified data model + */ generateLargeEntityDetails(node) { - const attributes = node.attributes || {}; - const nodes = attributes.nodes || []; - const nodeType = attributes.node_type || 'nodes'; + // UPDATED: Look for attributes in the unified model structure + const nodesAttribute = this.findAttributeByName(node.attributes, 'nodes'); + const countAttribute = this.findAttributeByName(node.attributes, 'count'); + const nodeTypeAttribute = this.findAttributeByName(node.attributes, 'node_type'); + const sourceProviderAttribute = this.findAttributeByName(node.attributes, 'source_provider'); + const discoveryDepthAttribute = this.findAttributeByName(node.attributes, 'discovery_depth'); + + const nodes = nodesAttribute ? nodesAttribute.value : []; + const count = countAttribute ? countAttribute.value : 0; + const nodeType = nodeTypeAttribute ? nodeTypeAttribute.value : 'nodes'; + const sourceProvider = sourceProviderAttribute ? sourceProviderAttribute.value : 'Unknown'; + const discoveryDepth = discoveryDepthAttribute ? discoveryDepthAttribute.value : 'Unknown'; let html = `
@@ -1107,15 +1137,15 @@ class DNSReconApp {
Contains: - ${attributes.count} ${nodeType}s + ${count} ${nodeType}s
Provider: - ${attributes.source_provider || 'Unknown'} + ${sourceProvider}
Depth: - ${attributes.discovery_depth || 'Unknown'} + ${discoveryDepth}
@@ -1132,17 +1162,19 @@ class DNSReconApp { // Use node.id for the large_entity_id const largeEntityId = node.id; - nodes.forEach(innerNodeId => { - html += ` -
- ${innerNodeId} - -
- `; - }); + if (Array.isArray(nodes)) { + nodes.forEach(innerNodeId => { + html += ` +
+ ${innerNodeId} + +
+ `; + }); + } html += ''; @@ -1255,151 +1287,6 @@ class DNSReconApp { return valueSourceMap; } - generateCorrelationObjectLayout(node) { - const metadata = node.metadata || {}; - const values = metadata.values || []; - const mergeCount = metadata.merge_count || 1; - - let html = '
'; - - if (mergeCount > 1) { - html += ` -
-
-

🔗Merged Correlations

-
${mergeCount} values
-
-
- `; - - values.forEach((value, index) => { - const displayValue = typeof value === 'string' && value.length > 50 ? - value.substring(0, 47) + '...' : value; - - html += ` -
-
${displayValue}
- - -
- `; - }); - - html += '
'; - } else { - const singleValue = values.length > 0 ? values[0] : (metadata.value || 'Unknown'); - html += ` -
-
-

🔗Correlation Value

-
-
${singleValue}
-
- `; - } - - // Show correlated nodes - const correlatedNodes = metadata.correlated_nodes || []; - if (correlatedNodes.length > 0) { - html += ` -
-
-

🌐Correlated Nodes

-
${correlatedNodes.length}
-
-
- `; - - correlatedNodes.forEach(nodeId => { - html += ` -
- ● - ${nodeId} - -
- `; - }); - - html += '
'; - } - - html += '
'; - return html; - } - - generateLargeEntityLayout(node) { - const attributes = node.attributes || {}; - const nodes = attributes.nodes || []; - const nodeType = attributes.node_type || 'nodes'; - - let html = ` -
-
-
-

📦Large Entity Container

-
${attributes.count} ${nodeType}s
-
-
-
- Source Provider: - ${attributes.source_provider || 'Unknown'} -
-
- Discovery Depth: - ${attributes.discovery_depth || 'Unknown'} -
-
-
- -
-
-

📋Contained ${nodeType}s

- -
-
- `; - - nodes.forEach((innerNodeId, index) => { - const innerNode = this.graphManager.nodes.get(innerNodeId); - html += ` -
-
- ● - ${innerNodeId} - ▼ -
-
- ${innerNode ? this.generateStandardNodeLayout(innerNode) : '
No details available
'} -
-
- `; - }); - - html += '
'; - return html; - } - - generateStandardNodeLayout(node) { - let html = '
'; - - // Relationships section - html += this.generateRelationshipsSection(node); - - // Attributes section with smart categorization - html += this.generateAttributesSection(node); - - // Description section - html += this.generateDescriptionSection(node); - - // Metadata section (collapsed by default) - html += this.generateMetadataSection(node); - - html += '
'; - return html; - } - generateRelationshipsSection(node) { let html = ''; @@ -1468,12 +1355,20 @@ class DNSReconApp { return html; } + /** + * UPDATED: Generate attributes section for unified data model + * Now processes StandardAttribute objects instead of key-value pairs + */ generateAttributesSection(attributes) { - const categorized = this.categorizeAttributes(attributes); + if (!Array.isArray(attributes) || attributes.length === 0) { + return ''; + } + + const categorized = this.categorizeStandardAttributes(attributes); let html = ''; Object.entries(categorized).forEach(([category, attrs]) => { - if (Object.keys(attrs).length === 0) return; + if (attrs.length === 0) return; html += `
@@ -1482,20 +1377,16 @@ class DNSReconApp {
`; - if (category === 'Certificates' && attrs.certificates) { - html += this.formatCertificateData(attrs.certificates); - } else { - html += '
'; - Object.entries(attrs).forEach(([key, value]) => { - html += ` -
- ${this.formatLabel(key)} - ${this.formatAttributeValue(value)} -
- `; - }); - html += '
'; - } + html += '
'; + attrs.forEach(attr => { + html += ` +
+ ${this.formatLabel(attr.name)} + ${this.formatStandardAttributeValue(attr)} +
+ `; + }); + html += '
'; html += '
'; }); @@ -1503,47 +1394,41 @@ class DNSReconApp { return html; } - formatCertificateData(certData) { - if (!certData || typeof certData !== 'object') { - return '

No certificate data available

'; - } + /** + * UPDATED: Categorize StandardAttribute objects by type and content + */ + categorizeStandardAttributes(attributes) { + const categories = { + 'DNS Records': [], + 'Network Info': [], + 'Provider Data': [], + 'Other': [] + }; - let html = '
'; + attributes.forEach(attr => { + const lowerName = attr.name.toLowerCase(); + const attrType = attr.type ? attr.type.toLowerCase() : ''; + + if (lowerName.includes('dns') || lowerName.includes('record') || attrType.includes('dns')) { + categories['DNS Records'].push(attr); + } else if (lowerName.includes('ip') || lowerName.includes('asn') || lowerName.includes('network') || attrType.includes('network')) { + categories['Network Info'].push(attr); + } else if (lowerName.includes('shodan') || lowerName.includes('crtsh') || lowerName.includes('provider') || attrType.includes('provider')) { + categories['Provider Data'].push(attr); + } else { + categories['Other'].push(attr); + } + }); - // Handle certificate summary - if (certData.total_certificates) { - html += ` -
-
- Total Certificates: ${certData.total_certificates} - - ${certData.has_valid_cert ? 'Valid' : 'Invalid'} - -
-
- `; - } - - // Handle unique issuers - if (certData.unique_issuers && Array.isArray(certData.unique_issuers)) { - html += ` -
-
- Issuers: -
-
- `; - certData.unique_issuers.forEach(issuer => { - html += `
${this.escapeHtml(String(issuer))}
`; - }); - html += '
'; - } - - html += '
'; - return html; + return categories; } - formatAttributeValue(value) { + /** + * UPDATED: Format StandardAttribute value for display + */ + formatStandardAttributeValue(attr) { + const value = attr.value; + if (value === null || value === undefined) { return 'None'; } @@ -1567,35 +1452,6 @@ class DNSReconApp { return this.escapeHtml(String(value)); } - - categorizeAttributes(attributes) { - const categories = { - 'DNS Records': {}, - 'Certificates': {}, - 'Network Info': {}, - 'Provider Data': {}, - 'Other': {} - }; - - for (const [key, value] of Object.entries(attributes)) { - const lowerKey = key.toLowerCase(); - - if (lowerKey.includes('dns') || lowerKey.includes('record') || key.endsWith('_record')) { - categories['DNS Records'][key] = value; - } else if (lowerKey.includes('cert') || lowerKey.includes('ssl') || lowerKey.includes('tls')) { - categories['Certificates'][key] = value; - } else if (lowerKey.includes('ip') || lowerKey.includes('asn') || lowerKey.includes('network')) { - categories['Network Info'][key] = value; - } else if (lowerKey.includes('shodan') || lowerKey.includes('crtsh') || lowerKey.includes('provider')) { - categories['Provider Data'][key] = value; - } else { - categories['Other'][key] = value; - } - } - - return categories; - } - formatObjectCompact(obj) { if (!obj || typeof obj !== 'object') return ''; @@ -1625,7 +1481,7 @@ class DNSReconApp { return `
-

📝Description

+

📄Description

${this.escapeHtml(node.description)} @@ -1827,7 +1683,7 @@ class DNSReconApp { getNodeTypeIcon(nodeType) { const icons = { 'domain': '🌐', - 'ip': '📍', + 'ip': '🔍', 'asn': '🏢', 'large_entity': '📦', 'correlation_object': '🔗' From 733e1da640eab56e9c75c5659d0384d1e418fd45 Mon Sep 17 00:00:00 2001 From: overcuriousity Date: Tue, 16 Sep 2025 21:23:02 +0200 Subject: [PATCH 02/20] new data model refinement --- core/graph_manager.py | 235 ++++----------------------- core/scanner.py | 1 + providers/crtsh_provider.py | 86 +++------- providers/shodan_provider.py | 144 ++++++----------- static/js/graph.js | 12 +- static/js/main.js | 298 +++-------------------------------- 6 files changed, 129 insertions(+), 647 deletions(-) diff --git a/core/graph_manager.py b/core/graph_manager.py index 69dbda8..b7c2159 100644 --- a/core/graph_manager.py +++ b/core/graph_manager.py @@ -40,6 +40,7 @@ class GraphManager: self.correlation_index = {} # Compile regex for date filtering for efficiency self.date_pattern = re.compile(r'^\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}') + self.EXCLUDED_KEYS = ['confidence', 'provider', 'timestamp', 'type'] def __getstate__(self): """Prepare GraphManager for pickling, excluding compiled regex.""" @@ -54,145 +55,44 @@ class GraphManager: self.__dict__.update(state) self.date_pattern = re.compile(r'^\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}') - def _update_correlation_index(self, node_id: str, data: Any, path: List[str] = [], parent_attr: str = ""): - """Recursively traverse metadata and add hashable values to the index with better path tracking.""" - if path is None: - path = [] - - if isinstance(data, dict): - for key, value in data.items(): - self._update_correlation_index(node_id, value, path + [key], key) - elif isinstance(data, list): - for i, item in enumerate(data): - # Instead of just using [i], include the parent attribute context - list_path_component = f"[{i}]" if not parent_attr else f"{parent_attr}[{i}]" - self._update_correlation_index(node_id, item, path + [list_path_component], parent_attr) - else: - self._add_to_correlation_index(node_id, data, ".".join(path), parent_attr) - - def _add_to_correlation_index(self, node_id: str, value: Any, path_str: str, parent_attr: str = ""): - """Add a hashable value to the correlation index, filtering out noise.""" - if not isinstance(value, (str, int, float, bool)) or value is None: + def process_correlations_for_node(self, node_id: str): + """Process correlations for a given node based on its attributes.""" + if not self.graph.has_node(node_id): return - # Ignore certain paths that contain noisy, non-unique identifiers - if any(keyword in path_str.lower() for keyword in ['count', 'total', 'timestamp', 'date']): - return + node_attributes = self.graph.nodes[node_id].get('attributes', []) + for attr in node_attributes: + attr_name = attr.get('name') + attr_value = attr.get('value') - # Filter out common low-entropy values and date-like strings - if isinstance(value, str): - # FIXED: Prevent correlation on date/time strings. - if self.date_pattern.match(value): - return - if len(value) < 4 or value.lower() in ['true', 'false', 'unknown', 'none', 'crt.sh']: - return - elif isinstance(value, int) and (abs(value) < 1024 or abs(value) > 65535): - return # Ignore small integers and common port numbers - elif isinstance(value, bool): - return # Ignore boolean values + if attr_name in self.EXCLUDED_KEYS or not isinstance(attr_value, (str, int, float, bool)) or attr_value is None: + continue - # Add the valuable correlation data to the index - if value not in self.correlation_index: - self.correlation_index[value] = {} - if node_id not in self.correlation_index[value]: - self.correlation_index[value][node_id] = [] - - # Store both the full path and the parent attribute for better edge labeling - correlation_entry = { - 'path': path_str, - 'parent_attr': parent_attr, - 'meaningful_attr': self._extract_meaningful_attribute(path_str, parent_attr) - } - - if correlation_entry not in self.correlation_index[value][node_id]: - self.correlation_index[value][node_id].append(correlation_entry) - - def _extract_meaningful_attribute(self, path_str: str, parent_attr: str = "") -> str: - """Extract the most meaningful attribute name from a path string.""" - if not path_str: - return "unknown" - - path_parts = path_str.split('.') - - # Look for the last non-array-index part - for part in reversed(path_parts): - # Skip array indices like [0], [1], etc. - if not (part.startswith('[') and part.endswith(']') and part[1:-1].isdigit()): - # Clean up compound names like "hostnames[0]" to just "hostnames" - clean_part = re.sub(r'\[\d+\]$', '', part) - if clean_part: - return clean_part - - # Fallback to parent attribute if available - if parent_attr: - return parent_attr - - # Last resort - use the first meaningful part - for part in path_parts: - if not (part.startswith('[') and part.endswith(']') and part[1:-1].isdigit()): - clean_part = re.sub(r'\[\d+\]$', '', part) - if clean_part: - return clean_part - - return "correlation" - - def _check_for_correlations(self, new_node_id: str, data: Any, path: List[str] = [], parent_attr: str = "") -> List[Dict]: - """Recursively traverse metadata to find correlations with existing data.""" - if path is None: - path = [] - - all_correlations = [] - if isinstance(data, dict): - for key, value in data.items(): - if key == 'source': # Avoid correlating on the provider name - continue - all_correlations.extend(self._check_for_correlations(new_node_id, value, path + [key], key)) - elif isinstance(data, list): - for i, item in enumerate(data): - list_path_component = f"[{i}]" if not parent_attr else f"{parent_attr}[{i}]" - all_correlations.extend(self._check_for_correlations(new_node_id, item, path + [list_path_component], parent_attr)) - else: - value = data - if value in self.correlation_index: - existing_nodes_with_paths = self.correlation_index[value] - unique_nodes = set(existing_nodes_with_paths.keys()) - unique_nodes.add(new_node_id) - - if len(unique_nodes) < 2: - return all_correlations # Correlation must involve at least two distinct nodes - - new_source = { - 'node_id': new_node_id, - 'path': ".".join(path), - 'parent_attr': parent_attr, - 'meaningful_attr': self._extract_meaningful_attribute(".".join(path), parent_attr) - } - all_sources = [new_source] + if isinstance(attr_value, bool): + continue - for node_id, path_entries in existing_nodes_with_paths.items(): - for entry in path_entries: - if isinstance(entry, dict): - all_sources.append({ - 'node_id': node_id, - 'path': entry['path'], - 'parent_attr': entry.get('parent_attr', ''), - 'meaningful_attr': entry.get('meaningful_attr', self._extract_meaningful_attribute(entry['path'], entry.get('parent_attr', ''))) - }) - else: - # Handle legacy string-only entries - all_sources.append({ - 'node_id': node_id, - 'path': str(entry), - 'parent_attr': '', - 'meaningful_attr': self._extract_meaningful_attribute(str(entry)) - }) + if isinstance(attr_value, str) and (len(attr_value) < 4 or self.date_pattern.match(attr_value)): + continue + + if attr_value not in self.correlation_index: + self.correlation_index[attr_value] = set() + + self.correlation_index[attr_value].add(node_id) + + if len(self.correlation_index[attr_value]) > 1: + self._create_correlation_node_and_edges(attr_value, self.correlation_index[attr_value]) + + def _create_correlation_node_and_edges(self, value, nodes): + """Create a correlation node and edges to the correlated nodes.""" + correlation_node_id = f"corr_{value}" + if not self.graph.has_node(correlation_node_id): + self.add_node(correlation_node_id, NodeType.CORRELATION_OBJECT, + metadata={'value': value, 'correlated_nodes': list(nodes)}) + + for node_id in nodes: + if self.graph.has_node(node_id) and not self.graph.has_edge(node_id, correlation_node_id): + self.add_edge(node_id, correlation_node_id, "correlation", confidence_score=0.9) - all_correlations.append({ - 'value': value, - 'sources': all_sources, - 'nodes': list(unique_nodes) - }) - return all_correlations def add_node(self, node_id: str, node_type: NodeType, attributes: Optional[List[Dict[str, Any]]] = None, description: str = "", metadata: Optional[Dict[str, Any]] = None) -> bool: @@ -232,78 +132,9 @@ class GraphManager: existing_metadata.update(metadata) self.graph.nodes[node_id]['metadata'] = existing_metadata - if attributes and node_type != NodeType.CORRELATION_OBJECT: - correlations = self._check_for_correlations(node_id, attributes) - for corr in correlations: - value = corr['value'] - - # STEP 1: Substring check against all existing nodes - if self._correlation_value_matches_existing_node(value): - # Skip creating correlation node - would be redundant - continue - - eligible_nodes = set(corr['nodes']) - - if len(eligible_nodes) < 2: - # Need at least 2 nodes to create a correlation - continue - - # STEP 3: Check for existing correlation node with same connection pattern - correlation_nodes_with_pattern = self._find_correlation_nodes_with_same_pattern(eligible_nodes) - - if correlation_nodes_with_pattern: - # STEP 4: Merge with existing correlation node - target_correlation_node = correlation_nodes_with_pattern[0] - self._merge_correlation_values(target_correlation_node, value, corr) - else: - # STEP 5: Create new correlation node for eligible nodes only - correlation_node_id = f"corr_{abs(hash(str(sorted(eligible_nodes))))}" - self.add_node(correlation_node_id, NodeType.CORRELATION_OBJECT, - metadata={'values': [value], 'sources': corr['sources'], - 'correlated_nodes': list(eligible_nodes)}) - - # Create edges from eligible nodes to this correlation node with better labeling - for c_node_id in eligible_nodes: - if self.graph.has_node(c_node_id): - # Find the best attribute name for this node - meaningful_attr = self._find_best_attribute_name_for_node(c_node_id, corr['sources']) - relationship_type = f"c_{meaningful_attr}" - self.add_edge(c_node_id, correlation_node_id, relationship_type, confidence_score=0.9) - - self._update_correlation_index(node_id, attributes) - self.last_modified = datetime.now(timezone.utc).isoformat() return is_new_node - def _find_best_attribute_name_for_node(self, node_id: str, sources: List[Dict]) -> str: - """Find the best attribute name for a correlation edge by looking at the sources.""" - node_sources = [s for s in sources if s['node_id'] == node_id] - - if not node_sources: - return "correlation" - - # Use the meaningful_attr if available - for source in node_sources: - meaningful_attr = source.get('meaningful_attr') - if meaningful_attr and meaningful_attr != "unknown": - return meaningful_attr - - # Fallback to parent_attr - for source in node_sources: - parent_attr = source.get('parent_attr') - if parent_attr: - return parent_attr - - # Last resort - extract from path - for source in node_sources: - path = source.get('path', '') - if path: - extracted = self._extract_meaningful_attribute(path) - if extracted != "unknown": - return extracted - - return "correlation" - def _has_direct_edge_bidirectional(self, node_a: str, node_b: str) -> bool: """ Check if there's a direct edge between two nodes in either direction. diff --git a/core/scanner.py b/core/scanner.py index 11b5493..6b234b6 100644 --- a/core/scanner.py +++ b/core/scanner.py @@ -506,6 +506,7 @@ class Scanner: large_entity_members.update(discovered) else: new_targets.update(discovered) + self.graph.process_correlations_for_node(target) else: print(f"Stop requested after processing results from {provider.get_name()}") except Exception as e: diff --git a/providers/crtsh_provider.py b/providers/crtsh_provider.py index 4ccce97..d8bdfba 100644 --- a/providers/crtsh_provider.py +++ b/providers/crtsh_provider.py @@ -134,7 +134,7 @@ class CrtShProvider(BaseProvider): self.logger.logger.info(f"Refreshed and merged cache for {domain}") else: # "not_found" # Create new result from processed certs - result = self._process_certificates_to_result(domain, current_processed_certs) + result = self._process_certificates_to_result(domain, raw_certificates) self.logger.logger.info(f"Created fresh result for {domain} ({result.get_relationship_count()} relationships)") # Save the result to cache @@ -272,109 +272,73 @@ class CrtShProvider(BaseProvider): Process certificates to create ProviderResult with relationships and attributes. """ result = ProviderResult() - + if self._stop_event and self._stop_event.is_set(): print(f"CrtSh processing cancelled before processing for domain: {domain}") return result - # Aggregate certificate data by domain - domain_certificates = {} all_discovered_domains = set() - - # Process certificates with cancellation checking + for i, cert_data in enumerate(certificates): if i % 5 == 0 and self._stop_event and self._stop_event.is_set(): print(f"CrtSh processing cancelled at certificate {i} for domain: {domain}") break - - cert_metadata = self._extract_certificate_metadata(cert_data) + cert_domains = self._extract_domains_from_certificate(cert_data) - all_discovered_domains.update(cert_domains) + for cert_domain in cert_domains: if not _is_valid_domain(cert_domain): continue - - if cert_domain not in domain_certificates: - domain_certificates[cert_domain] = [] - - domain_certificates[cert_domain].append(cert_metadata) - + + for key, value in self._extract_certificate_metadata(cert_data).items(): + if value is not None: + result.add_attribute( + target_node=cert_domain, + name=f"cert_{key}", + value=value, + attr_type='certificate_data', + provider=self.name, + confidence=0.9 + ) + if self._stop_event and self._stop_event.is_set(): print(f"CrtSh query cancelled before relationship creation for domain: {domain}") return result - # Create relationships from query domain to ALL discovered domains for i, discovered_domain in enumerate(all_discovered_domains): if discovered_domain == domain: - continue # Skip self-relationships - + continue + if i % 10 == 0 and self._stop_event and self._stop_event.is_set(): print(f"CrtSh relationship creation cancelled for domain: {domain}") break if not _is_valid_domain(discovered_domain): continue - - # Get certificates for both domains - query_domain_certs = domain_certificates.get(domain, []) - discovered_domain_certs = domain_certificates.get(discovered_domain, []) - - # Find shared certificates - shared_certificates = self._find_shared_certificates(query_domain_certs, discovered_domain_certs) - - # Calculate confidence + confidence = self._calculate_domain_relationship_confidence( - domain, discovered_domain, shared_certificates, all_discovered_domains + domain, discovered_domain, [], all_discovered_domains ) - - # Create comprehensive raw data for the relationship - relationship_raw_data = { - 'relationship_type': 'certificate_discovery', - 'shared_certificates': shared_certificates, - 'total_shared_certs': len(shared_certificates), - 'discovery_context': self._determine_relationship_context(discovered_domain, domain), - 'domain_certificates': { - domain: self._summarize_certificates(query_domain_certs), - discovered_domain: self._summarize_certificates(discovered_domain_certs) - } - } - - # Add relationship + result.add_relationship( source_node=domain, target_node=discovered_domain, relationship_type='san_certificate', provider=self.name, confidence=confidence, - raw_data=relationship_raw_data + raw_data={'relationship_type': 'certificate_discovery'} ) - - # Log the relationship discovery + self.log_relationship_discovery( source_node=domain, target_node=discovered_domain, relationship_type='san_certificate', confidence_score=confidence, - raw_data=relationship_raw_data, + raw_data={'relationship_type': 'certificate_discovery'}, discovery_method="certificate_transparency_analysis" ) - # Add certificate summary as attributes for all domains that have certificates - for cert_domain, cert_list in domain_certificates.items(): - if cert_list: - cert_summary = self._summarize_certificates(cert_list) - - result.add_attribute( - target_node=cert_domain, - name='certificates', - value=cert_summary, - attr_type='certificate_data', - provider=self.name, - confidence=0.9, - metadata={'total_certificates': len(cert_list)} - ) - return result def _extract_certificate_metadata(self, cert_data: Dict[str, Any]) -> Dict[str, Any]: diff --git a/providers/shodan_provider.py b/providers/shodan_provider.py index 21b530c..6f3a044 100644 --- a/providers/shodan_provider.py +++ b/providers/shodan_provider.py @@ -222,110 +222,62 @@ class ShodanProvider(BaseProvider): """ result = ProviderResult() - # Extract hostname relationships - hostnames = data.get('hostnames', []) - for hostname in hostnames: - if _is_valid_domain(hostname): + for key, value in data.items(): + if key == 'hostnames': + for hostname in value: + if _is_valid_domain(hostname): + result.add_relationship( + source_node=ip, + target_node=hostname, + relationship_type='a_record', + provider=self.name, + confidence=0.8, + raw_data=data + ) + self.log_relationship_discovery( + source_node=ip, + target_node=hostname, + relationship_type='a_record', + confidence_score=0.8, + raw_data=data, + discovery_method="shodan_host_lookup" + ) + elif key == 'asn': + asn_name = f"AS{value[2:]}" if isinstance(value, str) and value.startswith('AS') else f"AS{value}" result.add_relationship( source_node=ip, - target_node=hostname, - relationship_type='a_record', + target_node=asn_name, + relationship_type='asn_membership', provider=self.name, - confidence=0.8, + confidence=0.7, raw_data=data ) - self.log_relationship_discovery( source_node=ip, - target_node=hostname, - relationship_type='a_record', - confidence_score=0.8, + target_node=asn_name, + relationship_type='asn_membership', + confidence_score=0.7, raw_data=data, - discovery_method="shodan_host_lookup" + discovery_method="shodan_asn_lookup" + ) + elif key == 'ports': + for port in value: + result.add_attribute( + target_node=ip, + name='open_port', + value=port, + attr_type='network_info', + provider=self.name, + confidence=0.9 + ) + elif isinstance(value, (str, int, float, bool)) and value is not None: + result.add_attribute( + target_node=ip, + name=f"shodan_{key}", + value=value, + attr_type='shodan_info', + provider=self.name, + confidence=0.9 ) - # Extract ASN relationship - asn = data.get('asn') - if asn: - asn_name = f"AS{asn[2:]}" if isinstance(asn, str) and asn.startswith('AS') else f"AS{asn}" - result.add_relationship( - source_node=ip, - target_node=asn_name, - relationship_type='asn_membership', - provider=self.name, - confidence=0.7, - raw_data=data - ) - - self.log_relationship_discovery( - source_node=ip, - target_node=asn_name, - relationship_type='asn_membership', - confidence_score=0.7, - raw_data=data, - discovery_method="shodan_asn_lookup" - ) - - # Add comprehensive Shodan host information as attributes - if 'ports' in data: - result.add_attribute( - target_node=ip, - name='ports', - value=data['ports'], - attr_type='network_info', - provider=self.name, - confidence=0.9 - ) - - if 'os' in data and data['os']: - result.add_attribute( - target_node=ip, - name='operating_system', - value=data['os'], - attr_type='system_info', - provider=self.name, - confidence=0.8 - ) - - if 'org' in data: - result.add_attribute( - target_node=ip, - name='organization', - value=data['org'], - attr_type='network_info', - provider=self.name, - confidence=0.8 - ) - - if 'country_name' in data: - result.add_attribute( - target_node=ip, - name='country', - value=data['country_name'], - attr_type='location_info', - provider=self.name, - confidence=0.9 - ) - - if 'city' in data: - result.add_attribute( - target_node=ip, - name='city', - value=data['city'], - attr_type='location_info', - provider=self.name, - confidence=0.8 - ) - - # Store complete Shodan data as a comprehensive attribute - result.add_attribute( - target_node=ip, - name='shodan_host_info', - value=data, # Complete Shodan response for full forensic detail - attr_type='comprehensive_data', - provider=self.name, - confidence=0.9, - metadata={'data_source': 'shodan_api', 'query_type': 'host_lookup'} - ) - return result \ No newline at end of file diff --git a/static/js/graph.js b/static/js/graph.js index 6f8d6f3..dd5bc13 100644 --- a/static/js/graph.js +++ b/static/js/graph.js @@ -1,7 +1,7 @@ /** * Graph visualization module for DNSRecon * Handles network graph rendering using vis.js with proper large entity node hiding - * UPDATED: Now compatible with unified data model (StandardAttribute objects) + * UPDATED: Now compatible with a strictly flat, unified data model for attributes. */ const contextMenuCSS = ` .graph-context-menu { @@ -484,7 +484,7 @@ class GraphManager { } /** - * UPDATED: Process node data with styling and metadata for unified data model + * UPDATED: Process node data with styling and metadata for the flat data model * @param {Object} node - Raw node data with standardized attributes * @returns {Object} Processed node data */ @@ -508,14 +508,6 @@ class GraphManager { if (node.confidence) { processedNode.borderWidth = Math.max(2, Math.floor(node.confidence * 5)); } - - // UPDATED: Style based on certificate validity using unified data model - if (node.type === 'domain') { - const certificatesAttr = this.findAttributeByName(node.attributes, 'certificates'); - if (certificatesAttr && certificatesAttr.value && certificatesAttr.value.has_valid_cert === false) { - processedNode.color = { background: '#888888', border: '#666666' }; - } - } // Handle merged correlation objects (similar to large entities) if (node.type === 'correlation_object') { diff --git a/static/js/main.js b/static/js/main.js index 5ce2477..d6dccf7 100644 --- a/static/js/main.js +++ b/static/js/main.js @@ -1,7 +1,7 @@ /** * Main application logic for DNSRecon web interface * Handles UI interactions, API communication, and data flow - * UPDATED: Now compatible with unified data model (StandardAttribute objects) + * UPDATED: Now compatible with a strictly flat, unified data model for attributes. */ class DNSReconApp { @@ -879,21 +879,9 @@ class DNSReconApp { // Relationships sections html += this.generateRelationshipsSection(node); - // UPDATED: Enhanced attributes section with special certificate handling for unified model + // UPDATED: Simplified attributes section for the flat model if (node.attributes && Array.isArray(node.attributes) && node.attributes.length > 0) { - // Find certificate attribute separately - const certificatesAttr = this.findAttributeByName(node.attributes, 'certificates'); - - // Handle certificates separately with enhanced display - if (certificatesAttr) { - html += this.generateCertificateSection(certificatesAttr); - } - - // Handle other attributes normally (excluding certificates to avoid duplication) - const otherAttributes = node.attributes.filter(attr => attr.name !== 'certificates'); - if (otherAttributes.length > 0) { - html += this.generateAttributesSection(otherAttributes); - } + html += this.generateAttributesSection(node.attributes); } // Description section @@ -905,213 +893,6 @@ class DNSReconApp { return html; } - /** - * UPDATED: Enhanced certificate section generation for unified data model - */ - generateCertificateSection(certificatesAttr) { - const certificates = certificatesAttr.value; - if (!certificates || typeof certificates !== 'object') { - return ''; - } - - let html = ` -
-
- 🔒 SSL/TLS Certificates -
- `; - - // Certificate summary using existing grid pattern - html += this.generateCertificateSummary(certificates); - - // Latest certificate info using existing attribute display - if (certificates.latest_certificate) { - html += this.generateLatestCertificateInfo(certificates.latest_certificate); - } - - // Detailed certificate list if available - if (certificates.certificate_details && Array.isArray(certificates.certificate_details)) { - html += this.generateCertificateList(certificates.certificate_details); - } - - html += '
'; - return html; - } - - /** - * Generate latest certificate info using existing attribute list - */ - generateLatestCertificateInfo(latest) { - const isValid = latest.is_currently_valid; - const statusText = isValid ? 'Valid' : 'Invalid/Expired'; - const statusColor = isValid ? '#00ff41' : '#ff6b6b'; - - let html = ` -
-
Most Recent Certificate
-
-
- Status: - ${statusText} -
-
- Issued: - ${latest.not_before || 'Unknown'} -
-
- Expires: - ${latest.not_after || 'Unknown'} -
-
- Issuer: - ${this.escapeHtml(latest.issuer_name || 'Unknown')} -
- ${latest.certificate_id ? ` -
- Certificate: - - - View on crt.sh ↗ - - -
- ` : ''} -
-
- `; - - return html; - } - - /** - * Generate certificate list using existing collapsible structure - */ - generateCertificateList(certificateDetails) { - if (!certificateDetails || certificateDetails.length === 0) { - return ''; - } - - // Limit display to prevent overwhelming the UI - const maxDisplay = 8; - const certificates = certificateDetails.slice(0, maxDisplay); - const remaining = certificateDetails.length - maxDisplay; - - let html = ` -
- 📋 Certificate Details (${certificates.length}${remaining > 0 ? ` of ${certificateDetails.length}` : ''}) -
- `; - - certificates.forEach((cert, index) => { - const isValid = cert.is_currently_valid; - let statusText = isValid ? '✅ Valid' : '❌ Invalid/Expired'; - let statusColor = isValid ? '#00ff41' : '#ff6b6b'; - - if (cert.expires_soon && isValid) { - statusText = '⚠️ Valid (Expiring Soon)'; - statusColor = '#ff9900'; - } - - html += ` -
-
- #${index + 1} - ${statusText} - ${cert.certificate_id ? ` - crt.sh ↗ - ` : ''} -
-
-
- Common Name: - ${this.escapeHtml(cert.common_name || 'N/A')} -
-
- Issuer: - ${this.escapeHtml(cert.issuer_name || 'Unknown')} -
-
- Valid From: - ${cert.not_before || 'Unknown'} -
-
- Valid Until: - ${cert.not_after || 'Unknown'} -
- ${cert.validity_period_days ? ` -
- Period: - ${cert.validity_period_days} days -
- ` : ''} -
-
- `; - }); - - if (remaining > 0) { - html += ` -
- 📋 ${remaining} additional certificate${remaining > 1 ? 's' : ''} not shown.
- Use the export function to see all certificates. -
- `; - } - - html += '
'; - return html; - } - - /** - * Generate certificate summary using minimal new CSS - */ - generateCertificateSummary(certificates) { - const total = certificates.total_certificates || 0; - const valid = certificates.valid_certificates || 0; - const expired = certificates.expired_certificates || 0; - const expiringSoon = certificates.expires_soon_count || 0; - const issuers = certificates.unique_issuers || []; - - let html = ` -
-
-
${total}
-
Total
-
-
-
${valid}
-
Valid
-
-
-
${expired}
-
Expired
-
-
-
${expiringSoon}
-
Expiring Soon
-
-
- `; - - // Certificate authorities using existing array display - if (issuers.length > 0) { - html += ` -
- Certificate Authorities: - -
- `; - - issuers.forEach(issuer => { - html += `
${this.escapeHtml(issuer)}
`; - }); - - html += '
'; - } - - return html; - } - /** * UPDATED: Generate large entity details using unified data model */ @@ -1356,71 +1137,32 @@ class DNSReconApp { } /** - * UPDATED: Generate attributes section for unified data model - * Now processes StandardAttribute objects instead of key-value pairs + * UPDATED: Generate attributes section for the new flat data model */ generateAttributesSection(attributes) { if (!Array.isArray(attributes) || attributes.length === 0) { return ''; } - const categorized = this.categorizeStandardAttributes(attributes); - let html = ''; - - Object.entries(categorized).forEach(([category, attrs]) => { - if (attrs.length === 0) return; - - html += ` -
-
- 📊 ${category} -
- `; - - html += '
'; - attrs.forEach(attr => { - html += ` -
- ${this.formatLabel(attr.name)} - ${this.formatStandardAttributeValue(attr)} -
- `; - }); - html += '
'; - - html += '
'; - }); - - return html; - } + let html = ` +
+
+ 📊 Attributes (${attributes.length}) +
+
+ `; - /** - * UPDATED: Categorize StandardAttribute objects by type and content - */ - categorizeStandardAttributes(attributes) { - const categories = { - 'DNS Records': [], - 'Network Info': [], - 'Provider Data': [], - 'Other': [] - }; - attributes.forEach(attr => { - const lowerName = attr.name.toLowerCase(); - const attrType = attr.type ? attr.type.toLowerCase() : ''; - - if (lowerName.includes('dns') || lowerName.includes('record') || attrType.includes('dns')) { - categories['DNS Records'].push(attr); - } else if (lowerName.includes('ip') || lowerName.includes('asn') || lowerName.includes('network') || attrType.includes('network')) { - categories['Network Info'].push(attr); - } else if (lowerName.includes('shodan') || lowerName.includes('crtsh') || lowerName.includes('provider') || attrType.includes('provider')) { - categories['Provider Data'].push(attr); - } else { - categories['Other'].push(attr); - } + html += ` +
+ ${this.formatLabel(attr.name)} + ${this.formatStandardAttributeValue(attr)} +
+ `; }); - - return categories; + + html += '
'; + return html; } /** From 229746e1ec7dabd9242798cd2310090f668d0481 Mon Sep 17 00:00:00 2001 From: overcuriousity Date: Tue, 16 Sep 2025 22:25:46 +0200 Subject: [PATCH 03/20] improving the display --- core/graph_manager.py | 159 ++++++++++++++++++++++----- static/js/main.js | 246 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 372 insertions(+), 33 deletions(-) diff --git a/core/graph_manager.py b/core/graph_manager.py index b7c2159..6820ed5 100644 --- a/core/graph_manager.py +++ b/core/graph_manager.py @@ -56,15 +56,22 @@ class GraphManager: self.date_pattern = re.compile(r'^\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}') def process_correlations_for_node(self, node_id: str): - """Process correlations for a given node based on its attributes.""" + """ + UPDATED: Process correlations for a given node with enhanced tracking. + Now properly tracks which attribute/provider created each correlation. + """ if not self.graph.has_node(node_id): return node_attributes = self.graph.nodes[node_id].get('attributes', []) + + # Process each attribute for potential correlations for attr in node_attributes: attr_name = attr.get('name') attr_value = attr.get('value') + attr_provider = attr.get('provider', 'unknown') + # Skip excluded attributes and invalid values if attr_name in self.EXCLUDED_KEYS or not isinstance(attr_value, (str, int, float, bool)) or attr_value is None: continue @@ -74,25 +81,91 @@ class GraphManager: if isinstance(attr_value, str) and (len(attr_value) < 4 or self.date_pattern.match(attr_value)): continue + # Initialize correlation tracking for this value if attr_value not in self.correlation_index: - self.correlation_index[attr_value] = set() + self.correlation_index[attr_value] = { + 'nodes': set(), + 'sources': [] # Track which provider/attribute combinations contributed + } - self.correlation_index[attr_value].add(node_id) + # Add this node and source information + self.correlation_index[attr_value]['nodes'].add(node_id) + + # Track the source of this correlation value + source_info = { + 'node_id': node_id, + 'provider': attr_provider, + 'attribute': attr_name, + 'path': f"{attr_provider}_{attr_name}" + } + + # Add source if not already present (avoid duplicates) + existing_sources = [s for s in self.correlation_index[attr_value]['sources'] + if s['node_id'] == node_id and s['path'] == source_info['path']] + if not existing_sources: + self.correlation_index[attr_value]['sources'].append(source_info) - if len(self.correlation_index[attr_value]) > 1: - self._create_correlation_node_and_edges(attr_value, self.correlation_index[attr_value]) + # Create correlation node if we have multiple nodes with this value + if len(self.correlation_index[attr_value]['nodes']) > 1: + self._create_enhanced_correlation_node_and_edges(attr_value, self.correlation_index[attr_value]) - def _create_correlation_node_and_edges(self, value, nodes): - """Create a correlation node and edges to the correlated nodes.""" - correlation_node_id = f"corr_{value}" + def _create_enhanced_correlation_node_and_edges(self, value, correlation_data): + """ + UPDATED: Create correlation node and edges with detailed provider tracking. + """ + correlation_node_id = f"corr_{hash(str(value)) & 0x7FFFFFFF}" + nodes = correlation_data['nodes'] + sources = correlation_data['sources'] + + # Create or update correlation node if not self.graph.has_node(correlation_node_id): - self.add_node(correlation_node_id, NodeType.CORRELATION_OBJECT, - metadata={'value': value, 'correlated_nodes': list(nodes)}) + # Determine the most common provider/attribute combination + provider_counts = {} + for source in sources: + key = f"{source['provider']}_{source['attribute']}" + provider_counts[key] = provider_counts.get(key, 0) + 1 + + # Use the most common provider/attribute as the primary label + primary_source = max(provider_counts.items(), key=lambda x: x[1])[0] if provider_counts else "unknown_correlation" + + metadata = { + 'value': value, + 'correlated_nodes': list(nodes), + 'sources': sources, + 'primary_source': primary_source, + 'correlation_count': len(nodes) + } + + self.add_node(correlation_node_id, NodeType.CORRELATION_OBJECT, metadata=metadata) + print(f"Created correlation node {correlation_node_id} for value '{value}' with {len(nodes)} nodes") - for node_id in nodes: + # Create edges from each node to the correlation node + for source in sources: + node_id = source['node_id'] + provider = source['provider'] + attribute = source['attribute'] + if self.graph.has_node(node_id) and not self.graph.has_edge(node_id, correlation_node_id): - self.add_edge(node_id, correlation_node_id, "correlation", confidence_score=0.9) - + # Format relationship label as "provider: attribute" + display_provider = provider + display_attribute = attribute.replace('_', ' ').replace('cert ', '').strip() + + relationship_label = f"{display_provider}: {display_attribute}" + + self.add_edge( + source_id=node_id, + target_id=correlation_node_id, + relationship_type=relationship_label, + confidence_score=0.9, + source_provider=provider, + raw_data={ + 'correlation_value': value, + 'original_attribute': attribute, + 'correlation_type': 'attribute_matching' + } + ) + + print(f"Added correlation edge: {node_id} -> {correlation_node_id} ({relationship_label})") def add_node(self, node_id: str, node_type: NodeType, attributes: Optional[List[Dict[str, Any]]] = None, description: str = "", metadata: Optional[Dict[str, Any]] = None) -> bool: @@ -335,26 +408,56 @@ class GraphManager: def get_graph_data(self) -> Dict[str, Any]: """ Export graph data formatted for frontend visualization. - Compatible with unified data model - preserves all attribute information for frontend display. + UPDATED: Fixed certificate validity styling logic for unified data model. """ nodes = [] for node_id, attrs in self.graph.nodes(data=True): node_data = {'id': node_id, 'label': node_id, 'type': attrs.get('type', 'unknown'), - 'attributes': attrs.get('attributes', []), # Ensure attributes is a list - 'description': attrs.get('description', ''), - 'metadata': attrs.get('metadata', {}), - 'added_timestamp': attrs.get('added_timestamp')} + 'attributes': attrs.get('attributes', []), # Ensure attributes is a list + 'description': attrs.get('description', ''), + 'metadata': attrs.get('metadata', {}), + 'added_timestamp': attrs.get('added_timestamp')} - # Customize node appearance based on type and attributes + # UPDATED: Fixed certificate validity styling logic node_type = node_data['type'] attributes_list = node_data['attributes'] - # CORRECTED LOGIC: Handle certificate validity styling if node_type == 'domain' and isinstance(attributes_list, list): - # Find the certificates attribute in the list - cert_attr = next((attr for attr in attributes_list if attr.get('name') == 'certificates'), None) - if cert_attr and cert_attr.get('value', {}).get('has_valid_cert') is False: - node_data['color'] = {'background': '#c7c7c7', 'border': '#999'} # Gray for invalid cert + # Check for certificate-related attributes + has_certificates = False + has_valid_certificates = False + has_expired_certificates = False + + for attr in attributes_list: + attr_name = attr.get('name', '').lower() + attr_provider = attr.get('provider', '').lower() + attr_value = attr.get('value') + + # Look for certificate attributes from crt.sh provider + if attr_provider == 'crtsh' or 'cert' in attr_name: + has_certificates = True + + # Check certificate validity + if attr_name == 'cert_is_currently_valid': + if attr_value is True: + has_valid_certificates = True + elif attr_value is False: + has_expired_certificates = True + + # Also check for certificate expiry indicators + elif 'expires_soon' in attr_name and attr_value is True: + has_expired_certificates = True + elif 'expired' in attr_name and attr_value is True: + has_expired_certificates = True + + # Apply styling based on certificate status + if has_expired_certificates and not has_valid_certificates: + # Red for expired/invalid certificates + node_data['color'] = {'background': '#ff6b6b', 'border': '#cc5555'} + elif not has_certificates: + # Grey for domains with no certificates + node_data['color'] = {'background': '#c7c7c7', 'border': '#999999'} + # Default green styling is handled by the frontend for domains with valid certificates # Add incoming and outgoing edges to node data if self.graph.has_node(node_id): @@ -366,10 +469,10 @@ class GraphManager: edges = [] for source, target, attrs in self.graph.edges(data=True): edges.append({'from': source, 'to': target, - 'label': attrs.get('relationship_type', ''), - 'confidence_score': attrs.get('confidence_score', 0), - 'source_provider': attrs.get('source_provider', ''), - 'discovery_timestamp': attrs.get('discovery_timestamp')}) + 'label': attrs.get('relationship_type', ''), + 'confidence_score': attrs.get('confidence_score', 0), + 'source_provider': attrs.get('source_provider', ''), + 'discovery_timestamp': attrs.get('discovery_timestamp')}) return { 'nodes': nodes, 'edges': edges, 'statistics': self.get_statistics()['basic_metrics'] diff --git a/static/js/main.js b/static/js/main.js index d6dccf7..42c4a11 100644 --- a/static/js/main.js +++ b/static/js/main.js @@ -822,8 +822,8 @@ class DNSReconApp { } /** - * UPDATED: Enhanced node details HTML generation for unified data model - * Now processes StandardAttribute objects instead of simple key-value pairs + * Node details HTML generation for unified data model + * processes StandardAttribute objects */ generateNodeDetailsHtml(node) { if (!node) return '
Details not available.
'; @@ -871,7 +871,7 @@ class DNSReconApp { } /** - * UPDATED: Generate details for standard nodes using unified data model + * Generate details for standard nodes using unified data model */ generateStandardNodeDetails(node) { let html = ''; @@ -879,9 +879,9 @@ class DNSReconApp { // Relationships sections html += this.generateRelationshipsSection(node); - // UPDATED: Simplified attributes section for the flat model + // Attributes section with grouping if (node.attributes && Array.isArray(node.attributes) && node.attributes.length > 0) { - html += this.generateAttributesSection(node.attributes); + html += this.generateEnhancedAttributesSection(node.attributes, node.type); } // Description section @@ -893,6 +893,242 @@ class DNSReconApp { return html; } + generateEnhancedAttributesSection(attributes, nodeType) { + if (!Array.isArray(attributes) || attributes.length === 0) { + return ''; + } + + // Group attributes by provider and type for better organization + const groupedAttributes = this.groupAttributesIntelligently(attributes, nodeType); + + let html = ''; + + for (const [groupName, groupData] of Object.entries(groupedAttributes)) { + const isDefaultOpen = groupData.priority === 'high'; + + html += ` +
+
+ + ${groupData.icon} ${groupName} + ${groupData.attributes.length} + +
+
+ `; + + groupData.attributes.forEach(attr => { + html += ` +
+ ${this.formatAttributeLabel(attr.name)} + ${this.formatAttributeValueEnhanced(attr)} +
+ `; + }); + + html += '
'; + } + + return html; + } + + formatAttributeValueEnhanced(attr) { + const value = attr.value; + + if (value === null || value === undefined) { + return 'None'; + } + + if (Array.isArray(value)) { + if (value.length === 0) return 'None'; + if (value.length === 1) return this.escapeHtml(String(value[0])); + + // Complex array - make it collapsible + const previewItems = value.slice(0, 2); + const hasMore = value.length > 2; + + let html = '
'; + html += `
`; + + previewItems.forEach(item => { + html += `
${this.escapeHtml(String(item))}
`; + }); + + if (hasMore) { + html += ` + + `; + } + + html += '
'; + + if (hasMore) { + html += `'; + } + + html += '
'; + return html; + } + + if (typeof value === 'object' && value !== null) { + return this.formatObjectExpandable(value); + } + + return this.escapeHtml(String(value)); + } + + /** + * NEW: Format objects as expandable content + */ + formatObjectExpandable(obj) { + if (!obj || typeof obj !== 'object') return ''; + + const entries = Object.entries(obj); + if (entries.length === 0) return 'Empty'; + + if (entries.length <= 3) { + // Simple inline display for small objects + let html = '
'; + entries.forEach(([key, value]) => { + html += `
${key}: ${this.escapeHtml(String(value))}
`; + }); + html += '
'; + return html; + } + + // Expandable display for complex objects + let html = '
'; + html += ` +
+ ${entries[0][0]}: ${this.escapeHtml(String(entries[0][1]))} + +
+ +
+ `; + + return html; + } + + formatAttributeLabel(name) { + // Handle provider prefixed attributes + if (name.includes('_')) { + const parts = name.split('_'); + if (parts.length >= 2) { + const provider = parts[0]; + const attribute = parts.slice(1).join('_'); + return `${this.provider}: ${this.formatLabel(attribute)}`; + } + } + + return this.formatLabel(name); + } + + groupAttributesIntelligently(attributes, nodeType) { + const groups = {}; + + // Define group configurations + const groupConfigs = { + 'DNS Records': { + icon: '🔍', + priority: 'high', + keywords: ['dns', 'record', 'a_record', 'cname', 'mx', 'ns', 'txt', 'ptr'], + providers: ['dns'] + }, + 'Certificate Information': { + icon: '🔒', + priority: 'high', + keywords: ['cert', 'certificate', 'ssl', 'tls', 'issuer', 'validity', 'san'], + providers: ['crtsh'] + }, + 'Network Information': { + icon: '🌐', + priority: 'high', + keywords: ['port', 'service', 'banner', 'asn', 'organization', 'country', 'city'], + providers: ['shodan'] + }, + 'Correlation Data': { + icon: '🔗', + priority: 'medium', + keywords: ['correlation', 'shared', 'common'], + providers: [] + } + }; + + // Initialize groups + Object.entries(groupConfigs).forEach(([name, config]) => { + groups[name] = { + ...config, + attributes: [] + }; + }); + + // Add a catch-all group + groups['Other Information'] = { + icon: '📋', + priority: 'low', + attributes: [] + }; + + // Classify attributes into groups + attributes.forEach(attr => { + let assigned = false; + + // Try to assign to a specific group based on provider or keywords + for (const [groupName, config] of Object.entries(groupConfigs)) { + const matchesProvider = config.providers.includes(attr.provider); + const matchesKeyword = config.keywords.some(keyword => + attr.name.toLowerCase().includes(keyword) || + attr.type.toLowerCase().includes(keyword) + ); + + if (matchesProvider || matchesKeyword) { + groups[groupName].attributes.push(attr); + assigned = true; + break; + } + } + + // If not assigned to any specific group, put in "Other" + if (!assigned) { + groups['Other Information'].attributes.push(attr); + } + }); + + // Remove empty groups + Object.keys(groups).forEach(groupName => { + if (groups[groupName].attributes.length === 0) { + delete groups[groupName]; + } + }); + + return groups; + } + /** * UPDATED: Generate large entity details using unified data model */ From 47ce7ff8836efb6d7db0e5cc2b005d68915db9e1 Mon Sep 17 00:00:00 2001 From: overcuriousity Date: Tue, 16 Sep 2025 23:17:23 +0200 Subject: [PATCH 04/20] format keys reduction --- core/graph_manager.py | 129 +++++---- static/js/main.js | 618 ++++++++++++------------------------------ 2 files changed, 249 insertions(+), 498 deletions(-) diff --git a/core/graph_manager.py b/core/graph_manager.py index 6820ed5..edd31c3 100644 --- a/core/graph_manager.py +++ b/core/graph_manager.py @@ -4,6 +4,7 @@ Graph data model for DNSRecon using NetworkX. Manages in-memory graph storage with confidence scoring and forensic metadata. Now fully compatible with the unified ProviderResult data model. +UPDATED: Fixed certificate styling and correlation edge labeling. """ import re from datetime import datetime, timezone @@ -40,7 +41,7 @@ class GraphManager: self.correlation_index = {} # Compile regex for date filtering for efficiency self.date_pattern = re.compile(r'^\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}') - self.EXCLUDED_KEYS = ['confidence', 'provider', 'timestamp', 'type'] + self.EXCLUDED_KEYS = ['confidence', 'provider', 'timestamp', 'type','crtsh_cert_validity_period_days'] def __getstate__(self): """Prepare GraphManager for pickling, excluding compiled regex.""" @@ -101,7 +102,7 @@ class GraphManager: # Add source if not already present (avoid duplicates) existing_sources = [s for s in self.correlation_index[attr_value]['sources'] - if s['node_id'] == node_id and s['path'] == source_info['path']] + if s['node_id'] == node_id and s['path'] == source_info['path']] if not existing_sources: self.correlation_index[attr_value]['sources'].append(source_info) @@ -146,11 +147,8 @@ class GraphManager: attribute = source['attribute'] if self.graph.has_node(node_id) and not self.graph.has_edge(node_id, correlation_node_id): - # Format relationship label as "provider: attribute" - display_provider = provider - display_attribute = attribute.replace('_', ' ').replace('cert ', '').strip() - - relationship_label = f"{display_provider}: {display_attribute}" + # Format relationship label as "corr_provider_attribute" + relationship_label = f"corr_{provider}_{attribute}" self.add_edge( source_id=node_id, @@ -167,46 +165,6 @@ class GraphManager: print(f"Added correlation edge: {node_id} -> {correlation_node_id} ({relationship_label})") - def add_node(self, node_id: str, node_type: NodeType, attributes: Optional[List[Dict[str, Any]]] = None, - description: str = "", metadata: Optional[Dict[str, Any]] = None) -> bool: - """ - Add a node to the graph, update attributes, and process correlations. - Now compatible with unified data model - attributes are dictionaries from converted StandardAttribute objects. - """ - is_new_node = not self.graph.has_node(node_id) - if is_new_node: - self.graph.add_node(node_id, type=node_type.value, - added_timestamp=datetime.now(timezone.utc).isoformat(), - attributes=attributes or [], # Store as a list from the start - description=description, - metadata=metadata or {}) - else: - # Safely merge new attributes into the existing list of attributes - if attributes: - existing_attributes = self.graph.nodes[node_id].get('attributes', []) - - # Handle cases where old data might still be in dictionary format - if not isinstance(existing_attributes, list): - existing_attributes = [] - - # Create a set of existing attribute names for efficient duplicate checking - existing_attr_names = {attr['name'] for attr in existing_attributes} - - for new_attr in attributes: - if new_attr['name'] not in existing_attr_names: - existing_attributes.append(new_attr) - existing_attr_names.add(new_attr['name']) - - self.graph.nodes[node_id]['attributes'] = existing_attributes - if description: - self.graph.nodes[node_id]['description'] = description - if metadata: - existing_metadata = self.graph.nodes[node_id].get('metadata', {}) - existing_metadata.update(metadata) - self.graph.nodes[node_id]['metadata'] = existing_metadata - - self.last_modified = datetime.now(timezone.utc).isoformat() - return is_new_node def _has_direct_edge_bidirectional(self, node_a: str, node_b: str) -> bool: """ @@ -303,6 +261,47 @@ class GraphManager: f"across {node_count} nodes" ) + def add_node(self, node_id: str, node_type: NodeType, attributes: Optional[List[Dict[str, Any]]] = None, + description: str = "", metadata: Optional[Dict[str, Any]] = None) -> bool: + """ + Add a node to the graph, update attributes, and process correlations. + Now compatible with unified data model - attributes are dictionaries from converted StandardAttribute objects. + """ + is_new_node = not self.graph.has_node(node_id) + if is_new_node: + self.graph.add_node(node_id, type=node_type.value, + added_timestamp=datetime.now(timezone.utc).isoformat(), + attributes=attributes or [], # Store as a list from the start + description=description, + metadata=metadata or {}) + else: + # Safely merge new attributes into the existing list of attributes + if attributes: + existing_attributes = self.graph.nodes[node_id].get('attributes', []) + + # Handle cases where old data might still be in dictionary format + if not isinstance(existing_attributes, list): + existing_attributes = [] + + # Create a set of existing attribute names for efficient duplicate checking + existing_attr_names = {attr['name'] for attr in existing_attributes} + + for new_attr in attributes: + if new_attr['name'] not in existing_attr_names: + existing_attributes.append(new_attr) + existing_attr_names.add(new_attr['name']) + + self.graph.nodes[node_id]['attributes'] = existing_attributes + if description: + self.graph.nodes[node_id]['description'] = description + if metadata: + existing_metadata = self.graph.nodes[node_id].get('metadata', {}) + existing_metadata.update(metadata) + self.graph.nodes[node_id]['metadata'] = existing_metadata + + self.last_modified = datetime.now(timezone.utc).isoformat() + return is_new_node + def add_edge(self, source_id: str, target_id: str, relationship_type: str, confidence_score: float = 0.5, source_provider: str = "unknown", raw_data: Optional[Dict[str, Any]] = None) -> bool: @@ -369,11 +368,21 @@ class GraphManager: # Clean up the correlation index keys_to_delete = [] - for value, nodes in self.correlation_index.items(): - if node_id in nodes: - del nodes[node_id] - if not nodes: # If no other nodes are associated with this value, remove it - keys_to_delete.append(value) + for value, data in self.correlation_index.items(): + if isinstance(data, dict) and 'nodes' in data: + # Updated correlation structure + if node_id in data['nodes']: + data['nodes'].discard(node_id) + # Remove sources for this node + data['sources'] = [s for s in data['sources'] if s['node_id'] != node_id] + if not data['nodes']: # If no other nodes are associated, remove it + keys_to_delete.append(value) + else: + # Legacy correlation structure (fallback) + if isinstance(data, set) and node_id in data: + data.discard(node_id) + if not data: + keys_to_delete.append(value) for key in keys_to_delete: if key in self.correlation_index: @@ -413,10 +422,10 @@ class GraphManager: nodes = [] for node_id, attrs in self.graph.nodes(data=True): node_data = {'id': node_id, 'label': node_id, 'type': attrs.get('type', 'unknown'), - 'attributes': attrs.get('attributes', []), # Ensure attributes is a list - 'description': attrs.get('description', ''), - 'metadata': attrs.get('metadata', {}), - 'added_timestamp': attrs.get('added_timestamp')} + 'attributes': attrs.get('attributes', []), # Ensure attributes is a list + 'description': attrs.get('description', ''), + 'metadata': attrs.get('metadata', {}), + 'added_timestamp': attrs.get('added_timestamp')} # UPDATED: Fixed certificate validity styling logic node_type = node_data['type'] @@ -469,10 +478,10 @@ class GraphManager: edges = [] for source, target, attrs in self.graph.edges(data=True): edges.append({'from': source, 'to': target, - 'label': attrs.get('relationship_type', ''), - 'confidence_score': attrs.get('confidence_score', 0), - 'source_provider': attrs.get('source_provider', ''), - 'discovery_timestamp': attrs.get('discovery_timestamp')}) + 'label': attrs.get('relationship_type', ''), + 'confidence_score': attrs.get('confidence_score', 0), + 'source_provider': attrs.get('source_provider', ''), + 'discovery_timestamp': attrs.get('discovery_timestamp')}) return { 'nodes': nodes, 'edges': edges, 'statistics': self.get_statistics()['basic_metrics'] diff --git a/static/js/main.js b/static/js/main.js index 42c4a11..27be70b 100644 --- a/static/js/main.js +++ b/static/js/main.js @@ -484,18 +484,6 @@ class DNSReconApp { console.log('- Nodes:', graphData.nodes ? graphData.nodes.length : 0); console.log('- Edges:', graphData.edges ? graphData.edges.length : 0); - /*if (graphData.nodes) { - graphData.nodes.forEach(node => { - console.log(` Node: ${node.id} (${node.type})`); - }); - } - - if (graphData.edges) { - graphData.edges.forEach(edge => { - console.log(` Edge: ${edge.from} -> ${edge.to} (${edge.label})`); - }); - }*/ - // Only update if data has changed if (this.hasGraphChanged(graphData)) { console.log('*** GRAPH DATA CHANGED - UPDATING VISUALIZATION ***'); @@ -809,21 +797,8 @@ class DNSReconApp { } /** - * UPDATED: Helper method to find an attribute by name in the standardized attributes list - * @param {Array} attributes - List of StandardAttribute objects - * @param {string} name - Attribute name to find - * @returns {Object|null} The attribute object if found, null otherwise - */ - findAttributeByName(attributes, name) { - if (!Array.isArray(attributes)) { - return null; - } - return attributes.find(attr => attr.name === name) || null; - } - - /** - * Node details HTML generation for unified data model - * processes StandardAttribute objects + * UPDATED: Enhanced node details HTML generation for unified data model + * Now properly groups attributes by provider/type with organized sections */ generateNodeDetailsHtml(node) { if (!node) return '
Details not available.
'; @@ -857,7 +832,7 @@ class DNSReconApp {
`; - // Handle different node types with collapsible sections + // Handle different node types if (node.type === 'correlation_object') { detailsHtml += this.generateCorrelationDetails(node); } else if (node.type === 'large_entity') { @@ -871,7 +846,7 @@ class DNSReconApp { } /** - * Generate details for standard nodes using unified data model + * UPDATED: Generate details for standard nodes with organized attribute grouping */ generateStandardNodeDetails(node) { let html = ''; @@ -879,9 +854,9 @@ class DNSReconApp { // Relationships sections html += this.generateRelationshipsSection(node); - // Attributes section with grouping + // UPDATED: Enhanced attributes section with intelligent grouping (no formatting) if (node.attributes && Array.isArray(node.attributes) && node.attributes.length > 0) { - html += this.generateEnhancedAttributesSection(node.attributes, node.type); + html += this.generateOrganizedAttributesSection(node.attributes, node.type); } // Description section @@ -893,35 +868,58 @@ class DNSReconApp { return html; } - generateEnhancedAttributesSection(attributes, nodeType) { + /** + * NEW: Organized attributes section with provider/semantic grouping (no formatting) + */ + generateOrganizedAttributesSection(attributes, nodeType) { if (!Array.isArray(attributes) || attributes.length === 0) { return ''; } - // Group attributes by provider and type for better organization - const groupedAttributes = this.groupAttributesIntelligently(attributes, nodeType); + // Group attributes intelligently + const groups = this.groupAttributesByProviderAndType(attributes, nodeType); let html = ''; - for (const [groupName, groupData] of Object.entries(groupedAttributes)) { - const isDefaultOpen = groupData.priority === 'high'; + // Sort groups by priority + const sortedGroups = Object.entries(groups).sort((a, b) => { + const priorityOrder = { 'high': 0, 'medium': 1, 'low': 2 }; + return priorityOrder[a[1].priority] - priorityOrder[b[1].priority]; + }); + + for (const [groupName, groupData] of sortedGroups) { + if (groupData.attributes.length === 0) continue; + + const isOpen = groupData.priority === 'high'; html += `
-
+
${groupData.icon} ${groupName} - ${groupData.attributes.length} + ${groupData.attributes.length}
`; groupData.attributes.forEach(attr => { + // Format the value appropriately + let displayValue = ''; + if (attr.value === null || attr.value === undefined) { + displayValue = 'N/A'; + } else if (Array.isArray(attr.value)) { + displayValue = attr.value.length > 0 ? `Array (${attr.value.length} items)` : 'Empty Array'; + } else if (typeof attr.value === 'object') { + displayValue = 'Object'; + } else { + displayValue = String(attr.value); + } + html += `
- ${this.formatAttributeLabel(attr.name)} - ${this.formatAttributeValueEnhanced(attr)} + ${this.escapeHtml(attr.name || 'Unknown')} + ${this.escapeHtml(displayValue)}
`; }); @@ -932,192 +930,50 @@ class DNSReconApp { return html; } - formatAttributeValueEnhanced(attr) { - const value = attr.value; - - if (value === null || value === undefined) { - return 'None'; - } - - if (Array.isArray(value)) { - if (value.length === 0) return 'None'; - if (value.length === 1) return this.escapeHtml(String(value[0])); - - // Complex array - make it collapsible - const previewItems = value.slice(0, 2); - const hasMore = value.length > 2; - - let html = '
'; - html += `
`; - - previewItems.forEach(item => { - html += `
${this.escapeHtml(String(item))}
`; - }); - - if (hasMore) { - html += ` - - `; - } - - html += '
'; - - if (hasMore) { - html += `'; - } - - html += '
'; - return html; - } - - if (typeof value === 'object' && value !== null) { - return this.formatObjectExpandable(value); - } - - return this.escapeHtml(String(value)); - } - /** - * NEW: Format objects as expandable content + * NEW: Group attributes by provider and semantic meaning (no formatting) */ - formatObjectExpandable(obj) { - if (!obj || typeof obj !== 'object') return ''; - - const entries = Object.entries(obj); - if (entries.length === 0) return 'Empty'; - - if (entries.length <= 3) { - // Simple inline display for small objects - let html = '
'; - entries.forEach(([key, value]) => { - html += `
${key}: ${this.escapeHtml(String(value))}
`; - }); - html += '
'; - return html; - } - - // Expandable display for complex objects - let html = '
'; - html += ` -
- ${entries[0][0]}: ${this.escapeHtml(String(entries[0][1]))} - -
- -
- `; - - return html; - } - - formatAttributeLabel(name) { - // Handle provider prefixed attributes - if (name.includes('_')) { - const parts = name.split('_'); - if (parts.length >= 2) { - const provider = parts[0]; - const attribute = parts.slice(1).join('_'); - return `${this.provider}: ${this.formatLabel(attribute)}`; - } - } - - return this.formatLabel(name); - } - - groupAttributesIntelligently(attributes, nodeType) { - const groups = {}; - - // Define group configurations - const groupConfigs = { - 'DNS Records': { - icon: '🔍', - priority: 'high', - keywords: ['dns', 'record', 'a_record', 'cname', 'mx', 'ns', 'txt', 'ptr'], - providers: ['dns'] - }, - 'Certificate Information': { - icon: '🔒', - priority: 'high', - keywords: ['cert', 'certificate', 'ssl', 'tls', 'issuer', 'validity', 'san'], - providers: ['crtsh'] - }, - 'Network Information': { - icon: '🌐', - priority: 'high', - keywords: ['port', 'service', 'banner', 'asn', 'organization', 'country', 'city'], - providers: ['shodan'] - }, - 'Correlation Data': { - icon: '🔗', - priority: 'medium', - keywords: ['correlation', 'shared', 'common'], - providers: [] - } + groupAttributesByProviderAndType(attributes, nodeType) { + const groups = { + 'DNS Records': { icon: '🔍', priority: 'high', attributes: [] }, + 'Certificate Information': { icon: '🔒', priority: 'high', attributes: [] }, + 'Network Information': { icon: '🌐', priority: 'high', attributes: [] }, + 'Provider Data': { icon: '📊', priority: 'medium', attributes: [] }, + 'Technical Details': { icon: '⚙️', priority: 'low', attributes: [] } }; - // Initialize groups - Object.entries(groupConfigs).forEach(([name, config]) => { - groups[name] = { - ...config, - attributes: [] - }; - }); + for (const attr of attributes) { + const provider = attr.provider?.toLowerCase() || ''; + const name = attr.name?.toLowerCase() || ''; - // Add a catch-all group - groups['Other Information'] = { - icon: '📋', - priority: 'low', - attributes: [] - }; - - // Classify attributes into groups - attributes.forEach(attr => { let assigned = false; - - // Try to assign to a specific group based on provider or keywords - for (const [groupName, config] of Object.entries(groupConfigs)) { - const matchesProvider = config.providers.includes(attr.provider); - const matchesKeyword = config.keywords.some(keyword => - attr.name.toLowerCase().includes(keyword) || - attr.type.toLowerCase().includes(keyword) - ); - - if (matchesProvider || matchesKeyword) { - groups[groupName].attributes.push(attr); - assigned = true; - break; - } + + // DNS-related attributes + if (provider === 'dns' || ['dns', 'record', 'ptr', 'mx', 'cname', 'ns', 'txt', 'soa'].some(keyword => name.includes(keyword))) { + groups['DNS Records'].attributes.push(attr); + assigned = true; + } + // Certificate-related attributes + else if (provider === 'crtsh' || ['cert', 'certificate', 'ssl', 'tls', 'issuer', 'validity', 'san'].some(keyword => name.includes(keyword))) { + groups['Certificate Information'].attributes.push(attr); + assigned = true; + } + // Network/Shodan attributes + else if (provider === 'shodan' || ['port', 'service', 'banner', 'asn', 'organization', 'country', 'city', 'network'].some(keyword => name.includes(keyword))) { + groups['Network Information'].attributes.push(attr); + assigned = true; + } + // Provider-specific data + else if (provider && ['shodan_', 'crtsh_', 'dns_'].some(prefix => name.startsWith(prefix))) { + groups['Provider Data'].attributes.push(attr); + assigned = true; } - // If not assigned to any specific group, put in "Other" + // If not assigned to any specific group, put in technical details if (!assigned) { - groups['Other Information'].attributes.push(attr); + groups['Technical Details'].attributes.push(attr); } - }); + } // Remove empty groups Object.keys(groups).forEach(groupName => { @@ -1130,119 +986,47 @@ class DNSReconApp { } /** - * UPDATED: Generate large entity details using unified data model + * UPDATED: Enhanced correlation details showing the correlated attribute clearly (no formatting) */ - generateLargeEntityDetails(node) { - // UPDATED: Look for attributes in the unified model structure - const nodesAttribute = this.findAttributeByName(node.attributes, 'nodes'); - const countAttribute = this.findAttributeByName(node.attributes, 'count'); - const nodeTypeAttribute = this.findAttributeByName(node.attributes, 'node_type'); - const sourceProviderAttribute = this.findAttributeByName(node.attributes, 'source_provider'); - const discoveryDepthAttribute = this.findAttributeByName(node.attributes, 'discovery_depth'); + generateCorrelationDetails(node) { + const metadata = node.metadata || {}; + const value = metadata.value; + const correlatedNodes = metadata.correlated_nodes || []; + const sources = metadata.sources || []; - const nodes = nodesAttribute ? nodesAttribute.value : []; - const count = countAttribute ? countAttribute.value : 0; - const nodeType = nodeTypeAttribute ? nodeTypeAttribute.value : 'nodes'; - const sourceProvider = sourceProviderAttribute ? sourceProviderAttribute.value : 'Unknown'; - const discoveryDepth = discoveryDepthAttribute ? discoveryDepthAttribute.value : 'Unknown'; + let html = ''; - let html = ` + // Show what attribute is being correlated + const primarySource = metadata.primary_source || 'unknown'; + + html += `
- 📦 Entity Summary + + 🔗 Correlation: ${primarySource} + ${correlatedNodes.length} +
- Contains: - ${count} ${nodeType}s + Shared Value + ${this.escapeHtml(String(value))}
- Provider: - ${sourceProvider} + Attribute Type + ${primarySource}
- Depth: - ${discoveryDepth} + Correlated Nodes + ${correlatedNodes.length} nodes
- -
-
- 📋 Contained ${nodeType}s (${nodes.length}) -
-
`; - // Use node.id for the large_entity_id - const largeEntityId = node.id; - - if (Array.isArray(nodes)) { - nodes.forEach(innerNodeId => { - html += ` -
- ${innerNodeId} - -
- `; - }); - } - - html += '
'; - - return html; - } - - generateCorrelationDetails(node) { - const metadata = node.metadata || {}; - const values = metadata.values || []; - const sources = metadata.sources || []; - const mergeCount = metadata.merge_count || 1; - - let html = ''; - - // Correlation values section with meaningful labels - reuses existing modal structure - html += ` -
-
- - 🔗 Correlation Values - ${mergeCount} value${mergeCount > 1 ? 's' : ''} - -
-
- `; - - // Create a map of values to their source attributes for better labeling - const valueSourceMap = this.createValueSourceMap(values, sources); - - values.forEach((value, index) => { - const sourceInfo = valueSourceMap[index] || {}; - const attributeName = sourceInfo.meaningfulName || `Value ${index + 1}`; - const sourceDetails = sourceInfo.details || ''; - - html += ` -
- - ${this.escapeHtml(attributeName)} - ${sourceDetails ? ` ℹ️` : ''} - - - ${this.escapeHtml(String(value))} - -
- `; - }); - - html += '
'; - - // Correlated nodes section - reuses existing relationship display - const correlatedNodes = metadata.correlated_nodes || []; + // Show the correlated nodes if (correlatedNodes.length > 0) { html += `
@@ -1266,42 +1050,77 @@ class DNSReconApp { return html; } + + + + /** - * Create a mapping of values to their source attribute information + * UPDATED: Generate large entity details using unified data model */ - createValueSourceMap(values, sources) { - const valueSourceMap = {}; + generateLargeEntityDetails(node) { + // Look for attributes in the unified model structure + const attributes = node.attributes || []; + const nodesAttribute = attributes.find(attr => attr.name === 'nodes'); + const countAttribute = attributes.find(attr => attr.name === 'count'); + const nodeTypeAttribute = attributes.find(attr => attr.name === 'node_type'); + const sourceProviderAttribute = attributes.find(attr => attr.name === 'source_provider'); + const discoveryDepthAttribute = attributes.find(attr => attr.name === 'discovery_depth'); - // Group sources by their meaningful attributes - const attrGroups = {}; - sources.forEach(source => { - const meaningfulAttr = source.meaningful_attr || source.parent_attr || 'correlation'; + const nodes = nodesAttribute ? nodesAttribute.value : []; + const count = countAttribute ? countAttribute.value : 0; + const nodeType = nodeTypeAttribute ? nodeTypeAttribute.value : 'nodes'; + const sourceProvider = sourceProviderAttribute ? sourceProviderAttribute.value : 'Unknown'; + const discoveryDepth = discoveryDepthAttribute ? discoveryDepthAttribute.value : 'Unknown'; + + let html = ` +
+
+ 📦 Entity Summary +
+
+
+ Contains + ${count} ${nodeType}s +
+
+ Provider + ${sourceProvider} +
+
+ Depth + ${discoveryDepth} +
+
+
+
+
- if (!attrGroups[meaningfulAttr]) { - attrGroups[meaningfulAttr] = { - nodeIds: [], - paths: [] - }; - } - attrGroups[meaningfulAttr].nodeIds.push(source.node_id); - attrGroups[meaningfulAttr].paths.push(source.path || ''); - }); +
+
+ 📋 Contained ${nodeType}s (${Array.isArray(nodes) ? nodes.length : 0}) +
+
+ `; - // Map values to their best attribute names - values.forEach((value, index) => { - // Find the most meaningful attribute name - const attrNames = Object.keys(attrGroups); - const bestAttr = attrNames.find(attr => attr !== 'correlation' && attr !== 'unknown') || attrNames[0] || 'correlation'; - - if (attrGroups[bestAttr]) { - valueSourceMap[index] = { - meaningfulName: bestAttr, - details: `Found in: ${[...new Set(attrGroups[bestAttr].nodeIds)].join(', ')}` - }; - } - }); + const largeEntityId = node.id; + + if (Array.isArray(nodes)) { + nodes.forEach(innerNodeId => { + html += ` +
+ ${innerNodeId} + +
+ `; + }); + } - return valueSourceMap; + html += '
'; + + return html; } generateRelationshipsSection(node) { @@ -1372,85 +1191,30 @@ class DNSReconApp { return html; } - /** - * UPDATED: Generate attributes section for the new flat data model - */ - generateAttributesSection(attributes) { - if (!Array.isArray(attributes) || attributes.length === 0) { - return ''; - } - - let html = ` -
-
- 📊 Attributes (${attributes.length}) -
-
- `; - - attributes.forEach(attr => { - html += ` -
- ${this.formatLabel(attr.name)} - ${this.formatStandardAttributeValue(attr)} -
- `; - }); - - html += '
'; - return html; - } - - /** - * UPDATED: Format StandardAttribute value for display - */ - formatStandardAttributeValue(attr) { - const value = attr.value; - - if (value === null || value === undefined) { - return 'None'; - } - - if (Array.isArray(value)) { - if (value.length === 0) return 'None'; - if (value.length === 1) return this.escapeHtml(String(value[0])); - - let html = '
'; - value.forEach((item, index) => { - html += `
${this.escapeHtml(String(item))}
`; - }); - html += '
'; - return html; - } - - if (typeof value === 'object' && value !== null) { - return `
${this.formatObjectCompact(value)}
`; - } - - return this.escapeHtml(String(value)); - } - formatObjectCompact(obj) { if (!obj || typeof obj !== 'object') return ''; - let html = ''; const entries = Object.entries(obj); + if (entries.length <= 2) { + let html = ''; + entries.forEach(([key, value]) => { + html += `
${key}: ${this.escapeHtml(String(value))}
`; + }); + return html; + } - entries.forEach(([key, value]) => { - html += `
${key}: `; - if (typeof value === 'object' && value !== null) { - if (Array.isArray(value)) { - html += `[${value.length} items]`; - } else { - html += `{${Object.keys(value).length} properties}`; - } - } else { - html += this.escapeHtml(String(value)); - } - html += '
'; - }); - - return html; + // For complex objects, show first entry with expansion + return ` +
${entries[0][0]}: ${this.escapeHtml(String(entries[0][1]))}
+
+ +${entries.length - 1} more properties... +
+ ${entries.slice(1).map(([key, value]) => + `
${key}: ${this.escapeHtml(String(value))}
` + ).join('')} +
+
+ `; } generateDescriptionSection(node) { @@ -1660,8 +1424,8 @@ class DNSReconApp { */ getNodeTypeIcon(nodeType) { const icons = { - 'domain': '🌐', - 'ip': '🔍', + 'domain': '🌍', + 'ip': '📍', 'asn': '🏢', 'large_entity': '📦', 'correlation_object': '🔗' @@ -1710,28 +1474,6 @@ class DNSReconApp { } } - /** - * Toggle all entity nodes in large entity view - */ - toggleAllEntities() { - const entityCards = this.elements.modalDetails.querySelectorAll('.entity-node-card'); - const allExpanded = Array.from(entityCards).every(card => card.classList.contains('expanded')); - - entityCards.forEach(card => { - if (allExpanded) { - card.classList.remove('expanded'); - } else { - card.classList.add('expanded'); - } - }); - - // Update button text - const toggleBtn = this.elements.modalDetails.querySelector('.toggle-all-btn'); - if (toggleBtn) { - toggleBtn.textContent = allExpanded ? 'Expand All' : 'Collapse All'; - } - } - /** * Enhanced keyboard navigation for modals */ From f2db739fa10808844e151538e10b09bc1136fe54 Mon Sep 17 00:00:00 2001 From: overcuriousity Date: Wed, 17 Sep 2025 00:05:48 +0200 Subject: [PATCH 05/20] attempt to fix some logic --- core/graph_manager.py | 109 ++++++++++++-------------------- providers/crtsh_provider.py | 7 ++- providers/dns_provider.py | 22 +++---- providers/shodan_provider.py | 12 +--- static/js/graph.js | 92 ++++++++++++++++++++++----- static/js/main.js | 118 ++++++++++++++++++++++++----------- 6 files changed, 218 insertions(+), 142 deletions(-) diff --git a/core/graph_manager.py b/core/graph_manager.py index edd31c3..5054505 100644 --- a/core/graph_manager.py +++ b/core/graph_manager.py @@ -41,7 +41,7 @@ class GraphManager: self.correlation_index = {} # Compile regex for date filtering for efficiency self.date_pattern = re.compile(r'^\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}') - self.EXCLUDED_KEYS = ['confidence', 'provider', 'timestamp', 'type','crtsh_cert_validity_period_days'] + self.EXCLUDED_KEYS = ['confidence', 'provider', 'timestamp', 'type','crtsh_cert_validity_period_days','crtsh_cert_source'] def __getstate__(self): """Prepare GraphManager for pickling, excluding compiled regex.""" @@ -112,7 +112,7 @@ class GraphManager: def _create_enhanced_correlation_node_and_edges(self, value, correlation_data): """ - UPDATED: Create correlation node and edges with detailed provider tracking. + UPDATED: Create correlation node and edges with raw provider data (no formatting). """ correlation_node_id = f"corr_{hash(str(value)) & 0x7FFFFFFF}" nodes = correlation_data['nodes'] @@ -120,13 +120,14 @@ class GraphManager: # Create or update correlation node if not self.graph.has_node(correlation_node_id): - # Determine the most common provider/attribute combination + # Use raw provider/attribute data - no formatting provider_counts = {} for source in sources: + # Keep original provider and attribute names key = f"{source['provider']}_{source['attribute']}" provider_counts[key] = provider_counts.get(key, 0) + 1 - # Use the most common provider/attribute as the primary label + # Use the most common provider/attribute as the primary label (raw) primary_source = max(provider_counts.items(), key=lambda x: x[1])[0] if provider_counts else "unknown_correlation" metadata = { @@ -303,18 +304,18 @@ class GraphManager: return is_new_node def add_edge(self, source_id: str, target_id: str, relationship_type: str, - confidence_score: float = 0.5, source_provider: str = "unknown", - raw_data: Optional[Dict[str, Any]] = None) -> bool: - """Add or update an edge between two nodes, ensuring nodes exist.""" + confidence_score: float = 0.5, source_provider: str = "unknown", + raw_data: Optional[Dict[str, Any]] = None) -> bool: + """ + UPDATED: Add or update an edge between two nodes with raw relationship labels. + """ if not self.graph.has_node(source_id) or not self.graph.has_node(target_id): return False new_confidence = confidence_score - if relationship_type.startswith("c_"): - edge_label = relationship_type - else: - edge_label = f"{source_provider}_{relationship_type}" + # UPDATED: Use raw relationship type - no formatting + edge_label = relationship_type if self.graph.has_edge(source_id, target_id): # If edge exists, update confidence if the new score is higher. @@ -324,7 +325,7 @@ class GraphManager: self.graph.edges[source_id, target_id]['updated_by'] = source_provider return False - # Add a new edge with all attributes. + # Add a new edge with raw attributes self.graph.add_edge(source_id, target_id, relationship_type=edge_label, confidence_score=new_confidence, @@ -333,7 +334,7 @@ class GraphManager: raw_data=raw_data or {}) self.last_modified = datetime.now(timezone.utc).isoformat() return True - + def extract_node_from_large_entity(self, large_entity_id: str, node_id_to_extract: str) -> bool: """ Removes a node from a large entity's internal lists and updates its count. @@ -417,73 +418,45 @@ class GraphManager: def get_graph_data(self) -> Dict[str, Any]: """ Export graph data formatted for frontend visualization. - UPDATED: Fixed certificate validity styling logic for unified data model. + SIMPLIFIED: No certificate styling - frontend handles all visual styling. """ nodes = [] for node_id, attrs in self.graph.nodes(data=True): - node_data = {'id': node_id, 'label': node_id, 'type': attrs.get('type', 'unknown'), - 'attributes': attrs.get('attributes', []), # Ensure attributes is a list - 'description': attrs.get('description', ''), - 'metadata': attrs.get('metadata', {}), - 'added_timestamp': attrs.get('added_timestamp')} - - # UPDATED: Fixed certificate validity styling logic - node_type = node_data['type'] - attributes_list = node_data['attributes'] - - if node_type == 'domain' and isinstance(attributes_list, list): - # Check for certificate-related attributes - has_certificates = False - has_valid_certificates = False - has_expired_certificates = False - - for attr in attributes_list: - attr_name = attr.get('name', '').lower() - attr_provider = attr.get('provider', '').lower() - attr_value = attr.get('value') - - # Look for certificate attributes from crt.sh provider - if attr_provider == 'crtsh' or 'cert' in attr_name: - has_certificates = True - - # Check certificate validity - if attr_name == 'cert_is_currently_valid': - if attr_value is True: - has_valid_certificates = True - elif attr_value is False: - has_expired_certificates = True - - # Also check for certificate expiry indicators - elif 'expires_soon' in attr_name and attr_value is True: - has_expired_certificates = True - elif 'expired' in attr_name and attr_value is True: - has_expired_certificates = True - - # Apply styling based on certificate status - if has_expired_certificates and not has_valid_certificates: - # Red for expired/invalid certificates - node_data['color'] = {'background': '#ff6b6b', 'border': '#cc5555'} - elif not has_certificates: - # Grey for domains with no certificates - node_data['color'] = {'background': '#c7c7c7', 'border': '#999999'} - # Default green styling is handled by the frontend for domains with valid certificates + node_data = { + 'id': node_id, + 'label': node_id, + 'type': attrs.get('type', 'unknown'), + 'attributes': attrs.get('attributes', []), # Raw attributes list + 'description': attrs.get('description', ''), + 'metadata': attrs.get('metadata', {}), + 'added_timestamp': attrs.get('added_timestamp') + } # Add incoming and outgoing edges to node data if self.graph.has_node(node_id): - node_data['incoming_edges'] = [{'from': u, 'data': d} for u, _, d in self.graph.in_edges(node_id, data=True)] - node_data['outgoing_edges'] = [{'to': v, 'data': d} for _, v, d in self.graph.out_edges(node_id, data=True)] + node_data['incoming_edges'] = [ + {'from': u, 'data': d} for u, _, d in self.graph.in_edges(node_id, data=True) + ] + node_data['outgoing_edges'] = [ + {'to': v, 'data': d} for _, v, d in self.graph.out_edges(node_id, data=True) + ] nodes.append(node_data) edges = [] for source, target, attrs in self.graph.edges(data=True): - edges.append({'from': source, 'to': target, - 'label': attrs.get('relationship_type', ''), - 'confidence_score': attrs.get('confidence_score', 0), - 'source_provider': attrs.get('source_provider', ''), - 'discovery_timestamp': attrs.get('discovery_timestamp')}) + edges.append({ + 'from': source, + 'to': target, + 'label': attrs.get('relationship_type', ''), + 'confidence_score': attrs.get('confidence_score', 0), + 'source_provider': attrs.get('source_provider', ''), + 'discovery_timestamp': attrs.get('discovery_timestamp') + }) + return { - 'nodes': nodes, 'edges': edges, + 'nodes': nodes, + 'edges': edges, 'statistics': self.get_statistics()['basic_metrics'] } diff --git a/providers/crtsh_provider.py b/providers/crtsh_provider.py index d8bdfba..6adde15 100644 --- a/providers/crtsh_provider.py +++ b/providers/crtsh_provider.py @@ -355,7 +355,7 @@ class CrtShProvider(BaseProvider): 'not_before': cert_data.get('not_before'), 'not_after': cert_data.get('not_after'), 'entry_timestamp': cert_data.get('entry_timestamp'), - 'source': 'crt.sh' + 'source': 'crtsh' } try: @@ -367,8 +367,9 @@ class CrtShProvider(BaseProvider): metadata['is_currently_valid'] = self._is_cert_valid(cert_data) metadata['expires_soon'] = (not_after - datetime.now(timezone.utc)).days <= 30 - metadata['not_before'] = not_before.strftime('%Y-%m-%d %H:%M:%S UTC') - metadata['not_after'] = not_after.strftime('%Y-%m-%d %H:%M:%S UTC') + # UPDATED: Keep raw date format or convert to standard format + metadata['not_before'] = not_before.isoformat() + metadata['not_after'] = not_after.isoformat() except Exception as e: self.logger.logger.debug(f"Error computing certificate metadata: {e}") diff --git a/providers/dns_provider.py b/providers/dns_provider.py index 5d972d1..7abaf58 100644 --- a/providers/dns_provider.py +++ b/providers/dns_provider.py @@ -155,12 +155,7 @@ class DNSProvider(BaseProvider): def _query_record(self, domain: str, record_type: str, result: ProviderResult) -> None: """ - Query a specific type of DNS record for the domain and add results to ProviderResult. - - Args: - domain: Domain to query - record_type: DNS record type (A, AAAA, CNAME, etc.) - result: ProviderResult to populate + UPDATED: Query DNS records with minimal formatting - keep raw values. """ try: self.total_requests += 1 @@ -180,13 +175,14 @@ class DNSProvider(BaseProvider): elif record_type == 'SOA': target = str(record.mname).rstrip('.') elif record_type in ['TXT']: - # TXT records are treated as attributes, not relationships + # UPDATED: Keep raw TXT record value txt_value = str(record).strip('"') dns_records.append(f"TXT: {txt_value}") continue elif record_type == 'SRV': target = str(record.target).rstrip('.') elif record_type == 'CAA': + # UPDATED: Keep raw CAA record format caa_value = f"{record.flags} {record.tag.decode('utf-8')} \"{record.value.decode('utf-8')}\"" dns_records.append(f"CAA: {caa_value}") continue @@ -200,8 +196,8 @@ class DNSProvider(BaseProvider): 'value': target, 'ttl': response.ttl } - relationship_type = f"{record_type.lower()}_record" - confidence = 0.8 # Standard confidence for DNS records + relationship_type = f"{record_type.lower()}_record" # Raw relationship type + confidence = 0.8 # Add relationship result.add_relationship( @@ -213,7 +209,7 @@ class DNSProvider(BaseProvider): raw_data=raw_data ) - # Add DNS record as attribute to the source domain + # UPDATED: Keep raw DNS record format dns_records.append(f"{record_type}: {target}") # Log relationship discovery @@ -226,7 +222,7 @@ class DNSProvider(BaseProvider): discovery_method=f"dns_{record_type.lower()}_record" ) - # Add DNS records as a consolidated attribute + # Add DNS records as a consolidated attribute (raw format) if dns_records: result.add_attribute( target_node=domain, @@ -241,5 +237,5 @@ class DNSProvider(BaseProvider): except Exception as e: self.failed_requests += 1 self.logger.logger.debug(f"{record_type} record query failed for {domain}: {e}") - # Re-raise the exception so the scanner can handle it - raise e \ No newline at end of file + raise e + diff --git a/providers/shodan_provider.py b/providers/shodan_provider.py index 6f3a044..00d0cea 100644 --- a/providers/shodan_provider.py +++ b/providers/shodan_provider.py @@ -211,14 +211,7 @@ class ShodanProvider(BaseProvider): def _process_shodan_data(self, ip: str, data: Dict[str, Any]) -> ProviderResult: """ - Process Shodan data to extract relationships and attributes. - - Args: - ip: IP address queried - data: Raw Shodan response data - - Returns: - ProviderResult with relationships and attributes + UPDATED: Process Shodan data with raw attribute names and values. """ result = ProviderResult() @@ -271,9 +264,10 @@ class ShodanProvider(BaseProvider): confidence=0.9 ) elif isinstance(value, (str, int, float, bool)) and value is not None: + # UPDATED: Keep raw Shodan field names (no "shodan_" prefix) result.add_attribute( target_node=ip, - name=f"shodan_{key}", + name=key, # Raw field name from Shodan API value=value, attr_type='shodan_info', provider=self.name, diff --git a/static/js/graph.js b/static/js/graph.js index dd5bc13..9391150 100644 --- a/static/js/graph.js +++ b/static/js/graph.js @@ -382,7 +382,6 @@ class GraphManager { graphData.nodes.forEach(node => { if (node.type === 'large_entity' && node.attributes) { - // UPDATED: Handle unified data model - look for 'nodes' attribute in the attributes list const nodesAttribute = this.findAttributeByName(node.attributes, 'nodes'); if (nodesAttribute && Array.isArray(nodesAttribute.value)) { nodesAttribute.value.forEach(nodeId => { @@ -394,15 +393,30 @@ class GraphManager { }); const filteredNodes = graphData.nodes.filter(node => { - // Only include nodes that are NOT members of large entities, but always include the container itself return !this.largeEntityMembers.has(node.id) || node.type === 'large_entity'; }); console.log(`Filtered ${graphData.nodes.length - filteredNodes.length} large entity member nodes from visualization`); - // Process only the filtered nodes + // FIXED: Process nodes with proper certificate coloring const processedNodes = filteredNodes.map(node => { - return this.processNode(node); + const processed = this.processNode(node); + + // FIXED: Apply certificate-based coloring here in frontend + if (node.type === 'domain' && Array.isArray(node.attributes)) { + const certInfo = this.analyzeCertificateInfo(node.attributes); + + if (certInfo.hasExpiredOnly) { + // Red for domains with only expired/invalid certificates + processed.color = { background: '#ff6b6b', border: '#cc5555' }; + } else if (!certInfo.hasCertificates) { + // Grey for domains with no certificates + processed.color = { background: '#c7c7c7', border: '#999999' }; + } + // Valid certificates use default green (handled by processNode) + } + + return processed; }); const mergedEdges = {}; @@ -439,24 +453,19 @@ class GraphManager { const existingNodeIds = this.nodes.getIds(); const existingEdgeIds = this.edges.getIds(); - // Add new nodes with fade-in animation const newNodes = processedNodes.filter(node => !existingNodeIds.includes(node.id)); const newEdges = processedEdges.filter(edge => !existingEdgeIds.includes(edge.id)); - // Update existing data this.nodes.update(processedNodes); this.edges.update(processedEdges); - // After data is loaded, apply filters this.updateFilterControls(); this.applyAllFilters(); - // Highlight new additions briefly if (newNodes.length > 0 || newEdges.length > 0) { setTimeout(() => this.highlightNewElements(newNodes, newEdges), 100); } - // Auto-fit view for small graphs or first update if (processedNodes.length <= 10 || existingNodeIds.length === 0) { setTimeout(() => this.fitView(), 800); } @@ -470,6 +479,46 @@ class GraphManager { } } + analyzeCertificateInfo(attributes) { + let hasCertificates = false; + let hasValidCertificates = false; + let hasExpiredCertificates = false; + + for (const attr of attributes) { + const attrName = (attr.name || '').toLowerCase(); + const attrProvider = (attr.provider || '').toLowerCase(); + const attrValue = attr.value; + + // Look for certificate attributes from crtsh provider + if (attrProvider === 'crtsh' || attrName.startsWith('cert_')) { + hasCertificates = true; + + // Check certificate validity using raw attribute names + if (attrName === 'cert_is_currently_valid') { + if (attrValue === true) { + hasValidCertificates = true; + } else if (attrValue === false) { + hasExpiredCertificates = true; + } + } + // Check for expiry indicators + else if (attrName === 'cert_expires_soon' && attrValue === true) { + hasExpiredCertificates = true; + } + else if (attrName.includes('expired') && attrValue === true) { + hasExpiredCertificates = true; + } + } + } + + return { + hasCertificates, + hasValidCertificates, + hasExpiredCertificates, + hasExpiredOnly: hasExpiredCertificates && !hasValidCertificates + }; + } + /** * UPDATED: Helper method to find an attribute by name in the standardized attributes list * @param {Array} attributes - List of StandardAttribute objects @@ -496,7 +545,7 @@ class GraphManager { size: this.getNodeSize(node.type), borderColor: this.getNodeBorderColor(node.type), shape: this.getNodeShape(node.type), - attributes: node.attributes || [], // Keep as standardized attributes list + attributes: node.attributes || [], description: node.description || '', metadata: node.metadata || {}, type: node.type, @@ -509,19 +558,33 @@ class GraphManager { processedNode.borderWidth = Math.max(2, Math.floor(node.confidence * 5)); } - // Handle merged correlation objects (similar to large entities) + // FIXED: Certificate-based domain coloring + if (node.type === 'domain' && Array.isArray(node.attributes)) { + const certInfo = this.analyzeCertificateInfo(node.attributes); + + if (certInfo.hasExpiredOnly) { + // Red for domains with only expired/invalid certificates + processedNode.color = '#ff6b6b'; + processedNode.borderColor = '#cc5555'; + } else if (!certInfo.hasCertificates) { + // Grey for domains with no certificates + processedNode.color = '#c7c7c7'; + processedNode.borderColor = '#999999'; + } + // Green for valid certificates (default color) + } + + // Handle merged correlation objects if (node.type === 'correlation_object') { const metadata = node.metadata || {}; const values = metadata.values || []; const mergeCount = metadata.merge_count || 1; if (mergeCount > 1) { - // Display as merged correlation container processedNode.label = `Correlations (${mergeCount})`; processedNode.title = `Merged correlation container with ${mergeCount} values: ${values.slice(0, 3).join(', ')}${values.length > 3 ? '...' : ''}`; - processedNode.borderWidth = 3; // Thicker border for merged nodes + processedNode.borderWidth = 3; } else { - // Single correlation value const value = Array.isArray(values) && values.length > 0 ? values[0] : (metadata.value || 'Unknown'); const displayValue = typeof value === 'string' && value.length > 20 ? value.substring(0, 17) + '...' : value; processedNode.label = `${displayValue}`; @@ -532,6 +595,7 @@ class GraphManager { return processedNode; } + /** * Process edge data with styling and metadata * @param {Object} edge - Raw edge data diff --git a/static/js/main.js b/static/js/main.js index 27be70b..72f5fdf 100644 --- a/static/js/main.js +++ b/static/js/main.js @@ -868,20 +868,14 @@ class DNSReconApp { return html; } - /** - * NEW: Organized attributes section with provider/semantic grouping (no formatting) - */ generateOrganizedAttributesSection(attributes, nodeType) { if (!Array.isArray(attributes) || attributes.length === 0) { return ''; } - // Group attributes intelligently const groups = this.groupAttributesByProviderAndType(attributes, nodeType); - let html = ''; - // Sort groups by priority const sortedGroups = Object.entries(groups).sort((a, b) => { const priorityOrder = { 'high': 0, 'medium': 1, 'low': 2 }; return priorityOrder[a[1].priority] - priorityOrder[b[1].priority]; @@ -904,22 +898,10 @@ class DNSReconApp { `; groupData.attributes.forEach(attr => { - // Format the value appropriately - let displayValue = ''; - if (attr.value === null || attr.value === undefined) { - displayValue = 'N/A'; - } else if (Array.isArray(attr.value)) { - displayValue = attr.value.length > 0 ? `Array (${attr.value.length} items)` : 'Empty Array'; - } else if (typeof attr.value === 'object') { - displayValue = 'Object'; - } else { - displayValue = String(attr.value); - } - html += `
${this.escapeHtml(attr.name || 'Unknown')} - ${this.escapeHtml(displayValue)} + ${this.formatAttributeValue(attr)}
`; }); @@ -930,12 +912,49 @@ class DNSReconApp { return html; } - /** - * NEW: Group attributes by provider and semantic meaning (no formatting) - */ + formatAttributeValue(attr) { + const value = attr.value; + const name = attr.name || ''; + + if (value === null || value === undefined) { + return 'N/A'; + } + + if (Array.isArray(value)) { + if (value.length === 0) { + return 'Empty Array'; + } + + // Special handling for DNS records and similar arrays + if (name === 'dns_records' || name.includes('record') || name.includes('hostname')) { + // Show DNS records as a readable list + if (value.length <= 5) { + return this.escapeHtml(value.join(', ')); + } else { + const preview = value.slice(0, 3).join(', '); + return this.escapeHtml(`${preview} ... (+${value.length - 3} more)`); + } + } + + // For other arrays + if (value.length <= 3) { + return this.escapeHtml(value.join(', ')); + } else { + const preview = value.slice(0, 2).join(', '); + return this.escapeHtml(`${preview} ... (${value.length} total)`); + } + } + + if (typeof value === 'object') { + return 'Object'; + } + + return this.escapeHtml(String(value)); + } + groupAttributesByProviderAndType(attributes, nodeType) { const groups = { - 'DNS Records': { icon: '🔍', priority: 'high', attributes: [] }, + 'DNS Records': { icon: '📋', priority: 'high', attributes: [] }, 'Certificate Information': { icon: '🔒', priority: 'high', attributes: [] }, 'Network Information': { icon: '🌐', priority: 'high', attributes: [] }, 'Provider Data': { icon: '📊', priority: 'medium', attributes: [] }, @@ -943,23 +962,29 @@ class DNSReconApp { }; for (const attr of attributes) { - const provider = attr.provider?.toLowerCase() || ''; - const name = attr.name?.toLowerCase() || ''; + const provider = (attr.provider || '').toLowerCase(); + const name = (attr.name || '').toLowerCase(); + const type = (attr.type || '').toLowerCase(); let assigned = false; - // DNS-related attributes - if (provider === 'dns' || ['dns', 'record', 'ptr', 'mx', 'cname', 'ns', 'txt', 'soa'].some(keyword => name.includes(keyword))) { + // DNS-related attributes (better detection) + if (provider === 'dns' || + name === 'dns_records' || + name.includes('record') || + ['ptr', 'mx', 'cname', 'ns', 'txt', 'soa'].some(keyword => name.includes(keyword))) { groups['DNS Records'].attributes.push(attr); assigned = true; } // Certificate-related attributes - else if (provider === 'crtsh' || ['cert', 'certificate', 'ssl', 'tls', 'issuer', 'validity', 'san'].some(keyword => name.includes(keyword))) { + else if (provider === 'crtsh' || name.startsWith('cert_') || + ['certificate', 'ssl', 'tls', 'issuer', 'validity', 'san'].some(keyword => name.includes(keyword))) { groups['Certificate Information'].attributes.push(attr); assigned = true; } // Network/Shodan attributes - else if (provider === 'shodan' || ['port', 'service', 'banner', 'asn', 'organization', 'country', 'city', 'network'].some(keyword => name.includes(keyword))) { + else if (provider === 'shodan' || + ['port', 'service', 'banner', 'asn', 'organization', 'country', 'city', 'network'].some(keyword => name.includes(keyword))) { groups['Network Information'].attributes.push(attr); assigned = true; } @@ -985,6 +1010,33 @@ class DNSReconApp { return groups; } + formatEdgeLabel(relationshipType, confidence) { + if (!relationshipType) return ''; + + // UPDATED: No formatting of relationship type - use raw values + const confidenceText = confidence >= 0.8 ? '●' : confidence >= 0.6 ? '◐' : '○'; + return `${relationshipType} ${confidenceText}`; + } + + createEdgeTooltip(edge) { + let tooltip = `
`; + tooltip += `
${edge.label || 'Relationship'}
`; + tooltip += `
Confidence: ${(edge.confidence_score * 100).toFixed(1)}%
`; + + // UPDATED: Use raw provider name (no formatting) + if (edge.source_provider) { + tooltip += `
Provider: ${edge.source_provider}
`; + } + + if (edge.discovery_timestamp) { + const date = new Date(edge.discovery_timestamp); + tooltip += `
Discovered: ${date.toLocaleString()}
`; + } + + tooltip += `
`; + return tooltip; + } + /** * UPDATED: Enhanced correlation details showing the correlated attribute clearly (no formatting) */ @@ -996,7 +1048,7 @@ class DNSReconApp { let html = ''; - // Show what attribute is being correlated + // Show what attribute is being correlated (raw names) const primarySource = metadata.primary_source || 'unknown'; html += ` @@ -1013,7 +1065,7 @@ class DNSReconApp { ${this.escapeHtml(String(value))}
- Attribute Type + Attribute Source ${primarySource}
@@ -1050,10 +1102,6 @@ class DNSReconApp { return html; } - - - - /** * UPDATED: Generate large entity details using unified data model */ From b984189e0853b1c6032cb395d15e9d0bc5cde30d Mon Sep 17 00:00:00 2001 From: overcuriousity Date: Wed, 17 Sep 2025 00:31:12 +0200 Subject: [PATCH 06/20] scheduler fixes --- core/graph_manager.py | 4 ++-- core/scanner.py | 17 +++++++++++++++-- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/core/graph_manager.py b/core/graph_manager.py index 5054505..cd84cca 100644 --- a/core/graph_manager.py +++ b/core/graph_manager.py @@ -41,7 +41,7 @@ class GraphManager: self.correlation_index = {} # Compile regex for date filtering for efficiency self.date_pattern = re.compile(r'^\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}') - self.EXCLUDED_KEYS = ['confidence', 'provider', 'timestamp', 'type','crtsh_cert_validity_period_days','crtsh_cert_source'] + self.EXCLUDED_KEYS = ['confidence', 'provider', 'timestamp', 'type','cert_validity_period_days','cert_source'] def __getstate__(self): """Prepare GraphManager for pickling, excluding compiled regex.""" @@ -73,7 +73,7 @@ class GraphManager: attr_provider = attr.get('provider', 'unknown') # Skip excluded attributes and invalid values - if attr_name in self.EXCLUDED_KEYS or not isinstance(attr_value, (str, int, float, bool)) or attr_value is None: + if any(excluded_key in attr_name for excluded_key in self.EXCLUDED_KEYS) or not isinstance(attr_value, (str, int, float, bool)) or attr_value is None: continue if isinstance(attr_value, bool): diff --git a/core/scanner.py b/core/scanner.py index 6b234b6..a3bdb7b 100644 --- a/core/scanner.py +++ b/core/scanner.py @@ -373,6 +373,7 @@ class Scanner: task_tuple = (provider_name, target_item) if task_tuple in processed_tasks: + self.indicators_completed += 1 continue if depth > max_depth: @@ -661,7 +662,7 @@ class Scanner: target_node_type = NodeType.DOMAIN if node_type == 'domain' else NodeType.IP self.graph.add_node(target, target_node_type) - attributes = { + attributes_dict = { 'count': len(targets), 'nodes': targets, 'node_type': node_type, @@ -669,9 +670,21 @@ class Scanner: 'discovery_depth': current_depth, 'threshold_exceeded': self.config.large_entity_threshold, } + + attributes_list = [] + for key, value in attributes_dict.items(): + attributes_list.append({ + "name": key, + "value": value, + "type": "large_entity_info", + "provider": provider_name, + "confidence": 0.9, + "metadata": {} + }) + description = f'Large entity created due to {len(targets)} relationships from {provider_name}' - self.graph.add_node(entity_id, NodeType.LARGE_ENTITY, attributes=attributes, description=description) + self.graph.add_node(entity_id, NodeType.LARGE_ENTITY, attributes=attributes_list, description=description) # Create edge from source to large entity if provider_result.relationships: From f775c61731aae908c7253097cc3bb1361002f1d2 Mon Sep 17 00:00:00 2001 From: overcuriousity Date: Wed, 17 Sep 2025 11:08:50 +0200 Subject: [PATCH 07/20] iterating on fixes --- app.py | 86 ++++++++++++++++++++++++++++++++------- core/graph_manager.py | 59 +++++++++++++++++++-------- providers/dns_provider.py | 56 ++++++++++++++----------- static/js/graph.js | 19 ++++++++- static/js/main.js | 78 +++++++++++++++++++++++++---------- 5 files changed, 220 insertions(+), 78 deletions(-) diff --git a/app.py b/app.py index b2233ff..1955e36 100644 --- a/app.py +++ b/app.py @@ -35,6 +35,29 @@ def get_user_scanner(): existing_scanner = session_manager.get_session(current_flask_session_id) if existing_scanner: return current_flask_session_id, existing_scanner + else: + print(f"Session {current_flask_session_id} not found in Redis, checking for active sessions...") + + # This prevents creating duplicate sessions when Flask session is lost but Redis session exists + stats = session_manager.get_statistics() + if stats['running_scans'] > 0: + # Get all session keys and find running ones + try: + import redis + redis_client = redis.StrictRedis(db=0, decode_responses=False) + session_keys = redis_client.keys("dnsrecon:session:*") + + for session_key in session_keys: + session_id = session_key.decode('utf-8').split(':')[-1] + scanner = session_manager.get_session(session_id) + if scanner and scanner.status in ['running', 'completed']: + print(f"Reusing active session: {session_id}") + # Update Flask session to match + session['dnsrecon_session_id'] = session_id + session.permanent = True + return session_id, scanner + except Exception as e: + print(f"Error finding active session: {e}") # Create new session if none exists print("Creating new session as none was found...") @@ -44,10 +67,11 @@ def get_user_scanner(): if not new_scanner: raise Exception("Failed to create new scanner session") - # Store in Flask session + # Store in Flask session with explicit settings session['dnsrecon_session_id'] = new_session_id session.permanent = True + print(f"Created new session: {new_session_id}") return new_session_id, new_scanner @app.route('/') @@ -240,29 +264,56 @@ def get_scan_status(): @app.route('/api/graph', methods=['GET']) def get_graph_data(): - """Get current graph data with error handling.""" + """Get current graph data with error handling and proper empty graph structure.""" try: # Get user-specific scanner user_session_id, scanner = get_user_scanner() if not scanner: - # Return empty graph if no scanner + # FIXED: Return proper empty graph structure instead of None + empty_graph = { + 'nodes': [], + 'edges': [], + 'statistics': { + 'node_count': 0, + 'edge_count': 0, + 'creation_time': datetime.now(timezone.utc).isoformat(), + 'last_modified': datetime.now(timezone.utc).isoformat() + } + } return jsonify({ 'success': True, - 'graph': { - 'nodes': [], - 'edges': [], - 'statistics': { - 'node_count': 0, - 'edge_count': 0, - 'creation_time': datetime.now(timezone.utc).isoformat(), - 'last_modified': datetime.now(timezone.utc).isoformat() - } - }, + 'graph': empty_graph, 'user_session_id': user_session_id }) graph_data = scanner.get_graph_data() + + if not graph_data: + graph_data = { + 'nodes': [], + 'edges': [], + 'statistics': { + 'node_count': 0, + 'edge_count': 0, + 'creation_time': datetime.now(timezone.utc).isoformat(), + 'last_modified': datetime.now(timezone.utc).isoformat() + } + } + + # FIXED: Ensure required fields exist + if 'nodes' not in graph_data: + graph_data['nodes'] = [] + if 'edges' not in graph_data: + graph_data['edges'] = [] + if 'statistics' not in graph_data: + graph_data['statistics'] = { + 'node_count': len(graph_data['nodes']), + 'edge_count': len(graph_data['edges']), + 'creation_time': datetime.now(timezone.utc).isoformat(), + 'last_modified': datetime.now(timezone.utc).isoformat() + } + return jsonify({ 'success': True, 'graph': graph_data, @@ -272,13 +323,20 @@ def get_graph_data(): except Exception as e: print(f"ERROR: Exception in get_graph_data endpoint: {e}") traceback.print_exc() + + # FIXED: Return proper error structure with empty graph fallback return jsonify({ 'success': False, 'error': f'Internal server error: {str(e)}', 'fallback_graph': { 'nodes': [], 'edges': [], - 'statistics': {'node_count': 0, 'edge_count': 0} + 'statistics': { + 'node_count': 0, + 'edge_count': 0, + 'creation_time': datetime.now(timezone.utc).isoformat(), + 'last_modified': datetime.now(timezone.utc).isoformat() + } } }), 500 diff --git a/core/graph_manager.py b/core/graph_manager.py index cd84cca..f0629cb 100644 --- a/core/graph_manager.py +++ b/core/graph_manager.py @@ -477,8 +477,13 @@ class GraphManager: } def _get_confidence_distribution(self) -> Dict[str, int]: - """Get distribution of edge confidence scores.""" + """Get distribution of edge confidence scores with empty graph handling.""" distribution = {'high': 0, 'medium': 0, 'low': 0} + + # FIXED: Handle empty graph case + if self.get_edge_count() == 0: + return distribution + for _, _, data in self.graph.edges(data=True): confidence = data.get('confidence_score', 0) if confidence >= 0.8: @@ -490,22 +495,42 @@ class GraphManager: return distribution def get_statistics(self) -> Dict[str, Any]: - """Get comprehensive statistics about the graph.""" - stats = {'basic_metrics': {'total_nodes': self.get_node_count(), - 'total_edges': self.get_edge_count(), - 'creation_time': self.creation_time, - 'last_modified': self.last_modified}, - 'node_type_distribution': {}, 'relationship_type_distribution': {}, - 'confidence_distribution': self._get_confidence_distribution(), - 'provider_distribution': {}} - # Calculate distributions - for node_type in NodeType: - stats['node_type_distribution'][node_type.value] = self.get_nodes_by_type(node_type).__len__() - for _, _, data in self.graph.edges(data=True): - rel_type = data.get('relationship_type', 'unknown') - stats['relationship_type_distribution'][rel_type] = stats['relationship_type_distribution'].get(rel_type, 0) + 1 - provider = data.get('source_provider', 'unknown') - stats['provider_distribution'][provider] = stats['provider_distribution'].get(provider, 0) + 1 + """Get comprehensive statistics about the graph with proper empty graph handling.""" + + # FIXED: Handle empty graph case properly + node_count = self.get_node_count() + edge_count = self.get_edge_count() + + stats = { + 'basic_metrics': { + 'total_nodes': node_count, + 'total_edges': edge_count, + 'creation_time': self.creation_time, + 'last_modified': self.last_modified + }, + 'node_type_distribution': {}, + 'relationship_type_distribution': {}, + 'confidence_distribution': self._get_confidence_distribution(), + 'provider_distribution': {} + } + + # FIXED: Only calculate distributions if we have data + if node_count > 0: + # Calculate node type distributions + for node_type in NodeType: + count = len(self.get_nodes_by_type(node_type)) + if count > 0: # Only include types that exist + stats['node_type_distribution'][node_type.value] = count + + if edge_count > 0: + # Calculate edge distributions + for _, _, data in self.graph.edges(data=True): + rel_type = data.get('relationship_type', 'unknown') + stats['relationship_type_distribution'][rel_type] = stats['relationship_type_distribution'].get(rel_type, 0) + 1 + + provider = data.get('source_provider', 'unknown') + stats['provider_distribution'][provider] = stats['provider_distribution'].get(provider, 0) + 1 + return stats def clear(self) -> None: diff --git a/providers/dns_provider.py b/providers/dns_provider.py index 7abaf58..6a855d9 100644 --- a/providers/dns_provider.py +++ b/providers/dns_provider.py @@ -50,6 +50,7 @@ class DNSProvider(BaseProvider): def query_domain(self, domain: str) -> ProviderResult: """ Query DNS records for the domain to discover relationships and attributes. + FIXED: Now creates separate attributes for each DNS record type. Args: domain: Domain to investigate @@ -62,7 +63,7 @@ class DNSProvider(BaseProvider): result = ProviderResult() - # Query all record types + # Query all record types - each gets its own attribute for record_type in ['A', 'AAAA', 'CNAME', 'MX', 'NS', 'SOA', 'TXT', 'SRV', 'CAA']: try: self._query_record(domain, record_type, result) @@ -97,6 +98,7 @@ class DNSProvider(BaseProvider): response = self.resolver.resolve(reverse_name, 'PTR') self.successful_requests += 1 + ptr_records = [] for ptr_record in response: hostname = str(ptr_record).rstrip('.') @@ -116,16 +118,8 @@ class DNSProvider(BaseProvider): } ) - # Add PTR record as attribute to the IP - result.add_attribute( - target_node=ip, - name='ptr_record', - value=hostname, - attr_type='dns_record', - provider=self.name, - confidence=0.8, - metadata={'ttl': response.ttl} - ) + # Add to PTR records list + ptr_records.append(f"PTR: {hostname}") # Log the relationship discovery self.log_relationship_discovery( @@ -142,6 +136,18 @@ class DNSProvider(BaseProvider): discovery_method="reverse_dns_lookup" ) + # Add PTR records as separate attribute + if ptr_records: + result.add_attribute( + target_node=ip, + name='ptr_records', # Specific name for PTR records + value=ptr_records, + attr_type='dns_record', + provider=self.name, + confidence=0.8, + metadata={'ttl': response.ttl} + ) + except resolver.NXDOMAIN: self.failed_requests += 1 self.logger.logger.debug(f"Reverse DNS lookup failed for {ip}: NXDOMAIN") @@ -155,7 +161,7 @@ class DNSProvider(BaseProvider): def _query_record(self, domain: str, record_type: str, result: ProviderResult) -> None: """ - UPDATED: Query DNS records with minimal formatting - keep raw values. + FIXED: Query DNS records with unique attribute names for each record type. """ try: self.total_requests += 1 @@ -175,16 +181,16 @@ class DNSProvider(BaseProvider): elif record_type == 'SOA': target = str(record.mname).rstrip('.') elif record_type in ['TXT']: - # UPDATED: Keep raw TXT record value + # Keep raw TXT record value txt_value = str(record).strip('"') - dns_records.append(f"TXT: {txt_value}") + dns_records.append(txt_value) # Just the value for TXT continue elif record_type == 'SRV': target = str(record.target).rstrip('.') elif record_type == 'CAA': - # UPDATED: Keep raw CAA record format + # Keep raw CAA record format caa_value = f"{record.flags} {record.tag.decode('utf-8')} \"{record.value.decode('utf-8')}\"" - dns_records.append(f"CAA: {caa_value}") + dns_records.append(caa_value) # Just the value for CAA continue else: target = str(record) @@ -196,7 +202,7 @@ class DNSProvider(BaseProvider): 'value': target, 'ttl': response.ttl } - relationship_type = f"{record_type.lower()}_record" # Raw relationship type + relationship_type = f"{record_type.lower()}_record" confidence = 0.8 # Add relationship @@ -209,8 +215,8 @@ class DNSProvider(BaseProvider): raw_data=raw_data ) - # UPDATED: Keep raw DNS record format - dns_records.append(f"{record_type}: {target}") + # Add target to records list + dns_records.append(target) # Log relationship discovery self.log_relationship_discovery( @@ -222,20 +228,22 @@ class DNSProvider(BaseProvider): discovery_method=f"dns_{record_type.lower()}_record" ) - # Add DNS records as a consolidated attribute (raw format) + # FIXED: Create attribute with specific name for each record type if dns_records: + # Use record type specific attribute name (e.g., 'a_records', 'mx_records', etc.) + attribute_name = f"{record_type.lower()}_records" + result.add_attribute( target_node=domain, - name='dns_records', + name=attribute_name, # UNIQUE name for each record type! value=dns_records, attr_type='dns_record_list', provider=self.name, confidence=0.8, - metadata={'record_types': [record_type]} + metadata={'record_type': record_type, 'ttl': response.ttl} ) except Exception as e: self.failed_requests += 1 self.logger.logger.debug(f"{record_type} record query failed for {domain}: {e}") - raise e - + raise e \ No newline at end of file diff --git a/static/js/graph.js b/static/js/graph.js index 9391150..9273ddf 100644 --- a/static/js/graph.js +++ b/static/js/graph.js @@ -377,6 +377,21 @@ class GraphManager { this.initialize(); } + // Check if we have actual data to display + const hasData = graphData.nodes.length > 0 || graphData.edges.length > 0; + + // Handle placeholder visibility + const placeholder = this.container.querySelector('.graph-placeholder'); + if (placeholder) { + if (hasData) { + placeholder.style.display = 'none'; + } else { + placeholder.style.display = 'flex'; + // Early return if no data to process + return; + } + } + this.largeEntityMembers.clear(); const largeEntityMap = new Map(); @@ -398,11 +413,11 @@ class GraphManager { console.log(`Filtered ${graphData.nodes.length - filteredNodes.length} large entity member nodes from visualization`); - // FIXED: Process nodes with proper certificate coloring + // Process nodes with proper certificate coloring const processedNodes = filteredNodes.map(node => { const processed = this.processNode(node); - // FIXED: Apply certificate-based coloring here in frontend + // Apply certificate-based coloring here in frontend if (node.type === 'domain' && Array.isArray(node.attributes)) { const certInfo = this.analyzeCertificateInfo(node.attributes); diff --git a/static/js/main.js b/static/js/main.js index 72f5fdf..ea96e9e 100644 --- a/static/js/main.js +++ b/static/js/main.js @@ -35,6 +35,9 @@ class DNSReconApp { this.loadProviders(); this.initializeEnhancedModals(); + // FIXED: Force initial graph update to handle empty sessions properly + this.updateGraph(); + console.log('DNSRecon application initialized successfully'); } catch (error) { console.error('Failed to initialize DNSRecon application:', error); @@ -42,7 +45,7 @@ class DNSReconApp { } }); } - + /** * Initialize DOM element references */ @@ -484,9 +487,8 @@ class DNSReconApp { console.log('- Nodes:', graphData.nodes ? graphData.nodes.length : 0); console.log('- Edges:', graphData.edges ? graphData.edges.length : 0); - // Only update if data has changed - if (this.hasGraphChanged(graphData)) { - console.log('*** GRAPH DATA CHANGED - UPDATING VISUALIZATION ***'); + // FIXED: Always update graph, even if empty - let GraphManager handle placeholder + if (this.graphManager) { this.graphManager.updateGraph(graphData); this.lastGraphUpdate = Date.now(); @@ -495,18 +497,30 @@ class DNSReconApp { if (this.elements.relationshipsDisplay) { this.elements.relationshipsDisplay.textContent = edgeCount; } - } else { - console.log('Graph data unchanged, skipping update'); } } else { console.error('Graph update failed:', response); + // FIXED: Show placeholder when graph update fails + if (this.graphManager && this.graphManager.container) { + const placeholder = this.graphManager.container.querySelector('.graph-placeholder'); + if (placeholder) { + placeholder.style.display = 'flex'; + } + } } } catch (error) { console.error('Failed to update graph:', error); - // Don't show error for graph updates to avoid spam + // FIXED: Show placeholder on error + if (this.graphManager && this.graphManager.container) { + const placeholder = this.graphManager.container.querySelector('.graph-placeholder'); + if (placeholder) { + placeholder.style.display = 'flex'; + } + } } } + /** * Update status display elements @@ -925,18 +939,32 @@ class DNSReconApp { return 'Empty Array'; } - // Special handling for DNS records and similar arrays - if (name === 'dns_records' || name.includes('record') || name.includes('hostname')) { - // Show DNS records as a readable list + // ENHANCED: Special handling for specific DNS record types + if (name.endsWith('_records') || name.includes('record')) { + const recordType = name.replace('_records', '').toUpperCase(); + + // Format nicely for DNS records if (value.length <= 5) { - return this.escapeHtml(value.join(', ')); + const formattedRecords = value.map(record => { + // Add record type prefix if not already present + if (recordType !== 'DNS' && !record.includes(':')) { + return `${recordType}: ${record}`; + } + return record; + }); + return this.escapeHtml(formattedRecords.join('\n')); } else { - const preview = value.slice(0, 3).join(', '); - return this.escapeHtml(`${preview} ... (+${value.length - 3} more)`); + const preview = value.slice(0, 3).map(record => { + if (recordType !== 'DNS' && !record.includes(':')) { + return `${recordType}: ${record}`; + } + return record; + }).join('\n'); + return this.escapeHtml(`${preview}\n... (+${value.length - 3} more ${recordType} records)`); } } - // For other arrays + // For other arrays (existing logic) if (value.length <= 3) { return this.escapeHtml(value.join(', ')); } else { @@ -953,6 +981,10 @@ class DNSReconApp { } groupAttributesByProviderAndType(attributes, nodeType) { + if (!Array.isArray(attributes) || attributes.length === 0) { + return {}; + } + const groups = { 'DNS Records': { icon: '📋', priority: 'high', attributes: [] }, 'Certificate Information': { icon: '🔒', priority: 'high', attributes: [] }, @@ -968,11 +1000,11 @@ class DNSReconApp { let assigned = false; - // DNS-related attributes (better detection) + // ENHANCED: Better DNS record detection for specific record types if (provider === 'dns' || - name === 'dns_records' || + name.endsWith('_records') || // Catches a_records, mx_records, txt_records, etc. name.includes('record') || - ['ptr', 'mx', 'cname', 'ns', 'txt', 'soa'].some(keyword => name.includes(keyword))) { + ['ptr', 'mx', 'cname', 'ns', 'txt', 'soa', 'srv', 'caa', 'a_records', 'aaaa_records'].some(keyword => name.includes(keyword))) { groups['DNS Records'].attributes.push(attr); assigned = true; } @@ -1013,7 +1045,6 @@ class DNSReconApp { formatEdgeLabel(relationshipType, confidence) { if (!relationshipType) return ''; - // UPDATED: No formatting of relationship type - use raw values const confidenceText = confidence >= 0.8 ? '●' : confidence >= 0.6 ? '◐' : '○'; return `${relationshipType} ${confidenceText}`; } @@ -1695,13 +1726,18 @@ class DNSReconApp { const newNodeCount = graphData.nodes ? graphData.nodes.length : 0; const newEdgeCount = graphData.edges ? graphData.edges.length : 0; + // FIXED: Always update if we currently have no data (ensures placeholder is handled correctly) + if (currentStats.nodeCount === 0 && currentStats.edgeCount === 0) { + return true; + } + // Check if counts changed const countsChanged = currentStats.nodeCount !== newNodeCount || currentStats.edgeCount !== newEdgeCount; // Also check if we have new timestamp data const hasNewTimestamp = graphData.statistics && - graphData.statistics.last_modified && - graphData.statistics.last_modified !== this.lastGraphTimestamp; + graphData.statistics.last_modified && + graphData.statistics.last_modified !== this.lastGraphTimestamp; if (hasNewTimestamp) { this.lastGraphTimestamp = graphData.statistics.last_modified; @@ -1713,7 +1749,7 @@ class DNSReconApp { return changed; } - + /** * Make API call to server * @param {string} endpoint - API endpoint From 469c133f1bc1f0b1bae4dbd70d978dae562ca86e Mon Sep 17 00:00:00 2001 From: overcuriousity Date: Wed, 17 Sep 2025 11:18:06 +0200 Subject: [PATCH 08/20] fix session handling --- app.py | 92 ++++++++++-------------- core/scanner.py | 27 ++++--- core/session_manager.py | 152 +++++++++++++++++++++++++++++----------- 3 files changed, 167 insertions(+), 104 deletions(-) diff --git a/app.py b/app.py index 1955e36..450337c 100644 --- a/app.py +++ b/app.py @@ -24,50 +24,28 @@ app.config['PERMANENT_SESSION_LIFETIME'] = timedelta(hours=config.flask_permanen def get_user_scanner(): """ - Retrieves the scanner for the current session, or creates a new - session and scanner if one doesn't exist. + Retrieves the scanner for the current session, or creates a new session only if none exists. """ - # Get current Flask session info for debugging current_flask_session_id = session.get('dnsrecon_session_id') - # Try to get existing session + # Try to get existing session first if current_flask_session_id: existing_scanner = session_manager.get_session(current_flask_session_id) if existing_scanner: + print(f"Reusing existing session: {current_flask_session_id}") return current_flask_session_id, existing_scanner else: - print(f"Session {current_flask_session_id} not found in Redis, checking for active sessions...") + print(f"Session {current_flask_session_id} expired, will create new one") - # This prevents creating duplicate sessions when Flask session is lost but Redis session exists - stats = session_manager.get_statistics() - if stats['running_scans'] > 0: - # Get all session keys and find running ones - try: - import redis - redis_client = redis.StrictRedis(db=0, decode_responses=False) - session_keys = redis_client.keys("dnsrecon:session:*") - - for session_key in session_keys: - session_id = session_key.decode('utf-8').split(':')[-1] - scanner = session_manager.get_session(session_id) - if scanner and scanner.status in ['running', 'completed']: - print(f"Reusing active session: {session_id}") - # Update Flask session to match - session['dnsrecon_session_id'] = session_id - session.permanent = True - return session_id, scanner - except Exception as e: - print(f"Error finding active session: {e}") - - # Create new session if none exists - print("Creating new session as none was found...") + # Only create new session if we absolutely don't have one + print("Creating new session (no valid session found)") new_session_id = session_manager.create_session() new_scanner = session_manager.get_session(new_session_id) if not new_scanner: raise Exception("Failed to create new scanner session") - # Store in Flask session with explicit settings + # Store in Flask session session['dnsrecon_session_id'] = new_session_id session.permanent = True @@ -83,8 +61,8 @@ def index(): @app.route('/api/scan/start', methods=['POST']) def start_scan(): """ - Start a new reconnaissance scan. Creates a new isolated scanner if - clear_graph is true, otherwise adds to the existing one. + FIXED: Start a new reconnaissance scan while preserving session configuration. + Only clears graph data, not the entire session with API keys. """ print("=== API: /api/scan/start called ===") @@ -96,7 +74,7 @@ def start_scan(): target = data['target'].strip() max_depth = data.get('max_depth', config.default_recursion_depth) clear_graph = data.get('clear_graph', True) - force_rescan_target = data.get('force_rescan_target', None) # **FIX**: Get the new parameter + force_rescan_target = data.get('force_rescan_target', None) print(f"Parsed - target: '{target}', max_depth: {max_depth}, clear_graph: {clear_graph}, force_rescan: {force_rescan_target}") @@ -108,28 +86,17 @@ def start_scan(): if not isinstance(max_depth, int) or not 1 <= max_depth <= 5: return jsonify({'success': False, 'error': 'Max depth must be an integer between 1 and 5'}), 400 - user_session_id, scanner = None, None - - if clear_graph: - print("Clear graph requested: Creating a new, isolated scanner session.") - old_session_id = session.get('dnsrecon_session_id') - if old_session_id: - session_manager.terminate_session(old_session_id) - - user_session_id = session_manager.create_session() - session['dnsrecon_session_id'] = user_session_id - session.permanent = True - scanner = session_manager.get_session(user_session_id) - else: - print("Adding to existing graph: Reusing the current scanner session.") - user_session_id, scanner = get_user_scanner() - + # FIXED: Always reuse existing session, preserve API keys + user_session_id, scanner = get_user_scanner() + if not scanner: - return jsonify({'success': False, 'error': 'Failed to get or create a scanner instance.'}), 500 + return jsonify({'success': False, 'error': 'Failed to get scanner instance.'}), 500 print(f"Using scanner {id(scanner)} in session {user_session_id}") + print(f"Scanner has {len(scanner.providers)} providers: {[p.get_name() for p in scanner.providers]}") - success = scanner.start_scan(target, max_depth, clear_graph=clear_graph, force_rescan_target=force_rescan_target) # **FIX**: Pass the new parameter + # FIXED: Pass clear_graph flag to scanner, let it handle graph clearing internally + success = scanner.start_scan(target, max_depth, clear_graph=clear_graph, force_rescan_target=force_rescan_target) if success: return jsonify({ @@ -137,6 +104,7 @@ def start_scan(): 'message': 'Scan started successfully', 'scan_id': scanner.logger.session_id, 'user_session_id': user_session_id, + 'available_providers': [p.get_name() for p in scanner.providers] # Show which providers are active }) else: return jsonify({ @@ -148,7 +116,7 @@ def start_scan(): print(f"ERROR: Exception in start_scan endpoint: {e}") traceback.print_exc() return jsonify({'success': False, 'error': f'Internal server error: {str(e)}'}), 500 - + @app.route('/api/scan/stop', methods=['POST']) def stop_scan(): """Stop the current scan with immediate GUI feedback.""" @@ -237,7 +205,9 @@ def get_scan_status(): status['debug_info'] = { 'scanner_object_id': id(scanner), 'session_id_set': bool(scanner.session_id), - 'has_scan_thread': bool(scanner.scan_thread and scanner.scan_thread.is_alive()) + 'has_scan_thread': bool(scanner.scan_thread and scanner.scan_thread.is_alive()), + 'provider_count': len(scanner.providers), + 'provider_names': [p.get_name() for p in scanner.providers] } return jsonify({ @@ -499,6 +469,7 @@ def get_providers(): completed_tasks = scanner.indicators_completed total_tasks = scanner.total_tasks_ever_enqueued print(f"DEBUG: Task Progress - Completed: {completed_tasks}, Total Enqueued: {total_tasks}") + print(f"DEBUG: Scanner has {len(scanner.providers)} providers: {[p.get_name() for p in scanner.providers]}") else: print("DEBUG: No active scanner session found.") @@ -522,7 +493,6 @@ def get_providers(): @app.route('/api/config/api-keys', methods=['POST']) def set_api_keys(): """ - Set API keys for providers for the user session only. """ try: data = request.get_json() @@ -537,6 +507,8 @@ def set_api_keys(): user_session_id, scanner = get_user_scanner() session_config = scanner.config + print(f"Setting API keys for session {user_session_id}: {list(data.keys())}") + updated_providers = [] # Iterate over the API keys provided in the request data @@ -548,10 +520,17 @@ def set_api_keys(): if success: updated_providers.append(provider_name) + print(f"API key {'set' if api_key_value else 'cleared'} for {provider_name}") if updated_providers: - # Reinitialize scanner providers to apply the new keys + # FIXED: Reinitialize scanner providers to apply the new keys + print("Reinitializing providers with new API keys...") + old_provider_count = len(scanner.providers) scanner._initialize_providers() + new_provider_count = len(scanner.providers) + + print(f"Providers reinitialized: {old_provider_count} -> {new_provider_count}") + print(f"Available providers: {[p.get_name() for p in scanner.providers]}") # Persist the updated scanner object back to the user's session session_manager.update_session_scanner(user_session_id, scanner) @@ -560,7 +539,8 @@ def set_api_keys(): 'success': True, 'message': f'API keys updated for session {user_session_id}: {", ".join(updated_providers)}', 'updated_providers': updated_providers, - 'user_session_id': user_session_id + 'user_session_id': user_session_id, + 'available_providers': [p.get_name() for p in scanner.providers] }) else: return jsonify({ @@ -597,7 +577,7 @@ def internal_error(error): if __name__ == '__main__': - print("Starting DNSRecon Flask application with user session support...") + print("Starting DNSRecon Flask application with streamlined session management...") # Load configuration from environment config.load_from_env() diff --git a/core/scanner.py b/core/scanner.py index a3bdb7b..735ddf3 100644 --- a/core/scanner.py +++ b/core/scanner.py @@ -30,7 +30,7 @@ class ScanStatus: class Scanner: """ Main scanning orchestrator for DNSRecon passive reconnaissance. - Now provider-agnostic, consuming standardized ProviderResult objects. + FIXED: Now preserves session configuration including API keys when clearing graphs. """ def __init__(self, session_config=None): @@ -220,13 +220,18 @@ class Scanner: print("Session configuration updated") def start_scan(self, target: str, max_depth: int = 2, clear_graph: bool = True, force_rescan_target: Optional[str] = None) -> bool: - """Start a new reconnaissance scan with proper cleanup of previous scans.""" + """ + FIXED: Start a new reconnaissance scan preserving session configuration. + Only clears graph data when requested, never destroys session/API keys. + """ print(f"=== STARTING SCAN IN SCANNER {id(self)} ===") print(f"Session ID: {self.session_id}") print(f"Initial scanner status: {self.status}") + print(f"Clear graph requested: {clear_graph}") + print(f"Current providers: {[p.get_name() for p in self.providers]}") self.total_tasks_ever_enqueued = 0 - # **IMPROVED**: More aggressive cleanup of previous scan + # FIXED: Improved cleanup of previous scan without destroying session config if self.scan_thread and self.scan_thread.is_alive(): print("A previous scan thread is still alive. Forcing termination...") @@ -251,15 +256,14 @@ class Scanner: if self.scan_thread.is_alive(): print("WARNING: Previous scan thread is still alive after 5 seconds") - # Continue anyway, but log the issue self.logger.logger.warning("Previous scan thread failed to terminate cleanly") - # Reset state for new scan with proper forensic logging - print("Resetting scanner state for new scan...") + # FIXED: Reset scan state but preserve session configuration (API keys, etc.) + print("Resetting scanner state for new scan (preserving session config)...") self.status = ScanStatus.IDLE self.stop_event.clear() - # **NEW**: Clear Redis stop signal explicitly + # Clear Redis stop signal explicitly if self.session_id: from core.session_manager import session_manager session_manager.clear_stop_signal(self.session_id) @@ -267,13 +271,14 @@ class Scanner: with self.processing_lock: self.currently_processing.clear() + # Reset scan-specific state but keep providers and config intact self.task_queue = PriorityQueue() self.target_retries.clear() self.scan_failed_due_to_retries = False # Update session state immediately for GUI feedback self._update_session_state() - print("Scanner state reset complete.") + print("Scanner state reset complete (providers preserved).") try: if not hasattr(self, 'providers') or not self.providers: @@ -282,9 +287,12 @@ class Scanner: print(f"Scanner {id(self)} validation passed, providers available: {[p.get_name() for p in self.providers]}") + # FIXED: Only clear graph if explicitly requested, don't destroy session if clear_graph: + print("Clearing graph data (preserving session configuration)") self.graph.clear() + # Handle force rescan by clearing provider states for that specific node if force_rescan_target and self.graph.graph.has_node(force_rescan_target): print(f"Forcing rescan of {force_rescan_target}, clearing provider states.") node_data = self.graph.graph.nodes[force_rescan_target] @@ -304,7 +312,7 @@ class Scanner: # Update GUI with scan preparation state self._update_session_state() - # Start new forensic session + # Start new forensic session (but don't reinitialize providers) print(f"Starting new forensic session for scanner {id(self)}...") self.logger = new_session() @@ -318,6 +326,7 @@ class Scanner: self.scan_thread.start() print(f"=== SCAN STARTED SUCCESSFULLY IN SCANNER {id(self)} ===") + print(f"Active providers for this scan: {[p.get_name() for p in self.providers]}") return True except Exception as e: diff --git a/core/session_manager.py b/core/session_manager.py index 4662c65..d44c9e8 100644 --- a/core/session_manager.py +++ b/core/session_manager.py @@ -12,7 +12,8 @@ from config import config class SessionManager: """ - Manages multiple scanner instances for concurrent user sessions using Redis. + FIXED: Manages multiple scanner instances for concurrent user sessions using Redis. + Now more conservative about session creation to preserve API keys and configuration. """ def __init__(self, session_timeout_minutes: int = 0): @@ -24,7 +25,10 @@ class SessionManager: self.redis_client = redis.StrictRedis(db=0, decode_responses=False) self.session_timeout = session_timeout_minutes * 60 # Convert to seconds - self.lock = threading.Lock() # Lock for local operations, Redis handles atomic ops + self.lock = threading.Lock() + + # FIXED: Add a creation lock to prevent race conditions + self.creation_lock = threading.Lock() # Start cleanup thread self.cleanup_thread = threading.Thread(target=self._cleanup_loop, daemon=True) @@ -36,7 +40,7 @@ class SessionManager: """Prepare SessionManager for pickling.""" state = self.__dict__.copy() # Exclude unpickleable attributes - Redis client and threading objects - unpicklable_attrs = ['lock', 'cleanup_thread', 'redis_client'] + unpicklable_attrs = ['lock', 'cleanup_thread', 'redis_client', 'creation_lock'] for attr in unpicklable_attrs: if attr in state: del state[attr] @@ -49,6 +53,7 @@ class SessionManager: import redis self.redis_client = redis.StrictRedis(db=0, decode_responses=False) self.lock = threading.Lock() + self.creation_lock = threading.Lock() self.cleanup_thread = threading.Thread(target=self._cleanup_loop, daemon=True) self.cleanup_thread.start() @@ -62,44 +67,106 @@ class SessionManager: def create_session(self) -> str: """ - Create a new user session and store it in Redis. + FIXED: Create a new user session with thread-safe creation to prevent duplicates. """ - session_id = str(uuid.uuid4()) - print(f"=== CREATING SESSION {session_id} IN REDIS ===") - - try: - from core.session_config import create_session_config - session_config = create_session_config() - scanner_instance = Scanner(session_config=session_config) + # FIXED: Use creation lock to prevent race conditions + with self.creation_lock: + session_id = str(uuid.uuid4()) + print(f"=== CREATING SESSION {session_id} IN REDIS ===") - # Set the session ID on the scanner for cross-process stop signal management - scanner_instance.session_id = session_id + try: + from core.session_config import create_session_config + session_config = create_session_config() + scanner_instance = Scanner(session_config=session_config) + + # Set the session ID on the scanner for cross-process stop signal management + scanner_instance.session_id = session_id + + session_data = { + 'scanner': scanner_instance, + 'config': session_config, + 'created_at': time.time(), + 'last_activity': time.time(), + 'status': 'active' + } + + # Serialize the entire session data dictionary using pickle + serialized_data = pickle.dumps(session_data) + + # Store in Redis + session_key = self._get_session_key(session_id) + self.redis_client.setex(session_key, self.session_timeout, serialized_data) + + # Initialize stop signal as False + stop_key = self._get_stop_signal_key(session_id) + self.redis_client.setex(stop_key, self.session_timeout, b'0') + + print(f"Session {session_id} stored in Redis with stop signal initialized") + print(f"Session has {len(scanner_instance.providers)} providers: {[p.get_name() for p in scanner_instance.providers]}") + return session_id + + except Exception as e: + print(f"ERROR: Failed to create session {session_id}: {e}") + raise + + def clone_session_preserving_config(self, source_session_id: str) -> str: + """ + FIXED: Create a new session that preserves the configuration (including API keys) from an existing session. + This is used when we need a fresh scanner but want to keep user configuration. + """ + with self.creation_lock: + print(f"=== CLONING SESSION {source_session_id} (PRESERVING CONFIG) ===") - session_data = { - 'scanner': scanner_instance, - 'config': session_config, - 'created_at': time.time(), - 'last_activity': time.time(), - 'status': 'active' - } - - # Serialize the entire session data dictionary using pickle - serialized_data = pickle.dumps(session_data) - - # Store in Redis - session_key = self._get_session_key(session_id) - self.redis_client.setex(session_key, self.session_timeout, serialized_data) - - # Initialize stop signal as False - stop_key = self._get_stop_signal_key(session_id) - self.redis_client.setex(stop_key, self.session_timeout, b'0') - - print(f"Session {session_id} stored in Redis with stop signal initialized") - return session_id - - except Exception as e: - print(f"ERROR: Failed to create session {session_id}: {e}") - raise + try: + # Get the source session data + source_session_data = self._get_session_data(source_session_id) + if not source_session_data: + print(f"ERROR: Source session {source_session_id} not found for cloning") + return self.create_session() # Fallback to new session + + # Create new session ID + new_session_id = str(uuid.uuid4()) + + # Get the preserved configuration + preserved_config = source_session_data.get('config') + if not preserved_config: + print(f"WARNING: No config found in source session, creating new") + from core.session_config import create_session_config + preserved_config = create_session_config() + + print(f"Preserving config with API keys: {list(preserved_config.api_keys.keys())}") + + # Create new scanner with preserved config + new_scanner = Scanner(session_config=preserved_config) + new_scanner.session_id = new_session_id + + print(f"New scanner has {len(new_scanner.providers)} providers: {[p.get_name() for p in new_scanner.providers]}") + + new_session_data = { + 'scanner': new_scanner, + 'config': preserved_config, + 'created_at': time.time(), + 'last_activity': time.time(), + 'status': 'active', + 'cloned_from': source_session_id + } + + # Store in Redis + serialized_data = pickle.dumps(new_session_data) + session_key = self._get_session_key(new_session_id) + self.redis_client.setex(session_key, self.session_timeout, serialized_data) + + # Initialize stop signal + stop_key = self._get_stop_signal_key(new_session_id) + self.redis_client.setex(stop_key, self.session_timeout, b'0') + + print(f"Cloned session {new_session_id} with preserved configuration") + return new_session_id + + except Exception as e: + print(f"ERROR: Failed to clone session {source_session_id}: {e}") + # Fallback to creating a new session + return self.create_session() def set_stop_signal(self, session_id: str) -> bool: """ @@ -208,7 +275,14 @@ class SessionManager: # Immediately save to Redis for GUI updates success = self._save_session_data(session_id, session_data) if success: - print(f"Scanner state updated for session {session_id} (status: {scanner.status})") + # Only log occasionally to reduce noise + if hasattr(self, '_last_update_log'): + if time.time() - self._last_update_log > 5: # Log every 5 seconds max + print(f"Scanner state updated for session {session_id} (status: {scanner.status})") + self._last_update_log = time.time() + else: + print(f"Scanner state updated for session {session_id} (status: {scanner.status})") + self._last_update_log = time.time() else: print(f"WARNING: Failed to save scanner state for session {session_id}") return success From ec755b17ad5bb7cabeefc80310868715394e83f8 Mon Sep 17 00:00:00 2001 From: overcuriousity Date: Wed, 17 Sep 2025 13:31:35 +0200 Subject: [PATCH 09/20] remove many unnecessary debug print, improve large entity handling --- .env.example | 2 +- app.py | 4 +-- config.py | 4 +-- core/graph_manager.py | 16 +++++++---- core/logger.py | 2 +- core/scanner.py | 63 +++++++++++++++++++++-------------------- core/session_manager.py | 2 -- static/js/graph.js | 46 +++--------------------------- static/js/main.js | 39 +++---------------------- 9 files changed, 57 insertions(+), 121 deletions(-) diff --git a/.env.example b/.env.example index 6c05aa7..002b9d5 100644 --- a/.env.example +++ b/.env.example @@ -25,7 +25,7 @@ DEFAULT_RECURSION_DEPTH=2 # Default timeout for provider API requests in seconds. DEFAULT_TIMEOUT=30 # The number of concurrent provider requests to make. -MAX_CONCURRENT_REQUESTS=5 +MAX_CONCURRENT_REQUESTS=1 # The number of results from a provider that triggers the "large entity" grouping. LARGE_ENTITY_THRESHOLD=100 # The number of times to retry a target if a provider fails. diff --git a/app.py b/app.py index 450337c..fceb61a 100644 --- a/app.py +++ b/app.py @@ -32,7 +32,7 @@ def get_user_scanner(): if current_flask_session_id: existing_scanner = session_manager.get_session(current_flask_session_id) if existing_scanner: - print(f"Reusing existing session: {current_flask_session_id}") + #print(f"Reusing existing session: {current_flask_session_id}") return current_flask_session_id, existing_scanner else: print(f"Session {current_flask_session_id} expired, will create new one") @@ -93,7 +93,6 @@ def start_scan(): return jsonify({'success': False, 'error': 'Failed to get scanner instance.'}), 500 print(f"Using scanner {id(scanner)} in session {user_session_id}") - print(f"Scanner has {len(scanner.providers)} providers: {[p.get_name() for p in scanner.providers]}") # FIXED: Pass clear_graph flag to scanner, let it handle graph clearing internally success = scanner.start_scan(target, max_depth, clear_graph=clear_graph, force_rescan_target=force_rescan_target) @@ -469,7 +468,6 @@ def get_providers(): completed_tasks = scanner.indicators_completed total_tasks = scanner.total_tasks_ever_enqueued print(f"DEBUG: Task Progress - Completed: {completed_tasks}, Total Enqueued: {total_tasks}") - print(f"DEBUG: Scanner has {len(scanner.providers)} providers: {[p.get_name() for p in scanner.providers]}") else: print("DEBUG: No active scanner session found.") diff --git a/config.py b/config.py index a01eb24..2b2ff1c 100644 --- a/config.py +++ b/config.py @@ -21,8 +21,8 @@ class Config: # --- General Settings --- self.default_recursion_depth = 2 - self.default_timeout = 30 - self.max_concurrent_requests = 5 + self.default_timeout = 60 + self.max_concurrent_requests = 1 self.large_entity_threshold = 100 self.max_retries_per_target = 8 self.cache_expiry_hours = 12 diff --git a/core/graph_manager.py b/core/graph_manager.py index f0629cb..0d21d2d 100644 --- a/core/graph_manager.py +++ b/core/graph_manager.py @@ -344,13 +344,19 @@ class GraphManager: return False node_data = self.graph.nodes[large_entity_id] - attributes = node_data.get('attributes', {}) + attributes = node_data.get('attributes', []) + + # Find the 'nodes' attribute dictionary in the list + nodes_attr = next((attr for attr in attributes if attr.get('name') == 'nodes'), None) # Remove from the list of member nodes - if 'nodes' in attributes and node_id_to_extract in attributes['nodes']: - attributes['nodes'].remove(node_id_to_extract) - # Update the count - attributes['count'] = len(attributes['nodes']) + if nodes_attr and 'value' in nodes_attr and isinstance(nodes_attr['value'], list) and node_id_to_extract in nodes_attr['value']: + nodes_attr['value'].remove(node_id_to_extract) + + # Find the 'count' attribute and update it + count_attr = next((attr for attr in attributes if attr.get('name') == 'count'), None) + if count_attr: + count_attr['value'] = len(nodes_attr['value']) else: # This can happen if the node was already extracted, which is not an error. print(f"Warning: Node {node_id_to_extract} not found in the 'nodes' list of {large_entity_id}.") diff --git a/core/logger.py b/core/logger.py index f391030..371cd7e 100644 --- a/core/logger.py +++ b/core/logger.py @@ -152,7 +152,7 @@ class ForensicLogger: # Log to standard logger if error: - self.logger.error(f"API Request Failed - {provider}: {url} - {error}") + self.logger.error(f"API Request Failed.") else: self.logger.info(f"API Request - {provider}: {url} - Status: {status_code}") diff --git a/core/scanner.py b/core/scanner.py index 735ddf3..55d6b1c 100644 --- a/core/scanner.py +++ b/core/scanner.py @@ -432,7 +432,7 @@ class Scanner: self.indicators_completed += 1 if not self._is_stop_requested(): - all_new_targets = new_targets.union(large_entity_members) + all_new_targets = new_targets for new_target in all_new_targets: is_ip_new = _is_valid_ip(new_target) eligible_providers_new = self._get_eligible_providers(new_target, is_ip_new, False) @@ -576,6 +576,33 @@ class Scanner: print(f"Stop requested before processing results from {provider_name} for {target}") return discovered_targets, False + attributes_by_node = defaultdict(list) + for attribute in provider_result.attributes: + # Convert the StandardAttribute object to a dictionary that the frontend can use + attr_dict = { + "name": attribute.name, + "value": attribute.value, + "type": attribute.type, + "provider": attribute.provider, + "confidence": attribute.confidence, + "metadata": attribute.metadata + } + attributes_by_node[attribute.target_node].append(attr_dict) + + # Add attributes to nodes + for node_id, node_attributes_list in attributes_by_node.items(): + if self.graph.graph.has_node(node_id): + # Determine node type + if _is_valid_ip(node_id): + node_type = NodeType.IP + elif node_id.startswith('AS') and node_id[2:].isdigit(): + node_type = NodeType.ASN + else: + node_type = NodeType.DOMAIN + + # Add node with the list of attributes + self.graph.add_node(node_id, node_type, attributes=node_attributes_list) + # Check for large entity based on relationship count if provider_result.get_relationship_count() > self.config.large_entity_threshold: print(f"Large entity detected: {provider_name} returned {provider_result.get_relationship_count()} relationships for {target}") @@ -619,34 +646,6 @@ class Scanner: if _is_valid_domain(target_node) or _is_valid_ip(target_node): discovered_targets.add(target_node) - # Process attributes, preserving them as a list of objects - attributes_by_node = defaultdict(list) - for attribute in provider_result.attributes: - # Convert the StandardAttribute object to a dictionary that the frontend can use - attr_dict = { - "name": attribute.name, - "value": attribute.value, - "type": attribute.type, - "provider": attribute.provider, - "confidence": attribute.confidence, - "metadata": attribute.metadata - } - attributes_by_node[attribute.target_node].append(attr_dict) - - # Add attributes to nodes - for node_id, node_attributes_list in attributes_by_node.items(): - if self.graph.graph.has_node(node_id): - # Determine node type - if _is_valid_ip(node_id): - node_type = NodeType.IP - elif node_id.startswith('AS') and node_id[2:].isdigit(): - node_type = NodeType.ASN - else: - node_type = NodeType.DOMAIN - - # Add node with the list of attributes - self.graph.add_node(node_id, node_type, attributes=node_attributes_list) - return discovered_targets, False def _create_large_entity_from_provider_result(self, source: str, provider_name: str, @@ -793,7 +792,11 @@ class Scanner: # 4. Re-queue the extracted node for full processing by all eligible providers print(f"Re-queueing extracted node {node_id_to_extract} for full reconnaissance...") is_ip = _is_valid_ip(node_id_to_extract) - current_depth = self.graph.graph.nodes[large_entity_id].get('attributes', {}).get('discovery_depth', 0) + + # FIX: Correctly retrieve discovery_depth from the list of attributes + large_entity_attributes = self.graph.graph.nodes[large_entity_id].get('attributes', []) + discovery_depth_attr = next((attr for attr in large_entity_attributes if attr.get('name') == 'discovery_depth'), None) + current_depth = discovery_depth_attr['value'] if discovery_depth_attr else 0 eligible_providers = self._get_eligible_providers(node_id_to_extract, is_ip, False) for provider in eligible_providers: diff --git a/core/session_manager.py b/core/session_manager.py index d44c9e8..266c356 100644 --- a/core/session_manager.py +++ b/core/session_manager.py @@ -50,7 +50,6 @@ class SessionManager: """Restore SessionManager after unpickling.""" self.__dict__.update(state) # Re-initialize unpickleable attributes - import redis self.redis_client = redis.StrictRedis(db=0, decode_responses=False) self.lock = threading.Lock() self.creation_lock = threading.Lock() @@ -140,7 +139,6 @@ class SessionManager: new_scanner = Scanner(session_config=preserved_config) new_scanner.session_id = new_session_id - print(f"New scanner has {len(new_scanner.providers)} providers: {[p.get_name() for p in new_scanner.providers]}") new_session_data = { 'scanner': new_scanner, diff --git a/static/js/graph.js b/static/js/graph.js index 9273ddf..a6693f0 100644 --- a/static/js/graph.js +++ b/static/js/graph.js @@ -214,7 +214,6 @@ class GraphManager { }); document.body.appendChild(this.contextMenu); - console.log('Context menu created and added to body'); } /** @@ -291,7 +290,6 @@ class GraphManager { // FIXED: Right-click context menu this.container.addEventListener('contextmenu', (event) => { event.preventDefault(); - console.log('Right-click detected at:', event.offsetX, event.offsetY); // Get coordinates relative to the canvas const pointer = { @@ -300,7 +298,6 @@ class GraphManager { }; const nodeId = this.network.getNodeAt(pointer); - console.log('Node at pointer:', nodeId); if (nodeId) { // Pass the original client event for positioning @@ -341,19 +338,12 @@ class GraphManager { // Stabilization events with progress this.network.on('stabilizationProgress', (params) => { const progress = params.iterations / params.total; - this.updateStabilizationProgress(progress); }); this.network.on('stabilizationIterationsDone', () => { this.onStabilizationComplete(); }); - // Selection events - this.network.on('select', (params) => { - console.log('Selected nodes:', params.nodes); - console.log('Selected edges:', params.edges); - }); - // Click away to hide context menu document.addEventListener('click', (e) => { if (!this.contextMenu.contains(e.target)) { @@ -979,15 +969,6 @@ class GraphManager { }, 2000); } - /** - * Update stabilization progress - * @param {number} progress - Progress value (0-1) - */ - updateStabilizationProgress(progress) { - // Could show a progress indicator if needed - console.log(`Graph stabilization: ${(progress * 100).toFixed(1)}%`); - } - /** * Handle stabilization completion */ @@ -1175,11 +1156,11 @@ class GraphManager { adjacencyList ); - console.log(`Reachability analysis complete:`, { + /*console.log(`Reachability analysis complete:`, { reachable: analysis.reachableNodes.size, unreachable: analysis.unreachableNodes.size, clusters: analysis.isolatedClusters.length - }); + });*/ return analysis; } @@ -1294,7 +1275,6 @@ class GraphManager { * Replaces the existing applyAllFilters() method */ applyAllFilters() { - console.log("Applying filters with enhanced reachability analysis..."); if (this.nodes.length === 0) return; // Get filter criteria from UI @@ -1350,23 +1330,10 @@ class GraphManager { operation: 'hide_with_reachability', timestamp: Date.now() }; - - // Apply hiding with forensic documentation - const updates = nodesToHide.map(id => ({ - id: id, - hidden: true, - forensicNote: `Hidden due to reachability analysis from ${nodeId}` - })); this.nodes.update(updates); this.addToHistory('hide', historyData); - console.log(`Forensic hide operation: ${nodesToHide.length} nodes hidden`, { - originalTarget: nodeId, - cascadeNodes: nodesToHide.length - 1, - isolatedClusters: analysis.isolatedClusters.length - }); - return { hiddenNodes: nodesToHide, isolatedClusters: analysis.isolatedClusters @@ -1449,9 +1416,7 @@ class GraphManager { // Handle operation results if (!operationFailed) { - this.addToHistory('delete', historyData); - console.log(`Forensic delete operation completed:`, historyData.forensicAnalysis); - + this.addToHistory('delete', historyData); return { success: true, deletedNodes: nodesToDelete, @@ -1542,7 +1507,6 @@ class GraphManager { e.stopPropagation(); const action = e.currentTarget.dataset.action; const nodeId = e.currentTarget.dataset.nodeId; - console.log('Context menu action:', action, 'for node:', nodeId); this.performContextMenuAction(action, nodeId); this.hideContextMenu(); }); @@ -1562,9 +1526,7 @@ class GraphManager { * UPDATED: Enhanced context menu actions using new methods * Updates the existing performContextMenuAction() method */ - performContextMenuAction(action, nodeId) { - console.log('Performing enhanced action:', action, 'on node:', nodeId); - + performContextMenuAction(action, nodeId) { switch (action) { case 'focus': this.focusOnNode(nodeId); diff --git a/static/js/main.js b/static/js/main.js index ea96e9e..f1d82c0 100644 --- a/static/js/main.js +++ b/static/js/main.js @@ -120,7 +120,6 @@ class DNSReconApp { max-width: 400px; `; document.body.appendChild(messageContainer); - console.log('Message container created'); } } @@ -135,7 +134,6 @@ class DNSReconApp { this.initializeModalFunctionality(); this.elements.startScan.addEventListener('click', (e) => { - console.log('Start scan button clicked'); e.preventDefault(); this.startScan(); }); @@ -146,13 +144,11 @@ class DNSReconApp { }); this.elements.stopScan.addEventListener('click', (e) => { - console.log('Stop scan button clicked'); e.preventDefault(); this.stopScan(); }); this.elements.exportResults.addEventListener('click', (e) => { - console.log('Export results button clicked'); e.preventDefault(); this.exportResults(); }); @@ -162,7 +158,6 @@ class DNSReconApp { // Enter key support for target domain input this.elements.targetInput.addEventListener('keypress', (e) => { if (e.key === 'Enter' && !this.isScanning) { - console.log('Enter key pressed in domain input'); this.startScan(); } }); @@ -287,12 +282,8 @@ class DNSReconApp { force_rescan_target: forceRescanTarget }; - console.log('Request data:', requestData); - const response = await this.apiCall('/api/scan/start', 'POST', requestData); - - console.log('API response received:', response); - + if (response.success) { this.currentSessionId = response.scan_id; this.showSuccess('Reconnaissance scan started successfully'); @@ -345,7 +336,6 @@ class DNSReconApp { if (response.success) { this.showSuccess('Scan stop requested'); - console.log('Scan stop requested successfully'); // Force immediate status update setTimeout(() => { @@ -414,7 +404,6 @@ class DNSReconApp { } this.pollInterval = setInterval(() => { - console.log('--- Polling tick ---'); this.updateStatus(); this.updateGraph(); this.loadProviders(); @@ -439,16 +428,11 @@ class DNSReconApp { */ async updateStatus() { try { - console.log('Updating status...'); const response = await this.apiCall('/api/scan/status'); - console.log('Status response:', response); if (response.success && response.status) { const status = response.status; - console.log('Current scan status:', status.status); - console.log('Current progress:', status.progress_percentage + '%'); - console.log('Graph stats:', status.graph_statistics); this.updateStatusDisplay(status); @@ -478,7 +462,6 @@ class DNSReconApp { console.log('Updating graph...'); const response = await this.apiCall('/api/graph'); - console.log('Graph response:', response); if (response.success) { const graphData = response.graph; @@ -527,9 +510,7 @@ class DNSReconApp { * @param {Object} status - Status object from server */ updateStatusDisplay(status) { - try { - console.log('Updating status display...'); - + try { // Update status text with animation if (this.elements.scanStatus) { const formattedStatus = this.formatStatus(status.status); @@ -584,8 +565,6 @@ class DNSReconApp { } this.setUIState(status.status, status.task_queue_size); - - console.log('Status display updated successfully'); } catch (error) { console.error('Error updating status display:', error); } @@ -680,9 +659,7 @@ class DNSReconApp { /** * UI state management with immediate button updates */ - setUIState(state, task_queue_size) { - console.log(`Setting UI state to: ${state}`); - + setUIState(state, task_queue_size) { const isQueueEmpty = task_queue_size === 0; switch (state) { @@ -743,7 +720,6 @@ class DNSReconApp { */ async loadProviders() { try { - console.log('Loading providers...'); const response = await this.apiCall('/api/providers'); if (response.success) { @@ -1401,7 +1377,6 @@ class DNSReconApp { e.stopPropagation(); const nodeId = e.target.dataset.nodeId || e.target.getAttribute('data-node-id'); - console.log('Node link clicked:', nodeId); if (nodeId && this.graphManager && this.graphManager.nodes) { const nextNode = this.graphManager.nodes.get(nodeId); @@ -1758,7 +1733,6 @@ class DNSReconApp { * @returns {Promise} Response data */ async apiCall(endpoint, method = 'GET', data = null) { - console.log(`Making API call: ${method} ${endpoint}`, data ? data : '(no data)'); try { const options = { @@ -1773,17 +1747,14 @@ class DNSReconApp { console.log('Request body:', options.body); } - console.log('Fetch options:', options); const response = await fetch(endpoint, options); - console.log('Response status:', response.status, response.statusText); if (!response.ok) { throw new Error(`HTTP ${response.status}: ${response.statusText}`); } const result = await response.json(); - console.log('Response data:', result); return result; @@ -1917,9 +1888,7 @@ class DNSReconApp { * @param {string} message - Message text * @param {string} type - Message type (success, error, warning, info) */ - showMessage(message, type = 'info') { - console.log(`${type.toUpperCase()}: ${message}`); - + showMessage(message, type = 'info') { // Create message element const messageElement = document.createElement('div'); messageElement.className = `message-toast message-${type}`; From 173c3dcf92ef29d25466420d6c6384e0f1bd2597 Mon Sep 17 00:00:00 2001 From: overcuriousity Date: Wed, 17 Sep 2025 17:10:11 +0200 Subject: [PATCH 10/20] some adjustments for clarity --- app.py | 15 ++++--- core/graph_manager.py | 2 +- core/scanner.py | 16 +++++-- providers/crtsh_provider.py | 86 +++++++++++++++++++----------------- providers/dns_provider.py | 6 +-- providers/shodan_provider.py | 12 ++--- 6 files changed, 77 insertions(+), 60 deletions(-) diff --git a/app.py b/app.py index fceb61a..d21e77a 100644 --- a/app.py +++ b/app.py @@ -463,12 +463,15 @@ def get_providers(): # Get user-specific scanner user_session_id, scanner = get_user_scanner() - if scanner: - # Updated debug print to be consistent with the new progress bar logic - completed_tasks = scanner.indicators_completed - total_tasks = scanner.total_tasks_ever_enqueued - print(f"DEBUG: Task Progress - Completed: {completed_tasks}, Total Enqueued: {total_tasks}") - else: + if scanner and scanner.status == 'running': + status = scanner.get_scan_status() + currently_processing = status.get('currently_processing') + if currently_processing: + provider_name, target_item = currently_processing[0] + print(f"DEBUG: RUNNING Task - Provider: {provider_name}, Target: {target_item}") + + print(f"DEBUG: Task Queue Status - In Queue: {status.get('tasks_in_queue', 0)}, Completed: {status.get('tasks_completed', 0)}, Skipped: {status.get('tasks_skipped', 0)}, Rescheduled: {status.get('tasks_rescheduled', 0)}") + elif not scanner: print("DEBUG: No active scanner session found.") provider_info = scanner.get_provider_info() diff --git a/core/graph_manager.py b/core/graph_manager.py index 0d21d2d..5a55052 100644 --- a/core/graph_manager.py +++ b/core/graph_manager.py @@ -41,7 +41,7 @@ class GraphManager: self.correlation_index = {} # Compile regex for date filtering for efficiency self.date_pattern = re.compile(r'^\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}') - self.EXCLUDED_KEYS = ['confidence', 'provider', 'timestamp', 'type','cert_validity_period_days','cert_source'] + self.EXCLUDED_KEYS = ['crtsh_cert_validity_period_days','crtsh_cert_source','crtsh_cert_common_name'] def __getstate__(self): """Prepare GraphManager for pickling, excluding compiled regex.""" diff --git a/core/scanner.py b/core/scanner.py index 55d6b1c..bc5f23a 100644 --- a/core/scanner.py +++ b/core/scanner.py @@ -396,7 +396,7 @@ class Scanner: if self._is_stop_requested(): print(f"Stop requested before processing {target_item}") break - self.currently_processing.add(target_item) + self.currently_processing.add(task_tuple) try: self.current_depth = depth @@ -444,7 +444,7 @@ class Scanner: self.total_tasks_ever_enqueued += 1 finally: with self.processing_lock: - self.currently_processing.discard(target_item) + self.currently_processing.discard(task_tuple) if self._is_stop_requested(): print("Scan terminated due to stop request") @@ -857,7 +857,11 @@ class Scanner: 'graph_statistics': self.graph.get_statistics(), 'task_queue_size': self.task_queue.qsize(), 'currently_processing_count': currently_processing_count, - 'currently_processing': currently_processing_list[:5] + 'currently_processing': currently_processing_list[:5], + 'tasks_in_queue': self.task_queue.qsize(), + 'tasks_completed': self.indicators_completed, + 'tasks_skipped': self.total_tasks_ever_enqueued - self.task_queue.qsize() - self.indicators_completed - self.tasks_re_enqueued, + 'tasks_rescheduled': self.tasks_re_enqueued, } except Exception as e: print(f"ERROR: Exception in get_scan_status: {e}") @@ -876,7 +880,11 @@ class Scanner: 'graph_statistics': {}, 'task_queue_size': 0, 'currently_processing_count': 0, - 'currently_processing': [] + 'currently_processing': [], + 'tasks_in_queue': 0, + 'tasks_completed': 0, + 'tasks_skipped': 0, + 'tasks_rescheduled': 0, } def _initialize_provider_states(self, target: str) -> None: diff --git a/providers/crtsh_provider.py b/providers/crtsh_provider.py index 6adde15..bb712a6 100644 --- a/providers/crtsh_provider.py +++ b/providers/crtsh_provider.py @@ -93,7 +93,7 @@ class CrtShProvider(BaseProvider): def query_domain(self, domain: str) -> ProviderResult: """ - Query crt.sh for certificates containing the domain with caching support. + Query crt.sh for certificates containing the domain with efficient, deduplicated caching. Args: domain: Domain to investigate @@ -110,35 +110,45 @@ class CrtShProvider(BaseProvider): cache_file = self._get_cache_file_path(domain) cache_status = self._get_cache_status(cache_file) - processed_certificates = [] result = ProviderResult() try: if cache_status == "fresh": result = self._load_from_cache(cache_file) - self.logger.logger.info(f"Using cached crt.sh data for {domain}") + self.logger.logger.info(f"Using fresh cached crt.sh data for {domain}") else: # "stale" or "not_found" - raw_certificates = self._query_crtsh_api(domain) + # Query the API for the latest certificates + new_raw_certs = self._query_crtsh_api(domain) if self._stop_event and self._stop_event.is_set(): return ProviderResult() - # Process raw data into the application's expected format - current_processed_certs = [self._extract_certificate_metadata(cert) for cert in raw_certificates] - + # Combine with old data if cache is stale if cache_status == "stale": - # Load existing and append new processed certs - existing_result = self._load_from_cache(cache_file) - result = self._merge_results(existing_result, current_processed_certs, domain) - self.logger.logger.info(f"Refreshed and merged cache for {domain}") + old_raw_certs = self._load_raw_data_from_cache(cache_file) + combined_certs = old_raw_certs + new_raw_certs + + # Deduplicate the combined list + seen_ids = set() + unique_certs = [] + for cert in combined_certs: + cert_id = cert.get('id') + if cert_id not in seen_ids: + unique_certs.append(cert) + seen_ids.add(cert_id) + + raw_certificates_to_process = unique_certs + self.logger.logger.info(f"Refreshed and merged cache for {domain}. Total unique certs: {len(raw_certificates_to_process)}") else: # "not_found" - # Create new result from processed certs - result = self._process_certificates_to_result(domain, raw_certificates) - self.logger.logger.info(f"Created fresh result for {domain} ({result.get_relationship_count()} relationships)") + raw_certificates_to_process = new_raw_certs + + # Process the clean, deduplicated list of certificates + result = self._process_certificates_to_result(domain, raw_certificates_to_process) + self.logger.logger.info(f"Created fresh result for {domain} ({result.get_relationship_count()} relationships)") - # Save the result to cache - self._save_result_to_cache(cache_file, result, domain) + # Save the new result and the raw data to the cache + self._save_result_to_cache(cache_file, result, raw_certificates_to_process, domain) except requests.exceptions.RequestException as e: self.logger.logger.error(f"API query failed for {domain}: {e}") @@ -200,12 +210,22 @@ class CrtShProvider(BaseProvider): self.logger.logger.error(f"Failed to load cached certificates from {cache_file_path}: {e}") return ProviderResult() - def _save_result_to_cache(self, cache_file_path: Path, result: ProviderResult, domain: str) -> None: - """Save processed crt.sh result to a cache file.""" + def _load_raw_data_from_cache(self, cache_file_path: Path) -> List[Dict[str, Any]]: + """Load only the raw certificate data from a cache file.""" + try: + with open(cache_file_path, 'r') as f: + cache_content = json.load(f) + return cache_content.get("raw_certificates", []) + except (json.JSONDecodeError, FileNotFoundError): + return [] + + def _save_result_to_cache(self, cache_file_path: Path, result: ProviderResult, raw_certificates: List[Dict[str, Any]], domain: str) -> None: + """Save processed crt.sh result and raw data to a cache file.""" try: cache_data = { "domain": domain, "last_upstream_query": datetime.now(timezone.utc).isoformat(), + "raw_certificates": raw_certificates, # Store the raw data for deduplication "relationships": [ { "source_node": rel.source_node, @@ -234,25 +254,6 @@ class CrtShProvider(BaseProvider): except Exception as e: self.logger.logger.warning(f"Failed to save cache file for {domain}: {e}") - def _merge_results(self, existing_result: ProviderResult, new_certificates: List[Dict[str, Any]], domain: str) -> ProviderResult: - """Merge new certificate data with existing cached result.""" - # Create a fresh result from the new certificates - new_result = self._process_certificates_to_result(domain, new_certificates) - - # Simple merge strategy: combine all relationships and attributes - # In practice, you might want more sophisticated deduplication - merged_result = ProviderResult() - - # Add existing relationships and attributes - merged_result.relationships.extend(existing_result.relationships) - merged_result.attributes.extend(existing_result.attributes) - - # Add new relationships and attributes - merged_result.relationships.extend(new_result.relationships) - merged_result.attributes.extend(new_result.attributes) - - return merged_result - def _query_crtsh_api(self, domain: str) -> List[Dict[str, Any]]: """Query crt.sh API for raw certificate data.""" url = f"{self.base_url}?q={quote(domain)}&output=json" @@ -261,7 +262,12 @@ class CrtShProvider(BaseProvider): if not response or response.status_code != 200: raise requests.exceptions.RequestException(f"crt.sh API returned status {response.status_code if response else 'None'}") - certificates = response.json() + try: + certificates = response.json() + except json.JSONDecodeError: + self.logger.logger.error(f"crt.sh returned invalid JSON for {domain}") + return [] + if not certificates: return [] @@ -324,7 +330,7 @@ class CrtShProvider(BaseProvider): result.add_relationship( source_node=domain, target_node=discovered_domain, - relationship_type='san_certificate', + relationship_type='crtsh_san_certificate', provider=self.name, confidence=confidence, raw_data={'relationship_type': 'certificate_discovery'} @@ -333,7 +339,7 @@ class CrtShProvider(BaseProvider): self.log_relationship_discovery( source_node=domain, target_node=discovered_domain, - relationship_type='san_certificate', + relationship_type='crtsh_san_certificate', confidence_score=confidence, raw_data={'relationship_type': 'certificate_discovery'}, discovery_method="certificate_transparency_analysis" diff --git a/providers/dns_provider.py b/providers/dns_provider.py index 6a855d9..2d03a9b 100644 --- a/providers/dns_provider.py +++ b/providers/dns_provider.py @@ -107,7 +107,7 @@ class DNSProvider(BaseProvider): result.add_relationship( source_node=ip, target_node=hostname, - relationship_type='ptr_record', + relationship_type='dns_ptr_record', provider=self.name, confidence=0.8, raw_data={ @@ -125,7 +125,7 @@ class DNSProvider(BaseProvider): self.log_relationship_discovery( source_node=ip, target_node=hostname, - relationship_type='ptr_record', + relationship_type='dns_ptr_record', confidence_score=0.8, raw_data={ 'query_type': 'PTR', @@ -202,7 +202,7 @@ class DNSProvider(BaseProvider): 'value': target, 'ttl': response.ttl } - relationship_type = f"{record_type.lower()}_record" + relationship_type = f"dns_{record_type.lower()}_record" confidence = 0.8 # Add relationship diff --git a/providers/shodan_provider.py b/providers/shodan_provider.py index 00d0cea..0e7004c 100644 --- a/providers/shodan_provider.py +++ b/providers/shodan_provider.py @@ -222,7 +222,7 @@ class ShodanProvider(BaseProvider): result.add_relationship( source_node=ip, target_node=hostname, - relationship_type='a_record', + relationship_type='shodan_a_record', provider=self.name, confidence=0.8, raw_data=data @@ -230,7 +230,7 @@ class ShodanProvider(BaseProvider): self.log_relationship_discovery( source_node=ip, target_node=hostname, - relationship_type='a_record', + relationship_type='shodan_a_record', confidence_score=0.8, raw_data=data, discovery_method="shodan_host_lookup" @@ -240,7 +240,7 @@ class ShodanProvider(BaseProvider): result.add_relationship( source_node=ip, target_node=asn_name, - relationship_type='asn_membership', + relationship_type='shodan_asn_membership', provider=self.name, confidence=0.7, raw_data=data @@ -248,7 +248,7 @@ class ShodanProvider(BaseProvider): self.log_relationship_discovery( source_node=ip, target_node=asn_name, - relationship_type='asn_membership', + relationship_type='shodan_asn_membership', confidence_score=0.7, raw_data=data, discovery_method="shodan_asn_lookup" @@ -257,9 +257,9 @@ class ShodanProvider(BaseProvider): for port in value: result.add_attribute( target_node=ip, - name='open_port', + name='shodan_open_port', value=port, - attr_type='network_info', + attr_type='shodan_network_info', provider=self.name, confidence=0.9 ) From d0ee415f0dabd271c9054a30e8e109fc37f53239 Mon Sep 17 00:00:00 2001 From: overcuriousity Date: Wed, 17 Sep 2025 19:42:14 +0200 Subject: [PATCH 11/20] enhancements --- app.py | 311 ++++++--------------------------------- core/graph_manager.py | 2 +- core/scanner.py | 329 +++++++++++++----------------------------- 3 files changed, 149 insertions(+), 493 deletions(-) diff --git a/app.py b/app.py index d21e77a..942277b 100644 --- a/app.py +++ b/app.py @@ -18,38 +18,29 @@ from utils.helpers import is_valid_target app = Flask(__name__) -# Use centralized configuration for Flask settings app.config['SECRET_KEY'] = config.flask_secret_key app.config['PERMANENT_SESSION_LIFETIME'] = timedelta(hours=config.flask_permanent_session_lifetime_hours) def get_user_scanner(): """ - Retrieves the scanner for the current session, or creates a new session only if none exists. + Retrieves the scanner for the current session, or creates a new one if none exists. """ current_flask_session_id = session.get('dnsrecon_session_id') - # Try to get existing session first if current_flask_session_id: existing_scanner = session_manager.get_session(current_flask_session_id) if existing_scanner: - #print(f"Reusing existing session: {current_flask_session_id}") return current_flask_session_id, existing_scanner - else: - print(f"Session {current_flask_session_id} expired, will create new one") - # Only create new session if we absolutely don't have one - print("Creating new session (no valid session found)") new_session_id = session_manager.create_session() new_scanner = session_manager.get_session(new_session_id) if not new_scanner: raise Exception("Failed to create new scanner session") - # Store in Flask session session['dnsrecon_session_id'] = new_session_id session.permanent = True - print(f"Created new session: {new_session_id}") return new_session_id, new_scanner @app.route('/') @@ -61,11 +52,8 @@ def index(): @app.route('/api/scan/start', methods=['POST']) def start_scan(): """ - FIXED: Start a new reconnaissance scan while preserving session configuration. - Only clears graph data, not the entire session with API keys. + Starts a new reconnaissance scan. """ - print("=== API: /api/scan/start called ===") - try: data = request.get_json() if not data or 'target' not in data: @@ -76,25 +64,18 @@ def start_scan(): clear_graph = data.get('clear_graph', True) force_rescan_target = data.get('force_rescan_target', None) - print(f"Parsed - target: '{target}', max_depth: {max_depth}, clear_graph: {clear_graph}, force_rescan: {force_rescan_target}") - - # Validation if not target: return jsonify({'success': False, 'error': 'Target cannot be empty'}), 400 if not is_valid_target(target): - return jsonify({'success': False, 'error': 'Invalid target format. Please enter a valid domain or IP address.'}), 400 + return jsonify({'success': False, 'error': 'Invalid target format.'}), 400 if not isinstance(max_depth, int) or not 1 <= max_depth <= 5: return jsonify({'success': False, 'error': 'Max depth must be an integer between 1 and 5'}), 400 - # FIXED: Always reuse existing session, preserve API keys user_session_id, scanner = get_user_scanner() if not scanner: return jsonify({'success': False, 'error': 'Failed to get scanner instance.'}), 500 - print(f"Using scanner {id(scanner)} in session {user_session_id}") - - # FIXED: Pass clear_graph flag to scanner, let it handle graph clearing internally success = scanner.start_scan(target, max_depth, clear_graph=clear_graph, force_rescan_target=force_rescan_target) if success: @@ -102,8 +83,7 @@ def start_scan(): 'success': True, 'message': 'Scan started successfully', 'scan_id': scanner.logger.session_id, - 'user_session_id': user_session_id, - 'available_providers': [p.get_name() for p in scanner.providers] # Show which providers are active + 'user_session_id': user_session_id }) else: return jsonify({ @@ -112,206 +92,98 @@ def start_scan(): }), 409 except Exception as e: - print(f"ERROR: Exception in start_scan endpoint: {e}") traceback.print_exc() return jsonify({'success': False, 'error': f'Internal server error: {str(e)}'}), 500 @app.route('/api/scan/stop', methods=['POST']) def stop_scan(): - """Stop the current scan with immediate GUI feedback.""" - print("=== API: /api/scan/stop called ===") - + """Stop the current scan.""" try: - # Get user-specific scanner user_session_id, scanner = get_user_scanner() - print(f"Stopping scan for session: {user_session_id}") if not scanner: - return jsonify({ - 'success': False, - 'error': 'No scanner found for session' - }), 404 + return jsonify({'success': False, 'error': 'No scanner found for session'}), 404 - # Ensure session ID is set if not scanner.session_id: scanner.session_id = user_session_id - # Use the stop mechanism - success = scanner.stop_scan() - - # Also set the Redis stop signal directly for extra reliability + scanner.stop_scan() session_manager.set_stop_signal(user_session_id) - - # Force immediate status update session_manager.update_scanner_status(user_session_id, 'stopped') - - # Update the full scanner state session_manager.update_session_scanner(user_session_id, scanner) - print(f"Stop scan completed. Success: {success}, Scanner status: {scanner.status}") - return jsonify({ 'success': True, - 'message': 'Scan stop requested - termination initiated', - 'user_session_id': user_session_id, - 'scanner_status': scanner.status, - 'stop_method': 'cross_process' + 'message': 'Scan stop requested', + 'user_session_id': user_session_id }) except Exception as e: - print(f"ERROR: Exception in stop_scan endpoint: {e}") traceback.print_exc() - return jsonify({ - 'success': False, - 'error': f'Internal server error: {str(e)}' - }), 500 + return jsonify({'success': False, 'error': f'Internal server error: {str(e)}'}), 500 @app.route('/api/scan/status', methods=['GET']) def get_scan_status(): - """Get current scan status with error handling.""" + """Get current scan status.""" try: - # Get user-specific scanner user_session_id, scanner = get_user_scanner() if not scanner: - # Return default idle status if no scanner return jsonify({ 'success': True, 'status': { - 'status': 'idle', - 'target_domain': None, - 'current_depth': 0, - 'max_depth': 0, - 'current_indicator': '', - 'total_indicators_found': 0, - 'indicators_processed': 0, - 'progress_percentage': 0.0, - 'enabled_providers': [], - 'graph_statistics': {}, + 'status': 'idle', 'target_domain': None, 'current_depth': 0, + 'max_depth': 0, 'progress_percentage': 0.0, 'user_session_id': user_session_id } }) - # Ensure session ID is set if not scanner.session_id: scanner.session_id = user_session_id status = scanner.get_scan_status() status['user_session_id'] = user_session_id - # Additional debug info - status['debug_info'] = { - 'scanner_object_id': id(scanner), - 'session_id_set': bool(scanner.session_id), - 'has_scan_thread': bool(scanner.scan_thread and scanner.scan_thread.is_alive()), - 'provider_count': len(scanner.providers), - 'provider_names': [p.get_name() for p in scanner.providers] - } - - return jsonify({ - 'success': True, - 'status': status - }) + return jsonify({'success': True, 'status': status}) except Exception as e: - print(f"ERROR: Exception in get_scan_status endpoint: {e}") traceback.print_exc() return jsonify({ - 'success': False, - 'error': f'Internal server error: {str(e)}', - 'fallback_status': { - 'status': 'error', - 'target_domain': None, - 'current_depth': 0, - 'max_depth': 0, - 'progress_percentage': 0.0 - } + 'success': False, 'error': f'Internal server error: {str(e)}', + 'fallback_status': {'status': 'error', 'progress_percentage': 0.0} }), 500 @app.route('/api/graph', methods=['GET']) def get_graph_data(): - """Get current graph data with error handling and proper empty graph structure.""" + """Get current graph data.""" try: - # Get user-specific scanner user_session_id, scanner = get_user_scanner() + empty_graph = { + 'nodes': [], 'edges': [], + 'statistics': {'node_count': 0, 'edge_count': 0} + } + if not scanner: - # FIXED: Return proper empty graph structure instead of None - empty_graph = { - 'nodes': [], - 'edges': [], - 'statistics': { - 'node_count': 0, - 'edge_count': 0, - 'creation_time': datetime.now(timezone.utc).isoformat(), - 'last_modified': datetime.now(timezone.utc).isoformat() - } - } - return jsonify({ - 'success': True, - 'graph': empty_graph, - 'user_session_id': user_session_id - }) + return jsonify({'success': True, 'graph': empty_graph, 'user_session_id': user_session_id}) - graph_data = scanner.get_graph_data() + graph_data = scanner.get_graph_data() or empty_graph - if not graph_data: - graph_data = { - 'nodes': [], - 'edges': [], - 'statistics': { - 'node_count': 0, - 'edge_count': 0, - 'creation_time': datetime.now(timezone.utc).isoformat(), - 'last_modified': datetime.now(timezone.utc).isoformat() - } - } - - # FIXED: Ensure required fields exist - if 'nodes' not in graph_data: - graph_data['nodes'] = [] - if 'edges' not in graph_data: - graph_data['edges'] = [] - if 'statistics' not in graph_data: - graph_data['statistics'] = { - 'node_count': len(graph_data['nodes']), - 'edge_count': len(graph_data['edges']), - 'creation_time': datetime.now(timezone.utc).isoformat(), - 'last_modified': datetime.now(timezone.utc).isoformat() - } - - return jsonify({ - 'success': True, - 'graph': graph_data, - 'user_session_id': user_session_id - }) + return jsonify({'success': True, 'graph': graph_data, 'user_session_id': user_session_id}) except Exception as e: - print(f"ERROR: Exception in get_graph_data endpoint: {e}") traceback.print_exc() - - # FIXED: Return proper error structure with empty graph fallback return jsonify({ - 'success': False, - 'error': f'Internal server error: {str(e)}', - 'fallback_graph': { - 'nodes': [], - 'edges': [], - 'statistics': { - 'node_count': 0, - 'edge_count': 0, - 'creation_time': datetime.now(timezone.utc).isoformat(), - 'last_modified': datetime.now(timezone.utc).isoformat() - } - } + 'success': False, 'error': f'Internal server error: {str(e)}', + 'fallback_graph': {'nodes': [], 'edges': [], 'statistics': {}} }), 500 @app.route('/api/graph/large-entity/extract', methods=['POST']) def extract_from_large_entity(): - """Extract a node from a large entity, making it a standalone node.""" + """Extract a node from a large entity.""" try: data = request.get_json() large_entity_id = data.get('large_entity_id') @@ -333,13 +205,12 @@ def extract_from_large_entity(): return jsonify({'success': False, 'error': f'Failed to extract node {node_id}.'}), 500 except Exception as e: - print(f"ERROR: Exception in extract_from_large_entity endpoint: {e}") traceback.print_exc() return jsonify({'success': False, 'error': f'Internal server error: {str(e)}'}), 500 @app.route('/api/graph/node/', methods=['DELETE']) def delete_graph_node(node_id): - """Delete a node from the graph for the current user session.""" + """Delete a node from the graph.""" try: user_session_id, scanner = get_user_scanner() if not scanner: @@ -348,14 +219,12 @@ def delete_graph_node(node_id): success = scanner.graph.remove_node(node_id) if success: - # Persist the change session_manager.update_session_scanner(user_session_id, scanner) return jsonify({'success': True, 'message': f'Node {node_id} deleted successfully.'}) else: - return jsonify({'success': False, 'error': f'Node {node_id} not found in graph.'}), 404 + return jsonify({'success': False, 'error': f'Node {node_id} not found.'}), 404 except Exception as e: - print(f"ERROR: Exception in delete_graph_node endpoint: {e}") traceback.print_exc() return jsonify({'success': False, 'error': f'Internal server error: {str(e)}'}), 500 @@ -376,7 +245,6 @@ def revert_graph_action(): action_data = data['data'] if action_type == 'delete': - # Re-add the node node_to_add = action_data.get('node') if node_to_add: scanner.graph.add_node( @@ -387,204 +255,121 @@ def revert_graph_action(): metadata=node_to_add.get('metadata') ) - # Re-add the edges edges_to_add = action_data.get('edges', []) for edge in edges_to_add: - # Add edge only if both nodes exist to prevent errors if scanner.graph.graph.has_node(edge['from']) and scanner.graph.graph.has_node(edge['to']): scanner.graph.add_edge( - source_id=edge['from'], - target_id=edge['to'], + source_id=edge['from'], target_id=edge['to'], relationship_type=edge['metadata']['relationship_type'], confidence_score=edge['metadata']['confidence_score'], source_provider=edge['metadata']['source_provider'], raw_data=edge.get('raw_data', {}) ) - # Persist the change session_manager.update_session_scanner(user_session_id, scanner) return jsonify({'success': True, 'message': 'Delete action reverted successfully.'}) return jsonify({'success': False, 'error': f'Unknown revert action type: {action_type}'}), 400 except Exception as e: - print(f"ERROR: Exception in revert_graph_action endpoint: {e}") traceback.print_exc() return jsonify({'success': False, 'error': f'Internal server error: {str(e)}'}), 500 @app.route('/api/export', methods=['GET']) def export_results(): - """Export complete scan results as downloadable JSON for the user session.""" + """Export scan results as a JSON file.""" try: - # Get user-specific scanner user_session_id, scanner = get_user_scanner() - - # Get complete results results = scanner.export_results() - # Add session information to export results['export_metadata'] = { 'user_session_id': user_session_id, 'export_timestamp': datetime.now(timezone.utc).isoformat(), - 'export_type': 'user_session_results' } - # Create filename with timestamp timestamp = datetime.now(timezone.utc).strftime('%Y%m%d_%H%M%S') target = scanner.current_target or 'unknown' - filename = f"dnsrecon_{target}_{timestamp}_{user_session_id[:8]}.json" + filename = f"dnsrecon_{target}_{timestamp}.json" - # Create in-memory file - json_data = json.dumps(results, indent=2, ensure_ascii=False) + json_data = json.dumps(results, indent=2) file_obj = io.BytesIO(json_data.encode('utf-8')) return send_file( - file_obj, - as_attachment=True, - download_name=filename, - mimetype='application/json' + file_obj, as_attachment=True, + download_name=filename, mimetype='application/json' ) except Exception as e: - print(f"ERROR: Exception in export_results endpoint: {e}") traceback.print_exc() - return jsonify({ - 'success': False, - 'error': f'Export failed: {str(e)}' - }), 500 + return jsonify({'success': False, 'error': f'Export failed: {str(e)}'}), 500 @app.route('/api/providers', methods=['GET']) def get_providers(): - """Get information about available providers for the user session.""" - + """Get information about available providers.""" try: - # Get user-specific scanner user_session_id, scanner = get_user_scanner() - - if scanner and scanner.status == 'running': - status = scanner.get_scan_status() - currently_processing = status.get('currently_processing') - if currently_processing: - provider_name, target_item = currently_processing[0] - print(f"DEBUG: RUNNING Task - Provider: {provider_name}, Target: {target_item}") - - print(f"DEBUG: Task Queue Status - In Queue: {status.get('tasks_in_queue', 0)}, Completed: {status.get('tasks_completed', 0)}, Skipped: {status.get('tasks_skipped', 0)}, Rescheduled: {status.get('tasks_rescheduled', 0)}") - elif not scanner: - print("DEBUG: No active scanner session found.") - provider_info = scanner.get_provider_info() - return jsonify({ - 'success': True, - 'providers': provider_info, - 'user_session_id': user_session_id - }) + return jsonify({'success': True, 'providers': provider_info, 'user_session_id': user_session_id}) except Exception as e: - print(f"ERROR: Exception in get_providers endpoint: {e}") traceback.print_exc() - return jsonify({ - 'success': False, - 'error': f'Internal server error: {str(e)}' - }), 500 + return jsonify({'success': False, 'error': f'Internal server error: {str(e)}'}), 500 @app.route('/api/config/api-keys', methods=['POST']) def set_api_keys(): - """ - """ + """Set API keys for the current session.""" try: data = request.get_json() - if data is None: - return jsonify({ - 'success': False, - 'error': 'No API keys provided' - }), 400 + return jsonify({'success': False, 'error': 'No API keys provided'}), 400 - # Get user-specific scanner and config user_session_id, scanner = get_user_scanner() session_config = scanner.config - print(f"Setting API keys for session {user_session_id}: {list(data.keys())}") - updated_providers = [] - # Iterate over the API keys provided in the request data for provider_name, api_key in data.items(): - # This allows us to both set and clear keys. The config - # handles enabling/disabling based on if the key is empty. api_key_value = str(api_key or '').strip() success = session_config.set_api_key(provider_name.lower(), api_key_value) if success: updated_providers.append(provider_name) - print(f"API key {'set' if api_key_value else 'cleared'} for {provider_name}") if updated_providers: - # FIXED: Reinitialize scanner providers to apply the new keys - print("Reinitializing providers with new API keys...") - old_provider_count = len(scanner.providers) scanner._initialize_providers() - new_provider_count = len(scanner.providers) - - print(f"Providers reinitialized: {old_provider_count} -> {new_provider_count}") - print(f"Available providers: {[p.get_name() for p in scanner.providers]}") - - # Persist the updated scanner object back to the user's session session_manager.update_session_scanner(user_session_id, scanner) return jsonify({ 'success': True, - 'message': f'API keys updated for session {user_session_id}: {", ".join(updated_providers)}', - 'updated_providers': updated_providers, - 'user_session_id': user_session_id, - 'available_providers': [p.get_name() for p in scanner.providers] + 'message': f'API keys updated for: {", ".join(updated_providers)}', + 'user_session_id': user_session_id }) else: - return jsonify({ - 'success': False, - 'error': 'No valid API keys were provided or provider names were incorrect.' - }), 400 + return jsonify({'success': False, 'error': 'No valid API keys were provided.'}), 400 except Exception as e: - print(f"ERROR: Exception in set_api_keys endpoint: {e}") traceback.print_exc() - return jsonify({ - 'success': False, - 'error': f'Internal server error: {str(e)}' - }), 500 + return jsonify({'success': False, 'error': f'Internal server error: {str(e)}'}), 500 @app.errorhandler(404) def not_found(error): """Handle 404 errors.""" - return jsonify({ - 'success': False, - 'error': 'Endpoint not found' - }), 404 + return jsonify({'success': False, 'error': 'Endpoint not found'}), 404 @app.errorhandler(500) def internal_error(error): """Handle 500 errors.""" - print(f"ERROR: 500 Internal Server Error: {error}") traceback.print_exc() - return jsonify({ - 'success': False, - 'error': 'Internal server error' - }), 500 + return jsonify({'success': False, 'error': 'Internal server error'}), 500 if __name__ == '__main__': - print("Starting DNSRecon Flask application with streamlined session management...") - - # Load configuration from environment config.load_from_env() - - # Start Flask application - print(f"Starting server on {config.flask_host}:{config.flask_port}") app.run( host=config.flask_host, port=config.flask_port, diff --git a/core/graph_manager.py b/core/graph_manager.py index 5a55052..3a904f9 100644 --- a/core/graph_manager.py +++ b/core/graph_manager.py @@ -149,7 +149,7 @@ class GraphManager: if self.graph.has_node(node_id) and not self.graph.has_edge(node_id, correlation_node_id): # Format relationship label as "corr_provider_attribute" - relationship_label = f"corr_{provider}_{attribute}" + relationship_label = f"{provider}_{attribute}" self.add_edge( source_id=node_id, diff --git a/core/scanner.py b/core/scanner.py index bc5f23a..dcd86a2 100644 --- a/core/scanner.py +++ b/core/scanner.py @@ -5,6 +5,7 @@ import traceback import os import importlib import redis +import time from typing import List, Set, Dict, Any, Tuple, Optional from concurrent.futures import ThreadPoolExecutor from collections import defaultdict @@ -30,13 +31,11 @@ class ScanStatus: class Scanner: """ Main scanning orchestrator for DNSRecon passive reconnaissance. - FIXED: Now preserves session configuration including API keys when clearing graphs. + UNIFIED: Combines comprehensive features with improved display formatting. """ def __init__(self, session_config=None): """Initialize scanner with session-specific configuration.""" - print("Initializing Scanner instance...") - try: # Use provided session config or create default if session_config is None: @@ -57,15 +56,18 @@ class Scanner: self.target_retries = defaultdict(int) self.scan_failed_due_to_retries = False - # **NEW**: Track currently processing tasks to prevent processing after stop + # Thread-safe processing tracking (from Document 1) self.currently_processing = set() self.processing_lock = threading.Lock() + # Display-friendly processing list (from Document 2) + self.currently_processing_display = [] # Scanning progress tracking self.total_indicators_found = 0 self.indicators_processed = 0 self.indicators_completed = 0 self.tasks_re_enqueued = 0 + self.tasks_skipped = 0 # BUGFIX: Initialize tasks_skipped self.total_tasks_ever_enqueued = 0 self.current_indicator = "" @@ -73,19 +75,19 @@ class Scanner: self.max_workers = self.config.max_concurrent_requests self.executor = None + # Status logger thread with improved formatting + self.status_logger_thread = None + self.status_logger_stop_event = threading.Event() + # Initialize providers with session config - print("Calling _initialize_providers with session config...") self._initialize_providers() # Initialize logger - print("Initializing forensic logger...") self.logger = get_forensic_logger() # Initialize global rate limiter self.rate_limiter = GlobalRateLimiter(redis.StrictRedis(db=0)) - print("Scanner initialization complete") - except Exception as e: print(f"ERROR: Scanner initialization failed: {e}") traceback.print_exc() @@ -96,17 +98,14 @@ class Scanner: Check if stop is requested using both local and Redis-based signals. This ensures reliable termination across process boundaries. """ - # Check local threading event first (fastest) if self.stop_event.is_set(): return True - # Check Redis-based stop signal if session ID is available if self.session_id: try: from core.session_manager import session_manager return session_manager.is_stop_requested(self.session_id) except Exception as e: - print(f"Error checking Redis stop signal: {e}") # Fall back to local event return self.stop_event.is_set() @@ -116,22 +115,19 @@ class Scanner: """ Set stop signal both locally and in Redis. """ - # Set local event self.stop_event.set() - # Set Redis signal if session ID is available if self.session_id: try: from core.session_manager import session_manager session_manager.set_stop_signal(self.session_id) except Exception as e: - print(f"Error setting Redis stop signal: {e}") + pass def __getstate__(self): """Prepare object for pickling by excluding unpicklable attributes.""" state = self.__dict__.copy() - # Remove unpicklable threading objects unpicklable_attrs = [ 'stop_event', 'scan_thread', @@ -139,14 +135,15 @@ class Scanner: 'processing_lock', 'task_queue', 'rate_limiter', - 'logger' + 'logger', + 'status_logger_thread', + 'status_logger_stop_event' ] for attr in unpicklable_attrs: if attr in state: del state[attr] - # Handle providers separately to ensure they're picklable if 'providers' in state: for provider in state['providers']: if hasattr(provider, '_stop_event'): @@ -158,7 +155,6 @@ class Scanner: """Restore object after unpickling by reconstructing threading objects.""" self.__dict__.update(state) - # Reconstruct threading objects self.stop_event = threading.Event() self.scan_thread = None self.executor = None @@ -166,15 +162,18 @@ class Scanner: self.task_queue = PriorityQueue() self.rate_limiter = GlobalRateLimiter(redis.StrictRedis(db=0)) self.logger = get_forensic_logger() + self.status_logger_thread = None + self.status_logger_stop_event = threading.Event() if not hasattr(self, 'providers') or not self.providers: - print("Providers not found after loading session, re-initializing...") self._initialize_providers() if not hasattr(self, 'currently_processing'): self.currently_processing = set() - # Re-set stop events for providers + if not hasattr(self, 'currently_processing_display'): + self.currently_processing_display = [] + if hasattr(self, 'providers'): for provider in self.providers: if hasattr(provider, 'set_stop_event'): @@ -183,8 +182,6 @@ class Scanner: def _initialize_providers(self) -> None: """Initialize all available providers based on session configuration.""" self.providers = [] - print("Initializing providers with session config...") - provider_dir = os.path.join(os.path.dirname(__file__), '..', 'providers') for filename in os.listdir(provider_dir): if filename.endswith('_provider.py') and not filename.startswith('base'): @@ -202,99 +199,98 @@ class Scanner: if provider.is_available(): provider.set_stop_event(self.stop_event) self.providers.append(provider) - print(f"✓ {provider.get_display_name()} provider initialized successfully for session") - else: - print(f"✗ {provider.get_display_name()} provider is not available") except Exception as e: - print(f"✗ Failed to initialize provider from {filename}: {e}") traceback.print_exc() - print(f"Initialized {len(self.providers)} providers for session") + def _status_logger_thread(self): + """Periodically prints a clean, formatted scan status to the terminal.""" + # Color codes for improved display (from Document 2) + HEADER = "\033[95m" + CYAN = "\033[96m" + GREEN = "\033[92m" + YELLOW = "\033[93m" + ENDC = "\033[0m" + BOLD = "\033[1m" + + last_status_str = "" + while not self.status_logger_stop_event.is_set(): + try: + # Use thread-safe copy of currently processing + with self.processing_lock: + in_flight_tasks = list(self.currently_processing) + # Update display list for consistent formatting + self.currently_processing_display = in_flight_tasks.copy() - def update_session_config(self, new_config) -> None: - """Update session configuration and reinitialize providers.""" - print("Updating session configuration...") - self.config = new_config - self.max_workers = self.config.max_concurrent_requests - self._initialize_providers() - print("Session configuration updated") + status_str = ( + f"{BOLD}{HEADER}Scan Status: {self.status.upper()}{ENDC} | " + f"{CYAN}Queued: {self.task_queue.qsize()}{ENDC} | " + f"{YELLOW}In-Flight: {len(in_flight_tasks)}{ENDC} | " + f"{GREEN}Completed: {self.indicators_completed}{ENDC} | " + f"Skipped: {self.tasks_skipped} | " + f"Rescheduled: {self.tasks_re_enqueued}" + ) + + if status_str != last_status_str: + print(f"\n{'-'*80}") + print(status_str) + if in_flight_tasks: + print(f"{BOLD}{YELLOW}Currently Processing:{ENDC}") + # Display up to 3 currently processing tasks + display_tasks = [f" - {p}: {t}" for p, t in in_flight_tasks[:3]] + print("\n".join(display_tasks)) + if len(in_flight_tasks) > 3: + print(f" ... and {len(in_flight_tasks) - 3} more") + print(f"{'-'*80}") + last_status_str = status_str + + except Exception: + # Silently fail to avoid crashing the logger + pass + + time.sleep(2) # Update interval def start_scan(self, target: str, max_depth: int = 2, clear_graph: bool = True, force_rescan_target: Optional[str] = None) -> bool: """ - FIXED: Start a new reconnaissance scan preserving session configuration. - Only clears graph data when requested, never destroys session/API keys. + Starts a new reconnaissance scan. """ - print(f"=== STARTING SCAN IN SCANNER {id(self)} ===") - print(f"Session ID: {self.session_id}") - print(f"Initial scanner status: {self.status}") - print(f"Clear graph requested: {clear_graph}") - print(f"Current providers: {[p.get_name() for p in self.providers]}") - self.total_tasks_ever_enqueued = 0 - - # FIXED: Improved cleanup of previous scan without destroying session config if self.scan_thread and self.scan_thread.is_alive(): - print("A previous scan thread is still alive. Forcing termination...") - - # Set stop signals immediately self._set_stop_signal() self.status = ScanStatus.STOPPED - - # Clear all processing state with self.processing_lock: self.currently_processing.clear() + self.currently_processing_display = [] self.task_queue = PriorityQueue() - - # Shutdown executor aggressively if self.executor: - print("Shutting down executor forcefully...") self.executor.shutdown(wait=False, cancel_futures=True) self.executor = None - - # Wait for thread termination with shorter timeout - print("Waiting for previous scan thread to terminate...") - self.scan_thread.join(5.0) # Reduced from 10 seconds - - if self.scan_thread.is_alive(): - print("WARNING: Previous scan thread is still alive after 5 seconds") - self.logger.logger.warning("Previous scan thread failed to terminate cleanly") + self.scan_thread.join(5.0) - # FIXED: Reset scan state but preserve session configuration (API keys, etc.) - print("Resetting scanner state for new scan (preserving session config)...") self.status = ScanStatus.IDLE self.stop_event.clear() - # Clear Redis stop signal explicitly if self.session_id: from core.session_manager import session_manager session_manager.clear_stop_signal(self.session_id) with self.processing_lock: self.currently_processing.clear() + self.currently_processing_display = [] - # Reset scan-specific state but keep providers and config intact self.task_queue = PriorityQueue() self.target_retries.clear() self.scan_failed_due_to_retries = False + self.tasks_skipped = 0 # BUGFIX: Reset tasks_skipped for new scan - # Update session state immediately for GUI feedback self._update_session_state() - print("Scanner state reset complete (providers preserved).") try: if not hasattr(self, 'providers') or not self.providers: - print(f"ERROR: No providers available in scanner {id(self)}, cannot start scan") return False - print(f"Scanner {id(self)} validation passed, providers available: {[p.get_name() for p in self.providers]}") - - # FIXED: Only clear graph if explicitly requested, don't destroy session if clear_graph: - print("Clearing graph data (preserving session configuration)") self.graph.clear() - # Handle force rescan by clearing provider states for that specific node if force_rescan_target and self.graph.graph.has_node(force_rescan_target): - print(f"Forcing rescan of {force_rescan_target}, clearing provider states.") node_data = self.graph.graph.nodes[force_rescan_target] if 'metadata' in node_data and 'provider_states' in node_data['metadata']: node_data['metadata']['provider_states'] = {} @@ -307,17 +303,12 @@ class Scanner: self.indicators_processed = 0 self.indicators_completed = 0 self.tasks_re_enqueued = 0 + self.total_tasks_ever_enqueued = 0 self.current_indicator = self.current_target - # Update GUI with scan preparation state self._update_session_state() - - # Start new forensic session (but don't reinitialize providers) - print(f"Starting new forensic session for scanner {id(self)}...") self.logger = new_session() - # Start scan in a separate thread - print(f"Starting scan thread for scanner {id(self)}...") self.scan_thread = threading.Thread( target=self._execute_scan, args=(self.current_target, max_depth), @@ -325,12 +316,14 @@ class Scanner: ) self.scan_thread.start() - print(f"=== SCAN STARTED SUCCESSFULLY IN SCANNER {id(self)} ===") - print(f"Active providers for this scan: {[p.get_name() for p in self.providers]}") + # Start the status logger thread + self.status_logger_stop_event.clear() + self.status_logger_thread = threading.Thread(target=self._status_logger_thread, daemon=True) + self.status_logger_thread.start() + return True except Exception as e: - print(f"ERROR: Exception in start_scan for scanner {id(self)}: {e}") traceback.print_exc() self.status = ScanStatus.FAILED self._update_session_state() @@ -347,11 +340,9 @@ class Scanner: def _execute_scan(self, target: str, max_depth: int) -> None: """Execute the reconnaissance scan with proper termination handling.""" - print(f"_execute_scan started for {target} with depth {max_depth}") self.executor = ThreadPoolExecutor(max_workers=self.max_workers) processed_tasks = set() - # Initial task population for the main target is_ip = _is_valid_ip(target) initial_providers = self._get_eligible_providers(target, is_ip, False) for provider in initial_providers: @@ -366,23 +357,19 @@ class Scanner: enabled_providers = [provider.get_name() for provider in self.providers] self.logger.log_scan_start(target, max_depth, enabled_providers) - # Determine initial node type node_type = NodeType.IP if is_ip else NodeType.DOMAIN self.graph.add_node(target, node_type) - self._initialize_provider_states(target) - # Better termination checking in main loop while not self.task_queue.empty() and not self._is_stop_requested(): try: priority, (provider_name, target_item, depth) = self.task_queue.get() except IndexError: - # Queue became empty during processing break task_tuple = (provider_name, target_item) if task_tuple in processed_tasks: - self.indicators_completed += 1 + self.tasks_skipped += 1 continue if depth > max_depth: @@ -394,7 +381,6 @@ class Scanner: with self.processing_lock: if self._is_stop_requested(): - print(f"Stop requested before processing {target_item}") break self.currently_processing.add(task_tuple) @@ -404,7 +390,6 @@ class Scanner: self._update_session_state() if self._is_stop_requested(): - print(f"Stop requested during processing setup for {target_item}") break provider = next((p for p in self.providers if p.get_name() == provider_name), None) @@ -413,18 +398,15 @@ class Scanner: new_targets, large_entity_members, success = self._query_single_provider_for_target(provider, target_item, depth) if self._is_stop_requested(): - print(f"Stop requested after querying providers for {target_item}") break if not success: self.target_retries[task_tuple] += 1 if self.target_retries[task_tuple] <= self.config.max_retries_per_target: - print(f"Re-queueing task {task_tuple} (attempt {self.target_retries[task_tuple]})") self.task_queue.put((priority, (provider_name, target_item, depth))) self.tasks_re_enqueued += 1 self.total_tasks_ever_enqueued += 1 else: - print(f"ERROR: Max retries exceeded for task {task_tuple}") self.scan_failed_due_to_retries = True self._log_target_processing_error(str(task_tuple), "Max retries exceeded") else: @@ -446,21 +428,14 @@ class Scanner: with self.processing_lock: self.currently_processing.discard(task_tuple) - if self._is_stop_requested(): - print("Scan terminated due to stop request") - self.logger.logger.info("Scan terminated by user request") - elif self.task_queue.empty(): - print("Scan completed - no more targets to process") - self.logger.logger.info("Scan completed - all targets processed") - except Exception as e: - print(f"ERROR: Scan execution failed with error: {e}") traceback.print_exc() self.status = ScanStatus.FAILED self.logger.logger.error(f"Scan failed: {e}") finally: with self.processing_lock: self.currently_processing.clear() + self.currently_processing_display = [] if self._is_stop_requested(): self.status = ScanStatus.STOPPED @@ -469,31 +444,27 @@ class Scanner: else: self.status = ScanStatus.COMPLETED + # Stop the status logger + self.status_logger_stop_event.set() + if self.status_logger_thread: + self.status_logger_thread.join() + self._update_session_state() self.logger.log_scan_complete() if self.executor: self.executor.shutdown(wait=False, cancel_futures=True) self.executor = None - stats = self.graph.get_statistics() - print("Final scan statistics:") - print(f" - Total nodes: {stats['basic_metrics']['total_nodes']}") - print(f" - Total edges: {stats['basic_metrics']['total_edges']}") - print(f" - Tasks processed: {len(processed_tasks)}") def _query_single_provider_for_target(self, provider: BaseProvider, target: str, depth: int) -> Tuple[Set[str], Set[str], bool]: """ Query a single provider and process the unified ProviderResult. - Now provider-agnostic - handles any provider that returns ProviderResult. """ if self._is_stop_requested(): - print(f"Stop requested before querying {provider.get_name()} for {target}") return set(), set(), False is_ip = _is_valid_ip(target) target_type = NodeType.IP if is_ip else NodeType.DOMAIN - print(f"Querying {provider.get_name()} for {target_type.value}: {target} at depth {depth}") - # Ensure target node exists in graph self.graph.add_node(target, target_type) self._initialize_provider_states(target) @@ -502,13 +473,11 @@ class Scanner: provider_successful = True try: - # Query provider - now returns unified ProviderResult provider_result = self._query_single_provider_unified(provider, target, is_ip, depth) if provider_result is None: provider_successful = False elif not self._is_stop_requested(): - # Process the unified result discovered, is_large_entity = self._process_provider_result_unified( target, provider, provider_result, depth ) @@ -517,8 +486,6 @@ class Scanner: else: new_targets.update(discovered) self.graph.process_correlations_for_node(target) - else: - print(f"Stop requested after processing results from {provider.get_name()}") except Exception as e: provider_successful = False self._log_provider_error(target, provider.get_name(), str(e)) @@ -527,58 +494,45 @@ class Scanner: def _query_single_provider_unified(self, provider: BaseProvider, target: str, is_ip: bool, current_depth: int) -> Optional[ProviderResult]: """ - Query a single provider with stop signal checking, now returns ProviderResult. + Query a single provider with stop signal checking. """ provider_name = provider.get_name() start_time = datetime.now(timezone.utc) if self._is_stop_requested(): - print(f"Stop requested before querying {provider_name} for {target}") return None - print(f"Querying {provider_name} for {target}") - - self.logger.logger.info(f"Attempting {provider_name} query for {target} at depth {current_depth}") - try: - # Query the provider - returns unified ProviderResult if is_ip: result = provider.query_ip(target) else: result = provider.query_domain(target) if self._is_stop_requested(): - print(f"Stop requested after querying {provider_name} for {target}") return None - # Update provider state with relationship count (more meaningful than raw result count) relationship_count = result.get_relationship_count() if result else 0 self._update_provider_state(target, provider_name, 'success', relationship_count, None, start_time) - print(f"✓ {provider_name} returned {relationship_count} relationships for {target}") return result except Exception as e: self._update_provider_state(target, provider_name, 'failed', 0, str(e), start_time) - print(f"✗ {provider_name} failed for {target}: {e}") return None def _process_provider_result_unified(self, target: str, provider: BaseProvider, provider_result: ProviderResult, current_depth: int) -> Tuple[Set[str], bool]: """ Process a unified ProviderResult object to update the graph. - Returns (discovered_targets, is_large_entity). """ provider_name = provider.get_name() discovered_targets = set() if self._is_stop_requested(): - print(f"Stop requested before processing results from {provider_name} for {target}") return discovered_targets, False attributes_by_node = defaultdict(list) for attribute in provider_result.attributes: - # Convert the StandardAttribute object to a dictionary that the frontend can use attr_dict = { "name": attribute.name, "value": attribute.value, @@ -589,10 +543,8 @@ class Scanner: } attributes_by_node[attribute.target_node].append(attr_dict) - # Add attributes to nodes for node_id, node_attributes_list in attributes_by_node.items(): if self.graph.graph.has_node(node_id): - # Determine node type if _is_valid_ip(node_id): node_type = NodeType.IP elif node_id.startswith('AS') and node_id[2:].isdigit(): @@ -600,26 +552,19 @@ class Scanner: else: node_type = NodeType.DOMAIN - # Add node with the list of attributes self.graph.add_node(node_id, node_type, attributes=node_attributes_list) - # Check for large entity based on relationship count if provider_result.get_relationship_count() > self.config.large_entity_threshold: - print(f"Large entity detected: {provider_name} returned {provider_result.get_relationship_count()} relationships for {target}") members = self._create_large_entity_from_provider_result(target, provider_name, provider_result, current_depth) return members, True - # Process relationships for i, relationship in enumerate(provider_result.relationships): - if i % 5 == 0 and self._is_stop_requested(): # Check periodically for stop - print(f"Stop requested while processing relationships from {provider_name} for {target}") + if i % 5 == 0 and self._is_stop_requested(): break - # Add nodes for relationship endpoints source_node = relationship.source_node target_node = relationship.target_node - # Determine node types source_type = NodeType.IP if _is_valid_ip(source_node) else NodeType.DOMAIN if target_node.startswith('AS') and target_node[2:].isdigit(): target_type = NodeType.ASN @@ -628,11 +573,9 @@ class Scanner: else: target_type = NodeType.DOMAIN - # Add nodes to graph self.graph.add_node(source_node, source_type) self.graph.add_node(target_node, target_type) - # Add edge to graph if self.graph.add_edge( source_node, target_node, relationship.relationship_type, @@ -640,9 +583,8 @@ class Scanner: provider_name, relationship.raw_data ): - print(f"Added relationship: {source_node} -> {target_node} ({relationship.relationship_type})") + pass - # Track discovered targets for further processing if _is_valid_domain(target_node) or _is_valid_ip(target_node): discovered_targets.add(target_node) @@ -651,11 +593,10 @@ class Scanner: def _create_large_entity_from_provider_result(self, source: str, provider_name: str, provider_result: ProviderResult, current_depth: int) -> Set[str]: """ - Create a large entity node from a ProviderResult and return the members for DNS processing. + Create a large entity node from a ProviderResult. """ entity_id = f"large_entity_{provider_name}_{hash(source) & 0x7FFFFFFF}" - # Extract target nodes from relationships targets = [rel.target_node for rel in provider_result.relationships] node_type = 'unknown' @@ -665,7 +606,6 @@ class Scanner: elif _is_valid_ip(targets[0]): node_type = 'ip' - # Create nodes in graph (they exist but are grouped) for target in targets: target_node_type = NodeType.DOMAIN if node_type == 'domain' else NodeType.IP self.graph.add_node(target, target_node_type) @@ -694,106 +634,76 @@ class Scanner: self.graph.add_node(entity_id, NodeType.LARGE_ENTITY, attributes=attributes_list, description=description) - # Create edge from source to large entity if provider_result.relationships: rel_type = provider_result.relationships[0].relationship_type self.graph.add_edge(source, entity_id, rel_type, 0.9, provider_name, {'large_entity_info': f'Contains {len(targets)} {node_type}s'}) self.logger.logger.warning(f"Large entity created: {entity_id} contains {len(targets)} targets from {provider_name}") - print(f"Created large entity {entity_id} for {len(targets)} {node_type}s from {provider_name}") return set(targets) def stop_scan(self) -> bool: """Request immediate scan termination with proper cleanup.""" try: - print("=== INITIATING IMMEDIATE SCAN TERMINATION ===") self.logger.logger.info("Scan termination requested by user") - - # **IMPROVED**: More aggressive stop signal setting self._set_stop_signal() self.status = ScanStatus.STOPPED - # **NEW**: Clear processing state immediately with self.processing_lock: - currently_processing_copy = self.currently_processing.copy() self.currently_processing.clear() - print(f"Cleared {len(currently_processing_copy)} currently processing targets: {currently_processing_copy}") + self.currently_processing_display = [] - # **IMPROVED**: Clear task queue and log what was discarded discarded_tasks = [] while not self.task_queue.empty(): discarded_tasks.append(self.task_queue.get()) self.task_queue = PriorityQueue() - print(f"Discarded {len(discarded_tasks)} pending tasks") - # **IMPROVED**: Aggressively shut down executor if self.executor: - print("Shutting down executor with immediate cancellation...") try: - # Cancel all pending futures self.executor.shutdown(wait=False, cancel_futures=True) - print("Executor shutdown completed") except Exception as e: - print(f"Error during executor shutdown: {e}") + pass - # Immediately update GUI with stopped status self._update_session_state() - - print("Termination signals sent. The scan will stop as soon as possible.") return True except Exception as e: - print(f"ERROR: Exception in stop_scan: {e}") self.logger.logger.error(f"Error during scan termination: {e}") traceback.print_exc() return False def extract_node_from_large_entity(self, large_entity_id: str, node_id_to_extract: str) -> bool: """ - Extracts a node from a large entity, re-creates its original edge, and - re-queues it for full scanning. + Extracts a node from a large entity and re-queues it for scanning. """ if not self.graph.graph.has_node(large_entity_id): - print(f"ERROR: Large entity {large_entity_id} not found.") return False - # 1. Get the original source node that discovered the large entity predecessors = list(self.graph.graph.predecessors(large_entity_id)) if not predecessors: - print(f"ERROR: No source node found for large entity {large_entity_id}.") return False source_node_id = predecessors[0] - # Get the original edge data to replicate it for the extracted node original_edge_data = self.graph.graph.get_edge_data(source_node_id, large_entity_id) if not original_edge_data: - print(f"ERROR: Could not find original edge data from {source_node_id} to {large_entity_id}.") return False - # 2. Modify the graph data structure first success = self.graph.extract_node_from_large_entity(large_entity_id, node_id_to_extract) if not success: - print(f"ERROR: Node {node_id_to_extract} could not be removed from {large_entity_id}'s attributes.") return False - # 3. Create the direct edge from the original source to the newly extracted node - print(f"Re-creating direct edge from {source_node_id} to extracted node {node_id_to_extract}") self.graph.add_edge( source_id=source_node_id, target_id=node_id_to_extract, relationship_type=original_edge_data.get('relationship_type', 'extracted_from_large_entity'), - confidence_score=original_edge_data.get('confidence_score', 0.85), # Slightly lower confidence + confidence_score=original_edge_data.get('confidence_score', 0.85), source_provider=original_edge_data.get('source_provider', 'unknown'), raw_data={'context': f'Extracted from large entity {large_entity_id}'} ) - # 4. Re-queue the extracted node for full processing by all eligible providers - print(f"Re-queueing extracted node {node_id_to_extract} for full reconnaissance...") is_ip = _is_valid_ip(node_id_to_extract) - # FIX: Correctly retrieve discovery_depth from the list of attributes large_entity_attributes = self.graph.graph.nodes[large_entity_id].get('attributes', []) discovery_depth_attr = next((attr for attr in large_entity_attributes if attr.get('name') == 'discovery_depth'), None) current_depth = discovery_depth_attr['value'] if discovery_depth_attr else 0 @@ -804,9 +714,7 @@ class Scanner: self.task_queue.put((self._get_priority(provider_name), (provider_name, node_id_to_extract, current_depth))) self.total_tasks_ever_enqueued += 1 - # 5. If the scanner is not running, we need to kickstart it to process this one item. if self.status != ScanStatus.RUNNING: - print("Scanner is idle. Starting a mini-scan to process the extracted node.") self.status = ScanStatus.RUNNING self._update_session_state() @@ -818,25 +726,21 @@ class Scanner: ) self.scan_thread.start() - print(f"Successfully extracted and re-queued {node_id_to_extract} from {large_entity_id}.") return True def _update_session_state(self) -> None: """ Update the scanner state in Redis for GUI updates. - This ensures the web interface sees real-time updates. """ if self.session_id: try: from core.session_manager import session_manager - success = session_manager.update_session_scanner(self.session_id, self) - if not success: - print(f"WARNING: Failed to update session state for {self.session_id}") + session_manager.update_session_scanner(self.session_id, self) except Exception as e: - print(f"ERROR: Failed to update session state: {e}") + pass def get_scan_status(self) -> Dict[str, Any]: - """Get current scan status with processing information.""" + """Get current scan status with comprehensive processing information.""" try: with self.processing_lock: currently_processing_count = len(self.currently_processing) @@ -860,31 +764,18 @@ class Scanner: 'currently_processing': currently_processing_list[:5], 'tasks_in_queue': self.task_queue.qsize(), 'tasks_completed': self.indicators_completed, - 'tasks_skipped': self.total_tasks_ever_enqueued - self.task_queue.qsize() - self.indicators_completed - self.tasks_re_enqueued, + 'tasks_skipped': self.tasks_skipped, 'tasks_rescheduled': self.tasks_re_enqueued, } except Exception as e: - print(f"ERROR: Exception in get_scan_status: {e}") traceback.print_exc() return { - 'status': 'error', - 'target_domain': None, - 'current_depth': 0, - 'max_depth': 0, - 'current_indicator': '', - 'indicators_processed': 0, - 'indicators_completed': 0, - 'tasks_re_enqueued': 0, - 'progress_percentage': 0.0, - 'enabled_providers': [], - 'graph_statistics': {}, - 'task_queue_size': 0, - 'currently_processing_count': 0, - 'currently_processing': [], - 'tasks_in_queue': 0, - 'tasks_completed': 0, - 'tasks_skipped': 0, - 'tasks_rescheduled': 0, + 'status': 'error', 'target_domain': None, 'current_depth': 0, 'max_depth': 0, + 'current_indicator': '', 'indicators_processed': 0, 'indicators_completed': 0, + 'tasks_re_enqueued': 0, 'progress_percentage': 0.0, 'enabled_providers': [], + 'graph_statistics': {}, 'task_queue_size': 0, 'currently_processing_count': 0, + 'currently_processing': [], 'tasks_in_queue': 0, 'tasks_completed': 0, + 'tasks_skipped': 0, 'tasks_rescheduled': 0, } def _initialize_provider_states(self, target: str) -> None: @@ -910,8 +801,6 @@ class Scanner: if provider.get_eligibility().get(target_key): if not self._already_queried_provider(target, provider.get_name()): eligible.append(provider) - else: - print(f"Skipping {provider.get_name()} for {target} - already queried") return eligible @@ -923,7 +812,6 @@ class Scanner: node_data = self.graph.graph.nodes[target] provider_states = node_data.get('metadata', {}).get('provider_states', {}) - # A provider has been successfully queried if a state exists and its status is 'success' provider_state = provider_states.get(provider_name) return provider_state is not None and provider_state.get('status') == 'success' @@ -947,8 +835,6 @@ class Scanner: 'duration_ms': (datetime.now(timezone.utc) - start_time).total_seconds() * 1000 } - self.logger.logger.info(f"Provider state updated: {target} -> {provider_name} -> {status} ({results_count} results)") - def _log_target_processing_error(self, target: str, error: str) -> None: """Log target processing errors for forensic trail.""" self.logger.logger.error(f"Target processing failed for {target}: {error}") @@ -957,11 +843,6 @@ class Scanner: """Log provider query errors for forensic trail.""" self.logger.logger.error(f"Provider {provider_name} failed for {target}: {error}") - def _log_no_eligible_providers(self, target: str, is_ip: bool) -> None: - """Log when no providers are eligible for a target.""" - target_type = 'IP' if is_ip else 'domain' - self.logger.logger.warning(f"No eligible providers for {target_type}: {target}") - def _calculate_progress(self) -> float: """Calculate scan progress percentage based on task completion.""" if self.total_tasks_ever_enqueued == 0: @@ -996,13 +877,6 @@ class Scanner: } return export_data - def get_provider_statistics(self) -> Dict[str, Dict[str, Any]]: - """Get statistics for all providers with forensic information.""" - stats = {} - for provider in self.providers: - stats[provider.get_name()] = provider.get_statistics() - return stats - def get_provider_info(self) -> Dict[str, Dict[str, Any]]: """Get information about all available providers.""" info = {} @@ -1016,11 +890,9 @@ class Scanner: attribute = getattr(module, attribute_name) if isinstance(attribute, type) and issubclass(attribute, BaseProvider) and attribute is not BaseProvider: provider_class = attribute - # Instantiate to get metadata, even if not fully configured temp_provider = provider_class(name=attribute_name, session_config=self.config) provider_name = temp_provider.get_name() - # Find the actual provider instance if it exists, to get live stats live_provider = next((p for p in self.providers if p.get_name() == provider_name), None) info[provider_name] = { @@ -1031,6 +903,5 @@ class Scanner: 'rate_limit': self.config.get_rate_limit(provider_name), } except Exception as e: - print(f"✗ Failed to get info for provider from {filename}: {e}") traceback.print_exc() return info \ No newline at end of file From 8ae4fdbf80b9d655ffe308eccc913094885fb20b Mon Sep 17 00:00:00 2001 From: overcuriousity Date: Wed, 17 Sep 2025 21:12:11 +0200 Subject: [PATCH 12/20] UX improvements --- .env.example | 2 +- app.py | 116 +++++++- config.py | 56 +++- core/graph_manager.py | 68 +++-- core/provider_result.py | 7 +- core/scanner.py | 3 +- core/session_manager.py | 58 ---- providers/crtsh_provider.py | 70 ----- static/css/main.css | 181 ++++++++++++ static/js/main.js | 565 +++++++++++++++++++++++++++--------- templates/index.html | 139 +++++---- 11 files changed, 905 insertions(+), 360 deletions(-) diff --git a/.env.example b/.env.example index 002b9d5..47ee018 100644 --- a/.env.example +++ b/.env.example @@ -31,4 +31,4 @@ LARGE_ENTITY_THRESHOLD=100 # The number of times to retry a target if a provider fails. MAX_RETRIES_PER_TARGET=8 # How long cached provider responses are stored (in hours). -CACHE_EXPIRY_HOURS=12 +CACHE_TIMEOUT_HOURS=12 diff --git a/app.py b/app.py index 942277b..b4f3335 100644 --- a/app.py +++ b/app.py @@ -10,6 +10,7 @@ import traceback from flask import Flask, render_template, request, jsonify, send_file, session from datetime import datetime, timezone, timedelta import io +import os from core.session_manager import session_manager from config import config @@ -304,21 +305,6 @@ def export_results(): traceback.print_exc() return jsonify({'success': False, 'error': f'Export failed: {str(e)}'}), 500 - -@app.route('/api/providers', methods=['GET']) -def get_providers(): - """Get information about available providers.""" - try: - user_session_id, scanner = get_user_scanner() - provider_info = scanner.get_provider_info() - - return jsonify({'success': True, 'providers': provider_info, 'user_session_id': user_session_id}) - - except Exception as e: - traceback.print_exc() - return jsonify({'success': False, 'error': f'Internal server error: {str(e)}'}), 500 - - @app.route('/api/config/api-keys', methods=['POST']) def set_api_keys(): """Set API keys for the current session.""" @@ -355,6 +341,106 @@ def set_api_keys(): traceback.print_exc() return jsonify({'success': False, 'error': f'Internal server error: {str(e)}'}), 500 +@app.route('/api/providers', methods=['GET']) +def get_providers(): + """Get enhanced information about available providers including API key sources.""" + try: + user_session_id, scanner = get_user_scanner() + base_provider_info = scanner.get_provider_info() + + # Enhance provider info with API key source information + enhanced_provider_info = {} + + for provider_name, info in base_provider_info.items(): + enhanced_info = dict(info) # Copy base info + + if info['requires_api_key']: + # Determine API key source and configuration status + api_key = scanner.config.get_api_key(provider_name) + backend_api_key = os.getenv(f'{provider_name.upper()}_API_KEY') + + if backend_api_key: + # API key configured via backend/environment + enhanced_info.update({ + 'api_key_configured': True, + 'api_key_source': 'backend', + 'api_key_help': f'API key configured via environment variable {provider_name.upper()}_API_KEY' + }) + elif api_key: + # API key configured via web interface + enhanced_info.update({ + 'api_key_configured': True, + 'api_key_source': 'frontend', + 'api_key_help': f'API key set via web interface (session-only)' + }) + else: + # No API key configured + enhanced_info.update({ + 'api_key_configured': False, + 'api_key_source': None, + 'api_key_help': f'Requires API key to enable {info["display_name"]} integration' + }) + else: + # Provider doesn't require API key + enhanced_info.update({ + 'api_key_configured': True, # Always "configured" for non-API providers + 'api_key_source': None, + 'api_key_help': None + }) + + enhanced_provider_info[provider_name] = enhanced_info + + return jsonify({ + 'success': True, + 'providers': enhanced_provider_info, + 'user_session_id': user_session_id + }) + + except Exception as e: + traceback.print_exc() + return jsonify({'success': False, 'error': f'Internal server error: {str(e)}'}), 500 + + +@app.route('/api/config/providers', methods=['POST']) +def configure_providers(): + """Configure provider settings (enable/disable).""" + try: + data = request.get_json() + if data is None: + return jsonify({'success': False, 'error': 'No provider settings provided'}), 400 + + user_session_id, scanner = get_user_scanner() + session_config = scanner.config + + updated_providers = [] + + for provider_name, settings in data.items(): + provider_name_clean = provider_name.lower().strip() + + if 'enabled' in settings: + # Update the enabled state in session config + session_config.enabled_providers[provider_name_clean] = settings['enabled'] + updated_providers.append(provider_name_clean) + + if updated_providers: + # Reinitialize providers with new settings + scanner._initialize_providers() + session_manager.update_session_scanner(user_session_id, scanner) + + return jsonify({ + 'success': True, + 'message': f'Provider settings updated for: {", ".join(updated_providers)}', + 'user_session_id': user_session_id + }) + else: + return jsonify({'success': False, 'error': 'No valid provider settings were provided.'}), 400 + + except Exception as e: + traceback.print_exc() + return jsonify({'success': False, 'error': f'Internal server error: {str(e)}'}), 500 + + + @app.errorhandler(404) def not_found(error): """Handle 404 errors.""" diff --git a/config.py b/config.py index 2b2ff1c..3333846 100644 --- a/config.py +++ b/config.py @@ -25,7 +25,6 @@ class Config: self.max_concurrent_requests = 1 self.large_entity_threshold = 100 self.max_retries_per_target = 8 - self.cache_expiry_hours = 12 # --- Provider Caching Settings --- self.cache_timeout_hours = 6 # Provider-specific cache timeout @@ -69,7 +68,6 @@ class Config: self.max_concurrent_requests = int(os.getenv('MAX_CONCURRENT_REQUESTS', self.max_concurrent_requests)) self.large_entity_threshold = int(os.getenv('LARGE_ENTITY_THRESHOLD', self.large_entity_threshold)) self.max_retries_per_target = int(os.getenv('MAX_RETRIES_PER_TARGET', self.max_retries_per_target)) - self.cache_expiry_hours = int(os.getenv('CACHE_EXPIRY_HOURS', self.cache_expiry_hours)) self.cache_timeout_hours = int(os.getenv('CACHE_TIMEOUT_HOURS', self.cache_timeout_hours)) # Override Flask and session settings @@ -87,6 +85,60 @@ class Config: self.enabled_providers[provider] = True return True + def set_provider_enabled(self, provider: str, enabled: bool) -> bool: + """ + Set provider enabled status for the session. + + Args: + provider: Provider name + enabled: Whether the provider should be enabled + + Returns: + True if the setting was applied successfully + """ + provider_key = provider.lower() + self.enabled_providers[provider_key] = enabled + return True + + def get_provider_enabled(self, provider: str) -> bool: + """ + Get provider enabled status. + + Args: + provider: Provider name + + Returns: + True if the provider is enabled + """ + provider_key = provider.lower() + return self.enabled_providers.get(provider_key, True) # Default to enabled + + def bulk_set_provider_settings(self, provider_settings: dict) -> dict: + """ + Set multiple provider settings at once. + + Args: + provider_settings: Dict of provider_name -> {'enabled': bool, ...} + + Returns: + Dict with results for each provider + """ + results = {} + + for provider_name, settings in provider_settings.items(): + provider_key = provider_name.lower() + + try: + if 'enabled' in settings: + self.enabled_providers[provider_key] = settings['enabled'] + results[provider_key] = {'success': True, 'enabled': settings['enabled']} + else: + results[provider_key] = {'success': False, 'error': 'No enabled setting provided'} + except Exception as e: + results[provider_key] = {'success': False, 'error': str(e)} + + return results + def get_api_key(self, provider: str) -> Optional[str]: """Get API key for a provider.""" return self.api_keys.get(provider) diff --git a/core/graph_manager.py b/core/graph_manager.py index 3a904f9..5c28c79 100644 --- a/core/graph_manager.py +++ b/core/graph_manager.py @@ -4,7 +4,7 @@ Graph data model for DNSRecon using NetworkX. Manages in-memory graph storage with confidence scoring and forensic metadata. Now fully compatible with the unified ProviderResult data model. -UPDATED: Fixed certificate styling and correlation edge labeling. +UPDATED: Fixed correlation exclusion keys to match actual attribute names. """ import re from datetime import datetime, timezone @@ -41,7 +41,30 @@ class GraphManager: self.correlation_index = {} # Compile regex for date filtering for efficiency self.date_pattern = re.compile(r'^\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}') - self.EXCLUDED_KEYS = ['crtsh_cert_validity_period_days','crtsh_cert_source','crtsh_cert_common_name'] + + # These are the actual attribute names created in providers, WITHOUT provider prefix + self.EXCLUDED_KEYS = [ + # Certificate metadata that creates noise + 'cert_source', # Always 'crtsh' for crtsh provider + 'cert_common_name', + 'cert_validity_period_days', # Numerical, not useful for correlation + #'cert_certificate_id', # Unique per certificate + #'cert_serial_number', # Unique per certificate + 'cert_entry_timestamp', # Timestamp, filtered by date regex anyway + 'cert_not_before', # Date, filtered by date regex anyway + 'cert_not_after', # Date, filtered by date regex anyway + # DNS metadata that creates noise + 'dns_ttl', # TTL values are not meaningful for correlation + # Shodan metadata that might create noise + 'timestamp', # Generic timestamp fields + 'last_update', # Generic timestamp fields + #'org', # Too generic, causes false correlations + #'isp', # Too generic, causes false correlations + # Generic noisy attributes + 'updated_timestamp', # Any timestamp field + 'discovery_timestamp', # Any timestamp field + 'query_timestamp', # Any timestamp field + ] def __getstate__(self): """Prepare GraphManager for pickling, excluding compiled regex.""" @@ -72,14 +95,31 @@ class GraphManager: attr_value = attr.get('value') attr_provider = attr.get('provider', 'unknown') - # Skip excluded attributes and invalid values - if any(excluded_key in attr_name for excluded_key in self.EXCLUDED_KEYS) or not isinstance(attr_value, (str, int, float, bool)) or attr_value is None: - continue + # IMPROVED: More comprehensive exclusion logic + should_exclude = ( + # Check against excluded keys (exact match or substring) + any(excluded_key in attr_name or attr_name == excluded_key for excluded_key in self.EXCLUDED_KEYS) or + # Invalid value types + not isinstance(attr_value, (str, int, float, bool)) or + attr_value is None or + # Boolean values are not useful for correlation + isinstance(attr_value, bool) or + # String values that are too short or are dates + (isinstance(attr_value, str) and ( + len(attr_value) < 4 or + self.date_pattern.match(attr_value) or + # Exclude common generic values that create noise + attr_value.lower() in ['unknown', 'none', 'null', 'n/a', 'true', 'false', '0', '1'] + )) or + # Numerical values that are likely to be unique identifiers + (isinstance(attr_value, (int, float)) and ( + attr_value == 0 or # Zero values are not meaningful + attr_value == 1 or # One values are too common + abs(attr_value) > 1000000 # Very large numbers are likely IDs + )) + ) - if isinstance(attr_value, bool): - continue - - if isinstance(attr_value, str) and (len(attr_value) < 4 or self.date_pattern.match(attr_value)): + if should_exclude: continue # Initialize correlation tracking for this value @@ -149,7 +189,7 @@ class GraphManager: if self.graph.has_node(node_id) and not self.graph.has_edge(node_id, correlation_node_id): # Format relationship label as "corr_provider_attribute" - relationship_label = f"{provider}_{attribute}" + relationship_label = f"corr_{provider}_{attribute}" self.add_edge( source_id=node_id, @@ -170,7 +210,7 @@ class GraphManager: def _has_direct_edge_bidirectional(self, node_a: str, node_b: str) -> bool: """ Check if there's a direct edge between two nodes in either direction. - Returns True if node_a→node_b OR node_b→node_a exists. + Returns True if node_aâ†'node_b OR node_bâ†'node_a exists. """ return (self.graph.has_edge(node_a, node_b) or self.graph.has_edge(node_b, node_a)) @@ -410,12 +450,6 @@ class GraphManager: """Get all nodes of a specific type.""" return [n for n, d in self.graph.nodes(data=True) if d.get('type') == node_type.value] - def get_neighbors(self, node_id: str) -> List[str]: - """Get all unique neighbors (predecessors and successors) for a node.""" - if not self.graph.has_node(node_id): - return [] - return list(set(self.graph.predecessors(node_id)) | set(self.graph.successors(node_id))) - def get_high_confidence_edges(self, min_confidence: float = 0.8) -> List[Tuple[str, str, Dict]]: """Get edges with confidence score above a given threshold.""" return [(u, v, d) for u, v, d in self.graph.edges(data=True) diff --git a/core/provider_result.py b/core/provider_result.py index df2f6f1..7355cf4 100644 --- a/core/provider_result.py +++ b/core/provider_result.py @@ -101,6 +101,7 @@ class ProviderResult: """Get the total number of attributes in this result.""" return len(self.attributes) - def is_large_entity(self, threshold: int) -> bool: - """Check if this result qualifies as a large entity based on relationship count.""" - return self.get_relationship_count() > threshold \ No newline at end of file + ##TODO + #def is_large_entity(self, threshold: int) -> bool: + # """Check if this result qualifies as a large entity based on relationship count.""" + # return self.get_relationship_count() > threshold \ No newline at end of file diff --git a/core/scanner.py b/core/scanner.py index dcd86a2..5e5f8be 100644 --- a/core/scanner.py +++ b/core/scanner.py @@ -370,6 +370,7 @@ class Scanner: task_tuple = (provider_name, target_item) if task_tuple in processed_tasks: self.tasks_skipped += 1 + self.indicators_completed +=1 continue if depth > max_depth: @@ -405,7 +406,7 @@ class Scanner: if self.target_retries[task_tuple] <= self.config.max_retries_per_target: self.task_queue.put((priority, (provider_name, target_item, depth))) self.tasks_re_enqueued += 1 - self.total_tasks_ever_enqueued += 1 + #self.total_tasks_ever_enqueued += 1 else: self.scan_failed_due_to_retries = True self._log_target_processing_error(str(task_tuple), "Max retries exceeded") diff --git a/core/session_manager.py b/core/session_manager.py index 266c356..30a1940 100644 --- a/core/session_manager.py +++ b/core/session_manager.py @@ -108,64 +108,6 @@ class SessionManager: print(f"ERROR: Failed to create session {session_id}: {e}") raise - def clone_session_preserving_config(self, source_session_id: str) -> str: - """ - FIXED: Create a new session that preserves the configuration (including API keys) from an existing session. - This is used when we need a fresh scanner but want to keep user configuration. - """ - with self.creation_lock: - print(f"=== CLONING SESSION {source_session_id} (PRESERVING CONFIG) ===") - - try: - # Get the source session data - source_session_data = self._get_session_data(source_session_id) - if not source_session_data: - print(f"ERROR: Source session {source_session_id} not found for cloning") - return self.create_session() # Fallback to new session - - # Create new session ID - new_session_id = str(uuid.uuid4()) - - # Get the preserved configuration - preserved_config = source_session_data.get('config') - if not preserved_config: - print(f"WARNING: No config found in source session, creating new") - from core.session_config import create_session_config - preserved_config = create_session_config() - - print(f"Preserving config with API keys: {list(preserved_config.api_keys.keys())}") - - # Create new scanner with preserved config - new_scanner = Scanner(session_config=preserved_config) - new_scanner.session_id = new_session_id - - - new_session_data = { - 'scanner': new_scanner, - 'config': preserved_config, - 'created_at': time.time(), - 'last_activity': time.time(), - 'status': 'active', - 'cloned_from': source_session_id - } - - # Store in Redis - serialized_data = pickle.dumps(new_session_data) - session_key = self._get_session_key(new_session_id) - self.redis_client.setex(session_key, self.session_timeout, serialized_data) - - # Initialize stop signal - stop_key = self._get_stop_signal_key(new_session_id) - self.redis_client.setex(stop_key, self.session_timeout, b'0') - - print(f"Cloned session {new_session_id} with preserved configuration") - return new_session_id - - except Exception as e: - print(f"ERROR: Failed to clone session {source_session_id}: {e}") - # Fallback to creating a new session - return self.create_session() - def set_stop_signal(self, session_id: str) -> bool: """ Set the stop signal for a session (cross-process safe). diff --git a/providers/crtsh_provider.py b/providers/crtsh_provider.py index bb712a6..1e623c0 100644 --- a/providers/crtsh_provider.py +++ b/providers/crtsh_provider.py @@ -502,76 +502,6 @@ class CrtShProvider(BaseProvider): return [d for d in final_domains if _is_valid_domain(d)] - def _find_shared_certificates(self, certs1: List[Dict[str, Any]], certs2: List[Dict[str, Any]]) -> List[Dict[str, Any]]: - """Find certificates that are shared between two domain certificate lists.""" - shared = [] - - cert1_ids = set() - for cert in certs1: - cert_id = cert.get('certificate_id') - if cert_id and isinstance(cert_id, (int, str, float, bool, tuple)): - cert1_ids.add(cert_id) - - for cert in certs2: - cert_id = cert.get('certificate_id') - if cert_id and isinstance(cert_id, (int, str, float, bool, tuple)): - if cert_id in cert1_ids: - shared.append(cert) - - return shared - - def _summarize_certificates(self, certificates: List[Dict[str, Any]]) -> Dict[str, Any]: - """Create a summary of certificates for a domain.""" - if not certificates: - return { - 'total_certificates': 0, - 'valid_certificates': 0, - 'expired_certificates': 0, - 'expires_soon_count': 0, - 'unique_issuers': [], - 'latest_certificate': None, - 'has_valid_cert': False, - 'certificate_details': [] - } - - valid_count = sum(1 for cert in certificates if cert.get('is_currently_valid')) - expired_count = len(certificates) - valid_count - expires_soon_count = sum(1 for cert in certificates if cert.get('expires_soon')) - - unique_issuers = list(set(cert.get('issuer_name') for cert in certificates if cert.get('issuer_name'))) - - # Find the most recent certificate - latest_cert = None - latest_date = None - - for cert in certificates: - try: - if cert.get('not_before'): - cert_date = self._parse_certificate_date(cert['not_before']) - if latest_date is None or cert_date > latest_date: - latest_date = cert_date - latest_cert = cert - except Exception: - continue - - # Sort certificates by date for better display (newest first) - sorted_certificates = sorted( - certificates, - key=lambda c: self._get_certificate_sort_date(c), - reverse=True - ) - - return { - 'total_certificates': len(certificates), - 'valid_certificates': valid_count, - 'expired_certificates': expired_count, - 'expires_soon_count': expires_soon_count, - 'unique_issuers': unique_issuers, - 'latest_certificate': latest_cert, - 'has_valid_cert': valid_count > 0, - 'certificate_details': sorted_certificates - } - def _get_certificate_sort_date(self, cert: Dict[str, Any]) -> datetime: """Get a sortable date from certificate data for chronological ordering.""" try: diff --git a/static/css/main.css b/static/css/main.css index 0177c4b..af09f27 100644 --- a/static/css/main.css +++ b/static/css/main.css @@ -2114,4 +2114,185 @@ input[type="text"]:focus, select:focus { flex-direction: column; gap: 0.5rem; } +} + +/* Add these styles to the existing main.css file - they use existing design patterns */ + +/* Settings Modal Enhancements - using existing provider-item styles */ +#settings-modal .modal-section { + margin-bottom: 1.5rem; +} + +#settings-modal .modal-section:last-child { + margin-bottom: 0; +} + +/* Provider configuration section using existing provider-item styling */ +#provider-config-list .provider-item { + margin-bottom: 1rem; + background: linear-gradient(135deg, #2a2a2a 0%, #1e1e1e 100%); + border: 1px solid #333; + border-radius: 8px; + transition: all 0.3s ease; +} + +#provider-config-list .provider-item:last-child { + margin-bottom: 0; +} + +#provider-config-list .provider-item:hover { + border-color: #444; + transform: translateY(-1px); + box-shadow: 0 4px 12px rgba(0,0,0,0.3); +} + +/* Provider toggle styling - extends existing checkbox pattern */ +.provider-toggle { + appearance: none !important; + width: 16px !important; + height: 16px !important; + border: 2px solid #555 !important; + background: #1a1a1a !important; + cursor: pointer !important; + position: relative !important; + border-radius: 3px !important; + transition: all 0.3s ease !important; + margin: 0 !important; +} + +.provider-toggle:checked { + background: #00ff41 !important; + border-color: #00ff41 !important; +} + +.provider-toggle:checked::after { + content: '✓' !important; + position: absolute !important; + top: -3px !important; + left: 1px !important; + color: #1a1a1a !important; + font-size: 12px !important; + font-weight: bold !important; +} + +.provider-toggle:hover { + border-color: #00ff41 !important; + box-shadow: 0 0 5px rgba(0, 255, 65, 0.3) !important; +} + +/* API Key status row styling - uses existing status-row pattern */ +.api-key-status-row { + padding: 0.75rem; + border-radius: 4px; + border: 1px solid; + transition: all 0.3s ease; + margin-top: 0.5rem; +} + +.api-key-status-row.configured { + background: rgba(0, 255, 65, 0.1); + border-color: rgba(0, 255, 65, 0.3); +} + +.api-key-status-row.not-configured { + background: rgba(255, 153, 0, 0.1); + border-color: rgba(255, 153, 0, 0.3); +} + +/* Settings modal button styling - uses existing button classes */ +#settings-modal .button-group { + padding-top: 1.5rem; + border-top: 1px solid #333; + margin-top: 1.5rem; +} + +/* Clear button specific styling - extends btn-secondary */ +.clear-api-key-btn { + font-size: 0.8rem !important; + padding: 0.4rem 0.8rem !important; + min-width: auto !important; +} + +.clear-api-key-btn:disabled { + opacity: 0.5 !important; + cursor: not-allowed !important; +} + +/* Provider configuration checkbox label styling */ +#provider-config-list .status-row label { + color: #e0e0e0; + cursor: pointer; + transition: color 0.3s ease; + font-size: 0.9rem; + font-weight: 500; +} + +#provider-config-list .status-row label:hover { + color: #00ff41; +} + +/* Settings section count badges - uses existing merge-badge styling */ +#provider-count, +#api-key-count { + background: #444; + color: #fff; + padding: 0.25rem 0.5rem; + border-radius: 3px; + font-size: 0.8rem; + font-weight: 600; + min-width: 20px; + text-align: center; +} + +/* API key help text styling - uses existing patterns */ +#settings-modal .apikey-help { + font-size: 0.8rem; + color: #666; + margin-top: 0.5rem; + font-style: italic; + line-height: 1.4; +} + +/* Modal section content spacing */ +#settings-modal .modal-section-content { + padding: 1rem; +} + +#settings-modal .modal-section-content .provider-item { + padding: 1rem; +} + +/* Transition effects for API key clearing */ +.api-key-clearing { + opacity: 0.6; + pointer-events: none; + transition: all 0.3s ease; +} + +.api-key-cleared { + opacity: 0; + transform: translateX(-10px); + transition: all 0.3s ease; +} + +/* Settings modal responsiveness - extends existing responsive patterns */ +@media (max-width: 768px) { + #settings-modal .modal-content { + width: 95%; + margin: 5% auto; + max-height: 90vh; + } + + #provider-config-list .provider-item { + margin-bottom: 0.75rem; + } + + #settings-modal .provider-stats { + grid-template-columns: 1fr; + gap: 0.5rem; + } + + #settings-modal .button-group { + flex-direction: column; + } } \ No newline at end of file diff --git a/static/js/main.js b/static/js/main.js index f1d82c0..948182b 100644 --- a/static/js/main.js +++ b/static/js/main.js @@ -34,8 +34,8 @@ class DNSReconApp { this.updateStatus(); this.loadProviders(); this.initializeEnhancedModals(); + this.addCheckboxStyling(); - // FIXED: Force initial graph update to handle empty sessions properly this.updateGraph(); console.log('DNSRecon application initialized successfully'); @@ -81,9 +81,7 @@ class DNSReconApp { // Settings Modal elements settingsModal: document.getElementById('settings-modal'), settingsModalClose: document.getElementById('settings-modal-close'), - apiKeyInputs: document.getElementById('api-key-inputs'), - saveApiKeys: document.getElementById('save-api-keys'), - resetApiKeys: document.getElementById('reset-api-keys'), + // Other elements sessionId: document.getElementById('session-id'), @@ -182,10 +180,21 @@ class DNSReconApp { }); } if (this.elements.saveApiKeys) { - this.elements.saveApiKeys.addEventListener('click', () => this.saveApiKeys()); + this.elements.saveApiKeys.removeEventListener('click', this.saveApiKeys); } if (this.elements.resetApiKeys) { - this.elements.resetApiKeys.addEventListener('click', () => this.resetApiKeys()); + this.elements.resetApiKeys.removeEventListener('click', this.resetApiKeys); + } + + // Setup new handlers + const saveSettingsBtn = document.getElementById('save-settings'); + const resetSettingsBtn = document.getElementById('reset-settings'); + + if (saveSettingsBtn) { + saveSettingsBtn.addEventListener('click', () => this.saveSettings()); + } + if (resetSettingsBtn) { + resetSettingsBtn.addEventListener('click', () => this.resetSettings()); } // Listen for the custom event from the graph @@ -724,7 +733,7 @@ class DNSReconApp { if (response.success) { this.updateProviderDisplay(response.providers); - this.buildApiKeyModal(response.providers); + this.buildSettingsModal(response.providers); // Updated to use new function console.log('Providers loaded successfully'); } @@ -732,6 +741,411 @@ class DNSReconApp { console.error('Failed to load providers:', error); } } + + /** + * Build the enhanced settings modal with provider configuration and API keys + * @param {Object} providers - Provider information from backend + */ + buildSettingsModal(providers) { + this.buildProviderConfigSection(providers); + this.buildApiKeySection(providers); + this.updateSettingsCounts(providers); + } + + /** + * Build the provider configuration section with enable/disable checkboxes + * @param {Object} providers - Provider information + */ + buildProviderConfigSection(providers) { + const providerConfigList = document.getElementById('provider-config-list'); + if (!providerConfigList) return; + + providerConfigList.innerHTML = ''; + + for (const [name, info] of Object.entries(providers)) { + const providerConfig = document.createElement('div'); + providerConfig.className = 'provider-item'; + + const statusClass = info.enabled ? 'enabled' : 'disabled'; + const statusIcon = info.enabled ? '✓' : '✗'; + + providerConfig.innerHTML = ` +
+
${info.display_name}
+
+ ${statusIcon} ${info.enabled ? 'Enabled' : 'Disabled'} +
+
+
+
+ +
+
+ `; + + providerConfigList.appendChild(providerConfig); + } + + // Add checkbox styling and event handlers + this.setupProviderCheckboxes(); + } + + /** + * Setup provider checkbox styling and event handlers + */ + setupProviderCheckboxes() { + const checkboxes = document.querySelectorAll('.provider-toggle'); + + checkboxes.forEach(checkbox => { + // Apply existing checkbox styling + checkbox.style.cssText = ` + appearance: none; + width: 16px; + height: 16px; + border: 2px solid #555; + background: #1a1a1a; + cursor: pointer; + position: relative; + border-radius: 3px; + transition: all 0.3s ease; + `; + + // Update visual state + this.updateCheckboxAppearance(checkbox); + + // Add change event handler + checkbox.addEventListener('change', (e) => { + this.updateCheckboxAppearance(e.target); + }); + }); + } + + /** + * Add CSS for checkbox styling since we're using existing styles + */ + addCheckboxStyling() { + // Add CSS for the checkboxes to work with existing styles + const style = document.createElement('style'); + style.textContent = ` + .provider-toggle[data-checked="true"]::after { + content: '✓'; + position: absolute; + top: -2px; + left: 2px; + color: #1a1a1a; + font-size: 12px; + font-weight: bold; + } + + .provider-toggle:hover { + border-color: #00ff41; + } + + .api-key-status-row { + transition: all 0.3s ease; + } + + .provider-item { + margin-bottom: 1rem; + } + + .provider-item:last-child { + margin-bottom: 0; + } + `; + document.head.appendChild(style); + } + + + /** + * Update checkbox appearance based on checked state + */ + updateCheckboxAppearance(checkbox) { + if (checkbox.checked) { + checkbox.style.background = '#00ff41'; + checkbox.style.borderColor = '#00ff41'; + checkbox.style.setProperty('content', '"✓"', 'important'); + + // Add checkmark via pseudo-element simulation + checkbox.setAttribute('data-checked', 'true'); + } else { + checkbox.style.background = '#1a1a1a'; + checkbox.style.borderColor = '#555'; + checkbox.removeAttribute('data-checked'); + } + } + + /** + * Enhanced API key section builder - FIXED to always allow API key input + * @param {Object} providers - Provider information + */ + buildApiKeySection(providers) { + const apiKeyInputs = document.getElementById('api-key-inputs'); + if (!apiKeyInputs) return; + + apiKeyInputs.innerHTML = ''; + let hasApiKeyProviders = false; + + for (const [name, info] of Object.entries(providers)) { + if (info.requires_api_key) { + hasApiKeyProviders = true; + + const inputGroup = document.createElement('div'); + inputGroup.className = 'provider-item'; + + // Check if API key is set via backend (not clearable) or frontend (clearable) + const isBackendConfigured = info.api_key_source === 'backend'; + + if (info.api_key_configured && isBackendConfigured) { + // API key is configured via backend - show status only + inputGroup.innerHTML = ` +
+
${info.display_name}
+
✓ Backend Configured
+
+
+
+
+
API Key Active
+
+ Configured via environment variable +
+
+
+
+ `; + } else if (info.api_key_configured && !isBackendConfigured) { + // API key is configured via frontend - show status with clear option + inputGroup.innerHTML = ` +
+
${info.display_name}
+
✓ Web Configured
+
+
+
+
+
API Key Active
+
+ Set via web interface (session-only) +
+
+ +
+
+ `; + } else { + // API key not configured - ALWAYS show input field + const statusClass = info.enabled ? 'enabled' : 'api-key-required'; + const statusText = info.enabled ? '○ Ready for API Key' : '⚠️ API Key Required'; + + inputGroup.innerHTML = ` +
+
${info.display_name}
+
+ ${statusText} +
+
+
+ + +
+ ${info.api_key_help || `Provides enhanced ${info.display_name.toLowerCase()} data and context.`} + ${!info.enabled ? ' Enable the provider above to use this API key.' : ''} +
+
+ `; + } + + apiKeyInputs.appendChild(inputGroup); + } + } + + if (!hasApiKeyProviders) { + apiKeyInputs.innerHTML = ` +
+
No providers require API keys
+
All Active
+
+ `; + } + + // Setup clear button event handlers + this.setupApiKeyClearHandlers(); + } + + /** + * Setup API key clear button handlers + */ + setupApiKeyClearHandlers() { + document.querySelectorAll('.clear-api-key-btn').forEach(button => { + button.addEventListener('click', (e) => { + e.preventDefault(); + const provider = e.currentTarget.dataset.provider; + this.clearSingleApiKey(provider, e.currentTarget); + }); + }); + } + + /** + * Clear a single API key with immediate feedback + */ + async clearSingleApiKey(provider, buttonElement) { + try { + // Show immediate feedback + const originalContent = buttonElement.innerHTML; + buttonElement.innerHTML = '[...]Clearing...'; + buttonElement.disabled = true; + + const response = await this.apiCall('/api/config/api-keys', 'POST', { [provider]: '' }); + + if (response.success) { + // Find the parent container and update it + const providerContainer = buttonElement.closest('.provider-item'); + const statusRow = providerContainer.querySelector('.api-key-status-row'); + + // Animate out the current status + statusRow.style.transition = 'all 0.3s ease'; + statusRow.style.opacity = '0'; + statusRow.style.transform = 'translateX(-10px)'; + + setTimeout(() => { + // Replace with input field + const providerName = buttonElement.dataset.provider; + const apiKeySection = this.elements.apiKeyInputs; + + // Rebuild the API key section to reflect changes + this.loadProviders(); + + this.showSuccess(`API key for ${provider} has been cleared.`); + }, 300); + + } else { + throw new Error(response.error || 'Failed to clear API key'); + } + } catch (error) { + // Restore button on error + buttonElement.innerHTML = originalContent; + buttonElement.disabled = false; + this.showError(`Error clearing API key: ${error.message}`); + } + } + + /** + * Update settings modal counts + */ + updateSettingsCounts(providers) { + const providerCount = Object.keys(providers).length; + const apiKeyCount = Object.values(providers).filter(p => p.requires_api_key).length; + + const providerCountElement = document.getElementById('provider-count'); + const apiKeyCountElement = document.getElementById('api-key-count'); + + if (providerCountElement) providerCountElement.textContent = providerCount; + if (apiKeyCountElement) apiKeyCountElement.textContent = apiKeyCount; + } + + /** + * Enhanced save settings function + */ + async saveSettings() { + try { + const settings = { + apiKeys: {}, + providerSettings: {} + }; + + // Collect API key inputs + const apiKeyInputs = document.querySelectorAll('#api-key-inputs input[type="password"]'); + apiKeyInputs.forEach(input => { + const provider = input.dataset.provider; + const value = input.value.trim(); + if (provider && value) { + settings.apiKeys[provider] = value; + } + }); + + // Collect provider enable/disable settings + const providerCheckboxes = document.querySelectorAll('.provider-toggle'); + providerCheckboxes.forEach(checkbox => { + const provider = checkbox.dataset.provider; + if (provider) { + settings.providerSettings[provider] = { + enabled: checkbox.checked + }; + } + }); + + // Save API keys if any + if (Object.keys(settings.apiKeys).length > 0) { + const apiKeyResponse = await this.apiCall('/api/config/api-keys', 'POST', settings.apiKeys); + if (!apiKeyResponse.success) { + throw new Error(apiKeyResponse.error || 'Failed to save API keys'); + } + } + + // Save provider settings if any + if (Object.keys(settings.providerSettings).length > 0) { + const providerResponse = await this.apiCall('/api/config/providers', 'POST', settings.providerSettings); + if (!providerResponse.success) { + throw new Error(providerResponse.error || 'Failed to save provider settings'); + } + } + + this.showSuccess('Settings saved successfully'); + this.hideSettingsModal(); + + // Reload providers to reflect changes + this.loadProviders(); + + } catch (error) { + this.showError(`Error saving settings: ${error.message}`); + } + } + + /** + * Reset settings to defaults + */ + async resetSettings() { + try { + // Clear all API key inputs + const apiKeyInputs = document.querySelectorAll('#api-key-inputs input[type="password"]'); + apiKeyInputs.forEach(input => { + input.value = ''; + }); + + // Reset all provider checkboxes to enabled (default) + const providerCheckboxes = document.querySelectorAll('.provider-toggle'); + providerCheckboxes.forEach(checkbox => { + checkbox.checked = true; + this.updateCheckboxAppearance(checkbox); + }); + + // Reset recursion depth to default + const depthSelect = document.getElementById('max-depth'); + if (depthSelect) { + depthSelect.value = '2'; + } + + this.showInfo('Settings reset to defaults'); + + } catch (error) { + this.showError(`Error resetting settings: ${error.message}`); + } + } /** * Update provider display @@ -1501,33 +1915,6 @@ class DNSReconApp { } } - /** - * Copy text to clipboard with user feedback - */ - async copyToClipboard(text) { - try { - await navigator.clipboard.writeText(text); - this.showMessage('Copied to clipboard', 'success'); - } catch (err) { - // Fallback for older browsers - const textArea = document.createElement('textarea'); - textArea.value = text; - textArea.style.position = 'fixed'; - textArea.style.opacity = '0'; - document.body.appendChild(textArea); - textArea.select(); - - try { - document.execCommand('copy'); - this.showMessage('Copied to clipboard', 'success'); - } catch (fallbackErr) { - this.showMessage('Failed to copy text', 'error'); - } - - document.body.removeChild(textArea); - } - } - /** * Enhanced keyboard navigation for modals */ @@ -1690,40 +2077,7 @@ class DNSReconApp { }); } - /** - * Check if graph data has changed - * @param {Object} graphData - New graph data - * @returns {boolean} True if data has changed - */ - hasGraphChanged(graphData) { - // Simple check based on node and edge counts and timestamps - const currentStats = this.graphManager.getStatistics(); - const newNodeCount = graphData.nodes ? graphData.nodes.length : 0; - const newEdgeCount = graphData.edges ? graphData.edges.length : 0; - - // FIXED: Always update if we currently have no data (ensures placeholder is handled correctly) - if (currentStats.nodeCount === 0 && currentStats.edgeCount === 0) { - return true; - } - - // Check if counts changed - const countsChanged = currentStats.nodeCount !== newNodeCount || currentStats.edgeCount !== newEdgeCount; - - // Also check if we have new timestamp data - const hasNewTimestamp = graphData.statistics && - graphData.statistics.last_modified && - graphData.statistics.last_modified !== this.lastGraphTimestamp; - - if (hasNewTimestamp) { - this.lastGraphTimestamp = graphData.statistics.last_modified; - } - - const changed = countsChanged || hasNewTimestamp; - - console.log(`Graph change check: Current(${currentStats.nodeCount}n, ${currentStats.edgeCount}e) vs New(${newNodeCount}n, ${newEdgeCount}e) = ${changed}`); - - return changed; - } + /** * Make API call to server @@ -1823,15 +2177,6 @@ class DNSReconApp { return statusMap[status] || status; } - /** - * Format label for display - * @param {string} label - Raw label - * @returns {string} Formatted label - */ - formatLabel(label) { - return label.replace(/_/g, ' ').replace(/\b\w/g, l => l.toUpperCase()); - } - /** * Format value for display * @param {*} value - Raw value @@ -1955,73 +2300,7 @@ class DNSReconApp { return colors[type] || colors.info; } - /** - * Build the API key modal dynamically - * @param {Object} providers - Provider information - */ - buildApiKeyModal(providers) { - if (!this.elements.apiKeyInputs) return; - this.elements.apiKeyInputs.innerHTML = ''; // Clear existing inputs - let hasApiKeyProviders = false; - for (const [name, info] of Object.entries(providers)) { - if (info.requires_api_key) { - hasApiKeyProviders = true; - const inputGroup = document.createElement('div'); - inputGroup.className = 'apikey-section'; - - if (info.enabled) { - // If the API key is set and the provider is enabled - inputGroup.innerHTML = ` - -
- API Key is set - -
-

Provides infrastructure context and service information.

- `; - } else { - // If the API key is not set - inputGroup.innerHTML = ` - - -

Provides infrastructure context and service information.

- `; - } - this.elements.apiKeyInputs.appendChild(inputGroup); - } - } - - // Add event listeners for the new clear buttons - this.elements.apiKeyInputs.querySelectorAll('.clear-api-key-btn').forEach(button => { - button.addEventListener('click', (e) => { - const provider = e.target.dataset.provider; - this.clearApiKey(provider); - }); - }); - - if (!hasApiKeyProviders) { - this.elements.apiKeyInputs.innerHTML = '

No providers require API keys.

'; - } - } - - /** - * Clear an API key for a specific provider - * @param {string} provider The name of the provider to clear the API key for - */ - async clearApiKey(provider) { - try { - const response = await this.apiCall('/api/config/api-keys', 'POST', { [provider]: '' }); - if (response.success) { - this.showSuccess(`API key for ${provider} has been cleared.`); - this.loadProviders(); // This will rebuild the modal with the updated state - } else { - throw new Error(response.error || 'Failed to clear API key'); - } - } catch (error) { - this.showError(`Error clearing API key: ${error.message}`); - } - } } // Add CSS animations for message toasts diff --git a/templates/index.html b/templates/index.html index 730ab0b..1f9fad7 100644 --- a/templates/index.html +++ b/templates/index.html @@ -1,5 +1,6 @@ + @@ -7,8 +8,11 @@ - + +
@@ -29,13 +33,13 @@

Target Configuration

- +
- +
+ +
+
+ ISPs +
+
+
+ Certificate Authorities +
Domain (invalid cert) From cd141984527cd2b5db51b051f969494e14d6520f Mon Sep 17 00:00:00 2001 From: overcuriousity Date: Wed, 17 Sep 2025 22:39:26 +0200 Subject: [PATCH 18/20] smaller css --- static/css/main.css | 2388 +++++++++++++------------------------------ 1 file changed, 708 insertions(+), 1680 deletions(-) diff --git a/static/css/main.css b/static/css/main.css index af09f27..1d68ffd 100644 --- a/static/css/main.css +++ b/static/css/main.css @@ -1,6 +1,6 @@ -/* DNSRecon - Enhanced Tactical/Cybersecurity Theme */ +/* DNSRecon - Optimized Compact Theme */ -/* Reset and Base Styles */ +/* Reset and Base */ * { margin: 0; padding: 0; @@ -8,28 +8,27 @@ } body { - font-family: 'Roboto Mono', 'SF Mono', 'Monaco', 'Inconsolata', 'Fira Code', monospace; + font-family: 'Roboto Mono', monospace; background: linear-gradient(135deg, #1a1a1a 0%, #0f0f0f 100%); color: #e0e0e0; - line-height: 1.6; - letter-spacing: 0.5px; + line-height: 1.4; + letter-spacing: 0.3px; + font-size: 0.85rem; min-height: 100vh; overflow-x: hidden; } -/* Container and Layout */ .container { min-height: 100vh; display: flex; flex-direction: column; } -/* === ENHANCED HEADER === */ +/* Header */ .header { background: linear-gradient(135deg, #0a0a0a 0%, #1a1a1a 100%); - border-bottom: 2px solid #333; - padding: 1.5rem 2rem; - box-shadow: 0 4px 20px rgba(0,0,0,0.5); + border-bottom: 1px solid #333; + padding: 0.75rem 1rem; position: sticky; top: 0; z-index: 100; @@ -39,64 +38,44 @@ body { display: flex; justify-content: space-between; align-items: center; - max-width: 1400px; + max-width: 2400px; margin: 0 auto; } .logo { display: flex; align-items: center; - font-family: 'Special Elite', 'Courier New', monospace; - font-size: 1.6rem; + font-family: 'Special Elite', monospace; + font-size: 1.3rem; font-weight: 700; - gap: 0.5rem; + gap: 0.3rem; } .logo-icon { color: #00ff41; - margin-right: 0.5rem; - text-shadow: 0 0 15px rgba(0, 255, 65, 0.6); - animation: logoGlow 3s ease-in-out infinite alternate; -} - -@keyframes logoGlow { - from { text-shadow: 0 0 15px rgba(0, 255, 65, 0.6); } - to { text-shadow: 0 0 25px rgba(0, 255, 65, 0.8); } + text-shadow: 0 0 10px rgba(0, 255, 65, 0.5); } .logo-text { color: #c7c7c7; - text-shadow: 0 0 5px rgba(199, 199, 199, 0.3); } .status-indicator { display: flex; align-items: center; - gap: 0.75rem; - padding: 0.5rem 1rem; + gap: 0.5rem; + padding: 0.3rem 0.75rem; background: rgba(42, 42, 42, 0.6); - border-radius: 25px; + border-radius: 15px; border: 1px solid #333; } -.status-indicator.scanning { - animation: pulse 1.5s infinite; -} - -.status-indicator.completed { - background-color: #00ff41; -} - -.status-indicator.error { - background-color: #ff6b6b; -} - .status-dot { - width: 10px; - height: 10px; + width: 8px; + height: 8px; border-radius: 50%; background-color: #00ff41; - box-shadow: 0 0 10px rgba(0, 255, 65, 0.7); + box-shadow: 0 0 8px rgba(0, 255, 65, 0.6); animation: statusPulse 2s infinite; } @@ -105,140 +84,147 @@ body { 50% { opacity: 0.7; transform: scale(1.1); } } -@keyframes pulse { - 0%, 100% { opacity: 1; } - 50% { opacity: 0.5; } +.status-indicator.scanning { + animation: pulse 1.5s infinite; } .status-text { - font-size: 0.9rem; + font-size: 0.8rem; color: #00ff41; } -/* Main Content */ +/* Main Layout - Graph-Centric */ .main-content { flex: 1; - padding: 1.25rem; /* Reduced from 2rem */ - max-width: 1400px; + padding: 0.75rem; + max-width: 2400px; margin: 0 auto; width: 100%; display: grid; - grid-template-columns: 1fr 1fr; - grid-template-rows: auto auto 1fr auto; - gap: 1rem; /* Reduced from 1.5rem */ + grid-template-columns: 300px 1fr; + grid-template-rows: auto 1fr auto; + gap: 0.75rem; grid-template-areas: "control status" "visualization visualization" - "visualization visualization" "providers providers"; } -/* === ENHANCED PANELS === */ +/* Unified Panel Styles */ section { background: linear-gradient(135deg, #2a2a2a 0%, #1e1e1e 100%); border: 1px solid #444; - border-radius: 8px; - box-shadow: 0 8px 32px rgba(0,0,0,0.3); + border-radius: 6px; overflow: hidden; - transition: all 0.3s ease; -} - -section:hover { - border-color: #555; - box-shadow: 0 12px 40px rgba(0,0,0,0.4); + box-shadow: 0 4px 15px rgba(0,0,0,0.2); } .panel-header { background: linear-gradient(90deg, #333 0%, #2a2a2a 100%); - padding: 0.75rem 1rem; /* Reduced from 1.25rem 1.5rem */ - border-bottom: 2px solid #444; - position: relative; - overflow: hidden; - display: flex; - justify-content: space-between; - align-items: center; -} - -/* Remove the generic top line */ -.panel-header::before { - display: none; + padding: 0.5rem 0.75rem; + border-bottom: 1px solid #444; } .panel-header h2 { - font-size: 1.2rem; + font-size: 1rem; color: #00ff41; - text-shadow: 0 0 10px rgba(0, 255, 65, 0.4); font-weight: 600; margin: 0; - letter-spacing: 1px; } -/* Control Panel */ +/* Controls */ .control-panel { grid-area: control; } .form-container { - padding: 1rem; + padding: 0.75rem; } .input-group { - margin-bottom: 1.25rem; + margin-bottom: 0.75rem; } .input-group label { display: block; - margin-bottom: 0.75rem; + margin-bottom: 0.4rem; color: #e0e0e0; - font-size: 0.95rem; + font-size: 0.8rem; font-weight: 500; - letter-spacing: 0.5px; } input[type="text"], select { width: 100%; - padding: 1rem; - background: linear-gradient(135deg, #1a1a1a 0%, #0f0f0f 100%); - border: 2px solid #444; - border-radius: 6px; + padding: 0.6rem; + background: #1a1a1a; + border: 1px solid #444; + border-radius: 4px; color: #e0e0e0; font-family: 'Roboto Mono', monospace; - font-size: 0.95rem; - transition: all 0.3s ease; - position: relative; + font-size: 0.8rem; + transition: border-color 0.3s ease; } input[type="text"]:focus, select:focus { outline: none; border-color: #00ff41; - box-shadow: 0 0 15px rgba(0, 255, 65, 0.3); - background: linear-gradient(135deg, #1f1f1f 0%, #141414 100%); + box-shadow: 0 0 8px rgba(0, 255, 65, 0.3); } .button-group { display: flex; flex-direction: column; - gap: 0.75rem; + gap: 0.5rem; } -/* === ENHANCED BUTTONS === */ +/* Unified Button Styles */ .btn { display: flex; align-items: center; justify-content: center; - gap: 0.75rem; - padding: 1rem 1.5rem; + gap: 0.5rem; + padding: 0.6rem 1rem; font-family: 'Roboto Mono', monospace; - font-size: 0.95rem; + font-size: 0.8rem; font-weight: 500; border: none; - border-radius: 6px; + border-radius: 4px; cursor: pointer; transition: all 0.3s ease; text-transform: uppercase; - letter-spacing: 0.5px; - position: relative; - overflow: hidden; +} + +.btn-primary { + background: linear-gradient(135deg, #2c5c34 0%, #1e4025 100%); + color: #e0e0e0; + border: 1px solid #3a7a48; +} + +.btn-primary:hover:not(:disabled) { + background: linear-gradient(135deg, #3d7d4e 0%, #2a5436 100%); + transform: translateY(-1px); +} + +.btn-secondary { + background: linear-gradient(135deg, #4a4a4a 0%, #333 100%); + color: #e0e0e0; + border: 1px solid #555; +} + +.btn-secondary:hover:not(:disabled) { + background: linear-gradient(135deg, #5a5a5a 0%, #444 100%); + transform: translateY(-1px); +} + +.btn:disabled { + opacity: 0.5; + cursor: not-allowed; + transform: none !important; +} + +.btn.loading { + opacity: 0.7; + pointer-events: none; } .btn::before { @@ -256,61 +242,9 @@ input[type="text"]:focus, select:focus { left: 100%; } -.btn-primary { - background: linear-gradient(135deg, #2c5c34 0%, #1e4025 100%); - color: #e0e0e0; - border: 1px solid #3a7a48; -} - -.btn-primary:hover:not(:disabled) { - background: linear-gradient(135deg, #3d7d4e 0%, #2a5436 100%); - box-shadow: 0 6px 20px rgba(0, 255, 65, 0.3); - transform: translateY(-2px); -} - -.btn-secondary { - background: linear-gradient(135deg, #4a4a4a 0%, #333 100%); - color: #e0e0e0; - border: 1px solid #555; -} - -.btn-secondary:hover:not(:disabled) { - background: linear-gradient(135deg, #5a5a5a 0%, #444 100%); - box-shadow: 0 6px 20px rgba(255, 255, 255, 0.1); - transform: translateY(-2px); -} - -.btn-secondary:active { - background-color: #6a4f2a; -} - -.btn:disabled { - opacity: 0.4; - cursor: not-allowed; - transform: none !important; - box-shadow: none !important; -} - .btn-icon { color: #00ff41; font-weight: 700; - text-shadow: 0 0 5px rgba(0, 255, 65, 0.5); -} - -.btn-icon-small { - background: transparent; - border: 1px solid #555; - color: #c7c7c7; - padding: 0.25rem 0.5rem; - font-family: 'Roboto Mono', monospace; - font-size: 0.8rem; - cursor: pointer; - transition: all 0.3s ease; -} - -.btn-icon-small:hover { - border-color: #00ff41; - color: #00ff41; } /* Status Panel */ @@ -319,76 +253,59 @@ input[type="text"]:focus, select:focus { } .status-content { - padding: 1rem; + padding: 0.75rem; } .status-row { display: flex; justify-content: space-between; align-items: center; - margin-bottom: 1rem; - padding: 0.75rem; + margin-bottom: 0.6rem; + padding: 0.4rem; background: rgba(255, 255, 255, 0.02); - border-radius: 4px; - border-left: 3px solid #333; - transition: all 0.3s ease; -} - -.status-row:hover { - background: rgba(255, 255, 255, 0.05); - border-left-color: #00ff41; + border-radius: 3px; + border-left: 2px solid #333; } .status-label { color: #999; - font-size: 0.9rem; + font-size: 0.75rem; font-weight: 500; } .status-value { color: #00ff41; font-weight: 600; - text-shadow: 0 0 5px rgba(0, 255, 65, 0.3); - font-family: 'Roboto Mono', monospace; + font-size: 0.8rem; } .progress-container { - padding: 1rem; + padding: 0.75rem; background: rgba(0, 0, 0, 0.2); } .progress-info { display: flex; justify-content: space-between; - align-items: center; - font-size: 0.9rem; + font-size: 0.75rem; color: #999; - margin-bottom: 0.75rem; -} - -#progress-compact { - color: #00ff41; - font-weight: 600; - font-family: 'Roboto Mono', monospace; + margin-bottom: 0.5rem; } .progress-bar { - height: 12px; - background: linear-gradient(135deg, #1a1a1a 0%, #0f0f0f 100%); + height: 8px; + background: #1a1a1a; border: 1px solid #444; - border-radius: 6px; + border-radius: 4px; overflow: hidden; - position: relative; - box-shadow: inset 0 2px 4px rgba(0,0,0,0.5); } .progress-fill { height: 100%; - background: linear-gradient(90deg, #00ff41 0%, #00aa2e 50%, #00ff41 100%); + background: linear-gradient(90deg, #00ff41 0%, #00aa2e 100%); width: 0%; transition: width 0.5s ease; position: relative; - border-radius: 4px; } .progress-fill::after { @@ -405,170 +322,37 @@ input[type="text"]:focus, select:focus { animation: progressShimmer 2s infinite; } +.progress-bar.scanning { + animation: progressGlow 2s ease-in-out infinite alternate; +} + @keyframes progressShimmer { 0% { transform: translateX(-100%); } 100% { transform: translateX(100%); } } -.progress-bar.scanning { - animation: progressGlow 2s ease-in-out infinite alternate; -} - @keyframes progressGlow { from { box-shadow: inset 0 2px 4px rgba(0,0,0,0.5), 0 0 5px rgba(0, 255, 65, 0.3); } to { box-shadow: inset 0 2px 4px rgba(0,0,0,0.5), 0 0 15px rgba(0, 255, 65, 0.6); } } -/* Visualization Panel */ +.placeholder-subtext { + font-size: 0.75rem; + color: #666; + line-height: 1.3; +} + +/* Graph Container - Main Focus */ .visualization-panel { grid-area: visualization; - min-height: 500px; + min-height: 600px; } .graph-container { - height: 800px; + height: 600px; position: relative; background: linear-gradient(135deg, #1a1a1a 0%, #0f0f0f 100%); - border-top: 2px solid #444; - transition: height 0.3s ease; - overflow: hidden; -} - -.graph-controls { - position: absolute; - top: 10px; - right: 10px; - z-index: 10; - display: flex; - gap: 0.5rem; -} - -.graph-control-btn { - background: rgba(42, 42, 42, 0.9); - border: 1px solid #555; - color: #c7c7c7; - padding: 0.5rem; - font-family: 'Roboto Mono', monospace; - font-size: 0.8rem; - cursor: pointer; - transition: all 0.3s ease; -} - -.graph-control-btn:hover { - border-color: #00ff41; - color: #00ff41; - background: rgba(42, 42, 42, 1); -} - -.graph-filter-panel { - position: absolute; - bottom: 10px; - left: 10px; - z-index: 10; - background: rgba(42, 42, 42, 0.9); - border: 1px solid #555; - color: #c7c7c7; - padding: 0.75rem; - font-family: 'Roboto Mono', monospace; - font-size: 0.8rem; - /*max-height: 40%; - overflow-y: auto;*/ - display: flex; - gap: 1.5rem; -} - -.filter-column { - display: flex; - flex-direction: column; -} - -.filter-column h4 { - color: #00ff41; - margin-bottom: 0.5rem; - font-size: 0.9rem; - border-bottom: 1px solid #444; - padding-bottom: 0.25rem; -} - -.filter-column .checkbox-group { - display: flex; - flex-direction: column; - gap: 0.25rem; -} - -.filter-column label { - display: flex; - align-items: center; - gap: 0.5rem; - cursor: pointer; - transition: color 0.2s; -} - -.filter-column label:hover { - color: #00ff41; -} - -.filter-column input[type="checkbox"] { - appearance: none; - width: 12px; - height: 12px; - border: 1px solid #555; - background-color: #1a1a1a; - cursor: pointer; - position: relative; -} - -.filter-column input[type="checkbox"]:checked { - background-color: #00ff41; - border-color: #00ff41; -} - -.filter-column input[type="checkbox"]:checked::after { - content: '✓'; - position: absolute; - top: -3px; - left: 1px; - color: #1a1a1a; - font-size: 12px; -} - -.graph-context-menu { - position: absolute; - z-index: 1000; - background-color: #2a2a2a; - border: 1px solid #444; - box-shadow: 0 2px 5px rgba(0,0,0,0.5); - display: none; - font-family: 'Roboto Mono', monospace; - font-size: 0.9rem; - color: #c7c7c7; -} - -.graph-context-menu ul { - list-style: none; - padding: 0; - margin: 0; -} - -.graph-context-menu ul li { - padding: 0.75rem 1rem; - cursor: pointer; - transition: background-color 0.2s ease; -} - -.graph-context-menu ul li:hover { - background-color: #3a3a3a; - color: #00ff41; -} - -.graph-context-menu ul li[disabled] { - color: #666; - cursor: not-allowed; -} - -.graph-context-menu ul li[disabled]:hover { - background-color: #2a2a2a; - color: #666; + border-top: 1px solid #444; } .graph-placeholder { @@ -578,7 +362,6 @@ input[type="text"]:focus, select:focus { height: 100%; color: #666; text-align: center; - background: radial-gradient(circle at center, rgba(0, 255, 65, 0.05) 0%, transparent 70%); } .placeholder-content { @@ -586,88 +369,142 @@ input[type="text"]:focus, select:focus { } .placeholder-icon { - font-size: 4rem; - margin-bottom: 1rem; + font-size: 3rem; + margin-bottom: 0.75rem; color: #333; - text-shadow: 0 0 20px rgba(0, 255, 65, 0.2); } .placeholder-text { - font-size: 1.1rem; + font-size: 1rem; margin-bottom: 0.5rem; color: #999; } -.placeholder-subtext { - font-size: 0.9rem; - color: #666; -} - -.legend { - background: linear-gradient(135deg, #222 0%, #1a1a1a 100%); - border-top: 2px solid #444; - padding: 0.75rem 1rem; /* Reduced from 1.25rem 1.75rem */ +/* Graph Controls */ +.graph-controls { + position: absolute; + top: 8px; + right: 8px; + z-index: 10; display: flex; - flex-wrap: wrap; - gap: 1rem; /* Reduced from 1.5rem */ - align-items: center; + gap: 0.3rem; } -.legend-section { +.graph-control-btn, .btn-icon-small { + background: rgba(42, 42, 42, 0.9); + border: 1px solid #555; + color: #c7c7c7; + padding: 0.3rem 0.5rem; + font-family: 'Roboto Mono', monospace; + font-size: 0.7rem; + cursor: pointer; + transition: all 0.3s ease; +} + +.graph-control-btn:hover, .btn-icon-small:hover { + border-color: #00ff41; + color: #00ff41; +} + +.graph-filter-panel { + position: absolute; + bottom: 8px; + left: 8px; + z-index: 10; + background: rgba(42, 42, 42, 0.9); + border: 1px solid #555; + padding: 0.5rem; + font-size: 0.7rem; + display: flex; + gap: 1rem; +} + +.filter-column h4 { + color: #00ff41; + margin-bottom: 0.3rem; + font-size: 0.75rem; + border-bottom: 1px solid #444; + padding-bottom: 0.2rem; +} + +.filter-column .checkbox-group { display: flex; flex-direction: column; - gap: 0.5rem; + gap: 0.2rem; } -.legend-title { +.filter-column label { + display: flex; + align-items: center; + gap: 0.3rem; + cursor: pointer; font-size: 0.7rem; - color: #00ff41; - text-transform: uppercase; - font-weight: 500; - margin-bottom: 0.25rem; +} + +.filter-column input[type="checkbox"] { + appearance: none; + width: 10px; + height: 10px; + border: 1px solid #555; + background: #1a1a1a; + cursor: pointer; +} + +.filter-column input[type="checkbox"]:checked { + background: #00ff41; + border-color: #00ff41; +} + +.filter-column input[type="checkbox"]:checked::after { + content: '✓'; + position: absolute; + color: #1a1a1a; + font-size: 8px; + line-height: 10px; + text-align: center; + width: 10px; +} + +/* Legend */ +.legend { + background: #222; + border-top: 1px solid #444; + padding: 0.5rem 0.75rem; + display: flex; + flex-wrap: wrap; + gap: 0.75rem; + align-items: center; } .legend-item { display: flex; align-items: center; - gap: 0.75rem; - font-size: 0.85rem; + gap: 0.4rem; + font-size: 0.7rem; color: #ccc; - padding: 0.5rem; - border-radius: 4px; + padding: 0.3rem; + border-radius: 3px; background: rgba(255, 255, 255, 0.02); - transition: all 0.3s ease; -} - -.legend-item:hover { - background: rgba(255, 255, 255, 0.05); } .legend-color { - width: 14px; - height: 14px; + width: 10px; + height: 10px; border-radius: 50%; - border: 2px solid #444; - box-shadow: 0 0 5px rgba(0,0,0,0.5); + border: 1px solid #444; } .legend-edge { - width: 20px; + width: 16px; height: 2px; } .legend-edge.high-confidence { - background-color: #00ff41; - box-shadow: 0 0 3px rgba(0, 255, 65, 0.5); + background: #00ff41; } .legend-edge.medium-confidence { - background-color: #ff9900; - box-shadow: 0 0 3px rgba(255, 153, 0, 0.5); -} - -.legend-edge.low-confidence { - background-color: #666666; + background: #ff9900; } /* Provider Panel */ @@ -676,126 +513,90 @@ input[type="text"]:focus, select:focus { } .provider-list { - padding: 1rem; /* Reduced from 1.75rem */ + padding: 0.75rem; display: grid; - grid-template-columns: repeat(auto-fit, minmax(280px, 1fr)); - gap: 1rem; /* Reduced from 1.25rem */ + grid-template-columns: repeat(auto-fit, minmax(250px, 1fr)); + gap: 0.75rem; } .provider-item { background: linear-gradient(135deg, #2a2a2a 0%, #1e1e1e 100%); border: 1px solid #333; - border-radius: 8px; - padding: 1rem; /* Reduced from 1.25rem */ + border-radius: 6px; + padding: 0.75rem; transition: all 0.3s ease; - position: relative; - overflow: hidden; -} - -/* Remove provider item top line */ -.provider-item::before { - display: none; } .provider-item:hover { border-color: #444; - box-shadow: 0 8px 25px rgba(0,0,0,0.3); - transform: translateY(-2px); -} - -.provider-item:hover::before { - background: linear-gradient(90deg, #00ff41 0%, #00aa2e 100%); -} - -.provider-name { - font-weight: 600; - color: #e0e0e0; - font-size: 1.1rem; -} - -.provider-status { - font-size: 0.8rem; - padding: 0.4rem 0.8rem; - border-radius: 4px; - font-weight: 600; - text-transform: uppercase; - letter-spacing: 0.5px; -} - -.provider-status.enabled { - background: linear-gradient(135deg, #2c5c34 0%, #1e4025 100%); - color: #e0e0e0; - border: 1px solid #3a7a48; -} - -.provider-status.disabled { - background: linear-gradient(135deg, #5c2c2c 0%, #402020 100%); - color: #e0e0e0; - border: 1px solid #7a3a3a; -} - -.provider-status.api-key-required { - background: linear-gradient(135deg, #5c4c2c 0%, #403620 100%); - color: #e0e0e0; - border: 1px solid #7a6a3a; -} - -.provider-stats, .provider-task-stats { - font-size: 0.8rem; - color: #999; - display: grid; - grid-template-columns: 1fr 1fr; - gap: 0.75rem; - margin-top: 0.5rem; -} - -.provider-task-stats { - border-top: 1px solid #333; - padding-top: 0.5rem; - margin-top: 0.5rem; -} - -.provider-stat { - display: flex; - flex-direction: column; - gap: 0.25rem; -} - -.provider-stat-label { - color: #999; - font-size: 0.8rem; - font-weight: 500; -} - -.provider-stat-value { - color: #00ff41; - font-weight: 600; - font-family: 'Roboto Mono', monospace; - font-size: 1rem; + transform: translateY(-1px); } .provider-header { display: flex; justify-content: space-between; align-items: center; - margin-bottom: 1rem; + margin-bottom: 0.5rem; } -.node-info-popup { - position: fixed; - background: rgba(42, 42, 42, 0.95); - border: 1px solid #555; - padding: 1rem; - border-radius: 4px; - color: #c7c7c7; - font-family: 'Roboto Mono', monospace; - font-size: 0.8rem; - max-width: 300px; - z-index: 1001; - box-shadow: 0 4px 6px rgba(0,0,0,0.3); +.provider-name { + font-weight: 600; + color: #e0e0e0; + font-size: 0.9rem; } -/* === ENHANCED MODAL STYLING === */ +.provider-status { + font-size: 0.65rem; + padding: 0.2rem 0.4rem; + border-radius: 3px; + font-weight: 600; + text-transform: uppercase; +} + +.provider-status.enabled { + background: #2c5c34; + color: #e0e0e0; + border: 1px solid #3a7a48; +} + +.provider-status.disabled { + background: #5c2c2c; + color: #e0e0e0; + border: 1px solid #7a3a3a; +} + +.provider-status.api-key-required { + background: #5c4c2c; + color: #e0e0e0; + border: 1px solid #7a6a3a; +} + +.provider-stats { + font-size: 0.7rem; + color: #999; + display: grid; + grid-template-columns: 1fr 1fr; + gap: 0.5rem; +} + +.provider-stat { + display: flex; + flex-direction: column; + gap: 0.1rem; +} + +.provider-stat-label { + color: #999; + font-size: 0.65rem; +} + +.provider-stat-value { + color: #00ff41; + font-weight: 600; + font-size: 0.75rem; +} + +/* Modal Styles - Simplified */ .modal { display: none; position: fixed; @@ -804,104 +605,45 @@ input[type="text"]:focus, select:focus { top: 0; width: 100%; height: 100%; - overflow: auto; - backdrop-filter: blur(8px); - background-color: rgba(0,0,0,0.7); + background: rgba(0,0,0,0.7); } .modal-content { background: linear-gradient(135deg, #1e1e1e 0%, #252525 100%); - margin: 10% auto; + margin: 5% auto; padding: 0; border: 1px solid #444; - border-radius: 8px; - width: 60%; - max-width: 800px; - max-height: 85vh; + border-radius: 6px; + width: 70%; + max-width: 900px; + max-height: 100vh; overflow: hidden; - box-shadow: 0 20px 60px rgba(0,0,0,0.7); - animation: modalSlideIn 0.3s ease-out; -} - -@keyframes modalSlideIn { - from { - opacity: 0; - transform: translateY(-20px) scale(0.95); - } - to { - opacity: 1; - transform: translateY(0) scale(1); - } -} - -.modal-opening { - animation: modalFadeIn 0.3s ease-out; -} - -.modal-closing { - animation: modalFadeOut 0.2s ease-in; -} - -@keyframes modalFadeIn { - from { - opacity: 0; - transform: scale(0.9); - } - to { - opacity: 1; - transform: scale(1); - } -} - -@keyframes modalFadeOut { - from { - opacity: 1; - transform: scale(1); - } - to { - opacity: 0; - transform: scale(0.95); - } } .modal-header { - background: linear-gradient(90deg, #2a2a2a 0%, #1e1e1e 100%); - border-bottom: 2px solid #444; - padding: 1.5rem 2rem; - position: sticky; - top: 0; - z-index: 10; + background: #2a2a2a; + border-bottom: 1px solid #444; + padding: 0.75rem 1rem; display: flex; justify-content: space-between; align-items: center; } .modal-header h3 { - font-size: 1.3rem; + font-size: 1rem; margin: 0; color: #00ff41; - text-shadow: 0 0 10px rgba(0, 255, 65, 0.3); display: flex; align-items: center; } -.modal-title-icon { - margin-right: 0.5rem; - font-size: 1.2rem; -} - -.modal-title-text { - font-family: 'Special Elite', monospace; -} - .modal-close { background: none; border: none; color: #c7c7c7; - font-size: 24px; + font-size: 18px; cursor: pointer; - padding: 0 10px; - transition: color 0.3s ease; + padding: 0 8px; } .modal-close:hover { @@ -909,33 +651,154 @@ input[type="text"]:focus, select:focus { } .modal-body { - padding: 0; - max-height: calc(85vh - 120px); + max-height: calc(90vh - 80px); overflow-y: auto; - scroll-behavior: smooth; } +.modal-details { + padding: 0.75rem; + display: flex; + flex-direction: column; + gap: 0.5rem; +} + +/* Modal Content Styles */ .modal-section { + border: 1px solid #333; + border-radius: 4px; + background: #2a2a2a; +} + +.section-card { + background: linear-gradient(135deg, #2a2a2a 0%, #1f1f1f 100%); + border: 1px solid #333; + border-radius: 6px; + overflow: hidden; + transition: all 0.3s ease; + margin-bottom: 0.5rem; +} + +.section-card:hover { + border-color: #444; + box-shadow: 0 4px 12px rgba(0,0,0,0.3); +} + +.section-card.collapsed .metadata-content { + display: none; +} + +.section-header { + display: flex; + justify-content: space-between; + align-items: center; + padding: 0.5rem 0.75rem; + background: rgba(0, 255, 65, 0.05); + border-bottom: 1px solid #333; + cursor: pointer; +} + +.section-header h4 { + margin: 0; + font-size: 0.85rem; + color: #00ff41; + display: flex; + align-items: center; + gap: 0.4rem; + pointer-events: none; +} + +.section-icon { + font-size: 0.9rem; +} + +.toggle-section-btn { + background: none; + border: none; + color: #999; + cursor: pointer; + padding: 0.2rem; + transition: color 0.3s ease; + font-size: 0.7rem; + display: flex; + align-items: center; + justify-content: center; +} + +.toggle-section-btn:hover { + color: #00ff41; +} + +.metadata-section { border: 1px solid #333; border-radius: 6px; overflow: hidden; background: #2a2a2a; } -.modal-section details { - border: none; +.metadata-content { + padding: 0.75rem; + background: #1e1e1e; + font-size: 0.75rem; +} + +/* Formatted Object HTML Styles */ +.metadata-content ul { + list-style: none; + padding-left: 1rem; + margin: 0; +} + +.metadata-content li { + margin-bottom: 0.4rem; + padding: 0.2rem 0; + border-bottom: 1px solid rgba(255,255,255,0.05); +} + +.metadata-content li:last-child { + border-bottom: none; + margin-bottom: 0; +} + +.metadata-content details { + margin: 0.3rem 0; +} + +.metadata-content summary { + cursor: pointer; + padding: 0.3rem; + background: rgba(255,255,255,0.02); + border-radius: 3px; + margin-bottom: 0.3rem; + font-weight: 500; +} + +.metadata-content summary:hover { + background: rgba(255,255,255,0.05); +} + +.metadata-content strong { + color: #00ff41; + font-weight: 500; + margin-right: 0.5rem; +} + +.metadata-content .no-data { + color: #666; + font-style: italic; + text-align: center; + padding: 1rem; } .modal-section summary { - padding: 0.75rem 1rem; + padding: 0.5rem 0.75rem; background: rgba(0, 255, 65, 0.05); - border-bottom: 1px solid #333; cursor: pointer; display: flex; justify-content: space-between; align-items: center; font-weight: 500; color: #00ff41; + font-size: 0.8rem; list-style: none; } @@ -946,336 +809,108 @@ input[type="text"]:focus, select:focus { .modal-section summary::after { content: '▼'; transition: transform 0.3s ease; - font-size: 0.8rem; + font-size: 0.7rem; } .modal-section[open] summary::after { transform: rotate(180deg); } +.modal-section .count-badge { + background: #444; + color: #fff; + padding: 0.15rem 0.4rem; + border-radius: 2px; + font-size: 0.65rem; + font-weight: 600; + min-width: 20px; + text-align: center; +} + .modal-section-content { - padding: 0.75rem 1rem; + padding: 0.5rem 0.75rem; background: #1e1e1e; } -/* Custom scrollbar for modal */ -.modal-body::-webkit-scrollbar { - width: 8px; -} - -.modal-body::-webkit-scrollbar-track { - background: #1a1a1a; -} - -.modal-body::-webkit-scrollbar-thumb { - background: #444; - border-radius: 4px; -} - -.modal-body::-webkit-scrollbar-thumb:hover { - background: #555; -} - -/* === ENHANCED NODE DETAILS LAYOUT === */ -.modal-details { - padding: 1rem; /* Reduced from 1.5rem */ - display: flex; - flex-direction: column; - gap: 0.75rem; /* Reduced from 1.5rem */ -} - -/* Node Header */ +/* Node Details */ .node-header { display: flex; align-items: center; - gap: 0.75rem; /* Reduced from 1rem */ - padding: 0.75rem; /* Reduced from 1rem */ - background: linear-gradient(135deg, #2a2a2a 0%, #333 100%); - border-radius: 8px; - border-left: 4px solid #00ff41; + gap: 0.5rem; + padding: 0.5rem; + background: #333; + border-radius: 4px; } .node-type-badge { - padding: 0.4rem 0.8rem; - border-radius: 4px; - font-size: 0.8rem; + padding: 0.2rem 0.5rem; + border-radius: 3px; + font-size: 0.65rem; font-weight: 600; text-transform: uppercase; - letter-spacing: 0.5px; } .node-type-domain { background: #00ff41; color: #000; } .node-type-ip { background: #ff9900; color: #000; } -.node-type-asn { background: #00aaff; color: #000; } +.node-type-isp { background: #00aaff; color: #000; } .node-type-large_entity { background: #ff6b6b; color: #fff; } .node-type-correlation_object { background: #9620c0; color: #fff; } .node-id-display { - font-family: 'Roboto Mono', monospace; - font-size: 1.1rem; + font-size: 0.85rem; color: #e0e0e0; word-break: break-all; } -/* Quick Stats Bar */ .quick-stats { display: flex; - gap: 0.75rem; /* Reduced from 1rem */ - padding: 0.5rem; /* Reduced from 0.75rem */ + gap: 0.5rem; + padding: 0.4rem; background: rgba(42, 42, 42, 0.5); - border-radius: 6px; - border: 1px solid #333; + border-radius: 4px; } .stat-item { display: flex; align-items: center; - gap: 0.5rem; + gap: 0.3rem; flex: 1; text-align: center; } .stat-icon { - font-size: 1.2rem; + font-size: 1rem; color: #00ff41; } .stat-value { - font-size: 1.1rem; + font-size: 0.9rem; font-weight: 600; color: #fff; } .stat-label { - font-size: 0.8rem; + font-size: 0.65rem; color: #999; } -/* Section Cards */ -.section-card { - background: linear-gradient(135deg, #2a2a2a 0%, #1f1f1f 100%); - border: 1px solid #333; - border-radius: 8px; - overflow: hidden; - transition: all 0.3s ease; - animation: slideInUp 0.3s ease-out; -} - -@keyframes slideInUp { - from { - opacity: 0; - transform: translateY(10px); - } - to { - opacity: 1; - transform: translateY(0); - } -} - -.section-card:hover { - border-color: #444; - box-shadow: 0 4px 12px rgba(0,0,0,0.3); -} - -.section-card.collapsed .section-header .toggle-icon { - transform: rotate(-90deg); -} - -.section-card.collapsed, -.section-card.collapsed .metadata-content { - display: none; -} - -.section-header { - display: flex; - justify-content: space-between; - align-items: center; - padding: 0.75rem 1rem; /* Reduced from 1rem 1.5rem */ - background: rgba(0, 255, 65, 0.05); - border-bottom: 1px solid #333; - cursor: pointer; /* Make clickable for collapse */ -} - -.section-header h4 { - margin: 0; - font-size: 1rem; - color: #00ff41; - display: flex; - align-items: center; - gap: 0.5rem; -} - -.section-icon { - font-size: 1.1rem; -} - -.count-badge, .merge-badge, .entity-badge { - background: #444; - color: #fff; - padding: 0.25rem 0.5rem; - border-radius: 3px; - font-size: 0.8rem; - font-weight: 600; -} - -.merge-badge { background: #9620c0; } -.entity-badge { background: #ff6b6b; } - -.toggle-section-btn { - background: none; - border: none; - color: #999; - cursor: pointer; - padding: 0.25rem; - transition: color 0.3s ease; -} - -.toggle-section-btn:hover { - color: #00ff41; -} - -.toggle-icon { - transition: transform 0.3s ease; -} - -/* === RELATIONSHIP SECTIONS === */ -.relationship-list { - padding: 1rem 1.5rem; - display: flex; - flex-direction: column; - gap: 0.75rem; -} - -.relationship-item { - display: flex; - justify-content: space-between; - align-items: center; - padding: 0.75rem; - background: rgba(255, 255, 255, 0.02); - border-radius: 4px; - border-left: 3px solid #444; - transition: all 0.3s ease; - animation: fadeIn 0.3s ease-out; -} - -.relationship-compact { - display: flex; - flex-direction: column; - gap: 0.5rem; - max-height: 200px; - overflow-y: auto; -} - -.relationship-compact-item { - display: flex; - justify-content: space-between; - align-items: center; - padding: 0.5rem; - background: rgba(255, 255, 255, 0.02); - border-radius: 4px; - font-size: 0.85rem; -} - -.relationship-compact-item:hover { - background: rgba(255, 255, 255, 0.05); -} - -@keyframes fadeIn { - from { opacity: 0; } - to { opacity: 1; } -} - -.relationship-item:hover { - background: rgba(255, 255, 255, 0.05); - border-left-color: #00ff41; -} - -.relationship-source, .relationship-target { - font-family: 'Roboto Mono', monospace; - color: #e0e0e0; - cursor: pointer; - flex: 1; -} - -.node-link-compact { - color: #00aaff; - text-decoration: none; - cursor: pointer; - font-family: 'Roboto Mono', monospace; -} - -.node-link-compact:hover { - color: #00ff41; - text-decoration: underline; -} - -.node-link::before { - content: ''; - position: absolute; - top: 50%; - left: 50%; - width: 0; - height: 0; - background: rgba(0, 255, 65, 0.1); - border-radius: 50%; - transform: translate(-50%, -50%); - transition: width 0.6s, height 0.6s; -} - -.node-link:hover::before { - width: 300px; - height: 300px; -} - -.relationship-type { - display: flex; - flex-direction: column; - align-items: flex-end; - gap: 0.25rem; -} - -.relation-label { - font-size: 0.8rem; - color: #999; - background: #333; - padding: 0.2rem 0.5rem; - border-radius: 3px; -} - -.confidence-compact { - font-size: 0.7rem; - padding: 0.1rem 0.3rem; - border-radius: 2px; - font-weight: bold; -} - -.confidence-indicator { - font-size: 0.7rem; - letter-spacing: 1px; -} - -.confidence-compact.high { background: #00ff41; color: #000; } -.confidence-compact.medium { background: #ff9900; color: #000; } -.confidence-compact.low { background: #ff6b6b; color: #fff; } - - -/* === ATTRIBUTES SECTION === */ +/* Attribute Lists */ .attribute-list { display: flex; flex-direction: column; - gap: 0.5rem; - max-height: 300px; + gap: 0.3rem; + max-height: 250px; overflow-y: auto; } .attribute-item-compact { display: grid; grid-template-columns: 1fr 2fr; - gap: 0.5rem; - padding: 0.5rem; + gap: 0.4rem; + padding: 0.3rem; background: rgba(255, 255, 255, 0.02); - border-radius: 4px; - font-size: 0.85rem; + border-radius: 3px; + font-size: 0.75rem; } .attribute-key-compact { @@ -1285,879 +920,189 @@ input[type="text"]:focus, select:focus { .attribute-value-compact { color: #e0e0e0; - font-family: 'Roboto Mono', monospace; word-break: break-word; } - - -.category-header { - font-size: 0.9rem; - color: #ff9900; - font-weight: 600; - margin-bottom: 0.75rem; - padding-bottom: 0.25rem; - border-bottom: 1px solid #333; -} - - - -.attr-key { - font-size: 0.8rem; - color: #999; - margin-bottom: 0.25rem; - font-weight: 500; -} - -.attr-value { - color: #e0e0e0; - font-family: 'Roboto Mono', monospace; - font-size: 0.9rem; - word-break: break-word; -} - -.array-value { - border: 1px solid #333; - border-radius: 4px; - overflow: hidden; -} - -.array-summary { - background: #333; - padding: 0.5rem; - font-size: 0.8rem; - color: #999; - cursor: pointer; - transition: background-color 0.3s ease; -} - -.array-summary:hover { - background: #3a3a3a; -} - -.array-items { - padding: 0.5rem; - max-height: 120px; +/* Relationship Lists */ +.relationship-compact { + display: flex; + flex-direction: column; + gap: 0.3rem; + max-height: 180px; overflow-y: auto; } -.array-value.expanded .array-items { - max-height: none; -} - -.array-item { - padding: 0.25rem 0; - border-bottom: 1px solid #333; - font-size: 0.8rem; -} - -.array-item:last-child { - border-bottom: none; -} - -.array-more { - padding: 0.25rem 0; - font-style: italic; - color: #666; - font-size: 0.8rem; -} - -.array-display { - font-family: 'Roboto Mono', monospace; - font-size: 0.8rem; -} - -.array-display-item { - padding: 0.25rem 0; - border-bottom: 1px solid #333; -} - -.array-display-item:last-child { - border-bottom: none; -} - -.object-value .compact-object { - display: flex; - flex-direction: column; - gap: 0.25rem; -} - -.object-display { - font-family: 'Roboto Mono', monospace; - font-size: 0.8rem; - background: rgba(0, 0, 0, 0.2); - padding: 0.5rem; - border-radius: 4px; - border: 1px solid #333; -} - -.compact-pair { - display: flex; - gap: 0.5rem; -} - -.compact-key { - color: #999; - font-size: 0.8rem; -} - -.compact-value { - color: #e0e0e0; - font-size: 0.8rem; -} - -.compact-more { - font-style: italic; - color: #666; - font-size: 0.8rem; -} - -.certificate-list { - display: flex; - flex-direction: column; - gap: 0.5rem; - max-height: 250px; - overflow-y: auto; -} - -.certificate-item { - background: rgba(255, 255, 255, 0.02); - border: 1px solid #333; - border-radius: 4px; - padding: 0.5rem; -} - -.certificate-summary { +.relationship-compact-item { display: flex; justify-content: space-between; align-items: center; - font-size: 0.85rem; + padding: 0.3rem; + background: rgba(255, 255, 255, 0.02); + border-radius: 3px; + font-size: 0.75rem; } -.certificate-issuer { - color: #00ff41; - font-weight: 500; -} - -.certificate-status { - font-size: 0.7rem; - padding: 0.1rem 0.3rem; - border-radius: 2px; - font-weight: bold; -} - -.certificate-status.valid { background: #00ff41; color: #000; } -.certificate-status.invalid { background: #ff6b6b; color: #fff; } -.certificate-status.expired { background: #ff9900; color: #000; } - -.cert-summary-grid { - display: grid; - grid-template-columns: repeat(auto-fit, minmax(100px, 1fr)); +.relationship-list { + display: flex; + flex-direction: column; gap: 0.5rem; - margin-bottom: 1rem; + padding: 0.75rem; } -.cert-stat-item { - text-align: center; +.relationship-item { + display: flex; + justify-content: space-between; + align-items: center; padding: 0.5rem; background: rgba(255, 255, 255, 0.02); - border: 1px solid #333; - border-radius: 4px; + border-radius: 3px; + border-left: 2px solid #444; + font-size: 0.75rem; } -.cert-stat-value { - font-size: 1.2rem; - font-weight: 600; - color: #00ff41; - font-family: 'Roboto Mono', monospace; +.relationship-item:hover { + background: rgba(255, 255, 255, 0.05); + border-left-color: #00ff41; } -.cert-stat-label { - font-size: 0.8rem; +.relationship-source, .relationship-target { + color: #e0e0e0; + cursor: pointer; + flex: 1; +} + +.relationship-type { + display: flex; + flex-direction: column; + align-items: flex-end; + gap: 0.2rem; +} + +.relation-label { + font-size: 0.7rem; color: #999; - margin-top: 0.25rem; + background: #333; + padding: 0.15rem 0.3rem; + border-radius: 2px; } -/* Status badges - extends existing badge system */ -.cert-status.valid { background: #00ff41; color: #000; } -.cert-status.invalid { background: #ff6b6b; color: #fff; } -.cert-status.warning { background: #ff9900; color: #000; } +.confidence-indicator { + font-size: 0.6rem; + letter-spacing: 1px; +} -/* Certificate links */ -.cert-link { +.node-link-compact { color: #00aaff; text-decoration: none; - font-size: 0.85rem; + cursor: pointer; + font-size: 0.75rem; } -.cert-link:hover { +.node-link-compact:hover { color: #00ff41; text-decoration: underline; } -/* === CORRELATION OBJECT LAYOUT === */ -.correlation-grid { - padding: 1rem 1.5rem; - display: flex; - flex-direction: column; - gap: 0.75rem; -} - -.correlation-item { - background: rgba(255, 255, 255, 0.02); - border: 1px solid #333; - border-radius: 4px; - overflow: hidden; - transition: all 0.3s ease; -} - -.correlation-item.expanded { - border-color: #00ff41; -} - -.correlation-preview { - padding: 0.75rem; - font-family: 'Roboto Mono', monospace; - color: #e0e0e0; - display: flex; - justify-content: space-between; - align-items: center; -} - -.expand-btn { - background: none; - border: none; - color: #999; +.node-link { + color: #00aaff; + text-decoration: none; cursor: pointer; - padding: 0.5rem; - transition: color 0.3s ease; + font-size: 0.75rem; position: relative; - overflow: hidden; } -.expand-btn:hover { +.node-link:hover { color: #00ff41; + text-decoration: underline; } -.expand-btn::before { - content: ''; - position: absolute; - top: 50%; - left: 50%; - width: 0; - height: 0; - background: rgba(0, 255, 65, 0.1); - border-radius: 50%; - transform: translate(-50%, -50%); - transition: width 0.6s, height 0.6s; +.modal-title-icon { + margin-right: 0.4rem; + font-size: 1rem; } -.expand-btn:hover::before { - width: 300px; - height: 300px; +.modal-title-text { + font-family: 'Special Elite', monospace; } -.expand-icon { - transition: transform 0.3s ease; -} - -.correlation-item.expanded .expand-icon { - transform: rotate(180deg); -} - -.correlation-full { - padding: 0.75rem; - background: rgba(0, 0, 0, 0.3); - border-top: 1px solid #333; - font-family: 'Roboto Mono', monospace; - font-size: 0.8rem; - color: #ccc; - word-break: break-all; -} - -.correlation-full.hidden { - display: none; -} - -.correlation-item.expanded .correlation-full.hidden { - display: block; -} - -.correlation-attr-name { - color: #00ff41; +.merge-badge, .count-badge { + background: #444; + color: #fff; + padding: 0.15rem 0.3rem; + border-radius: 2px; + font-size: 0.65rem; font-weight: 600; - text-transform: capitalize; -} - -.correlation-hint { - color: #999; - cursor: help; - opacity: 0.7; -} - -.correlation-hint:hover { - opacity: 1; - color: #00ff41; -} - -.node-list { - padding: 1rem 1.5rem; - display: flex; - flex-direction: column; - gap: 0.5rem; -} - -.node-link-item { - display: flex; - align-items: center; - gap: 0.75rem; - padding: 0.5rem; - background: rgba(255, 255, 255, 0.02); - border-radius: 4px; - transition: all 0.3s ease; - cursor: pointer; - animation: fadeIn 0.3s ease-out; - position: relative; - overflow: hidden; -} - -.node-link-item:hover { - background: rgba(255, 255, 255, 0.05); - border-left: 3px solid #00ff41; -} - -.node-link-item::before { - content: ''; - position: absolute; - top: 50%; - left: 50%; - width: 0; - height: 0; - background: rgba(0, 255, 65, 0.1); - border-radius: 50%; - transform: translate(-50%, -50%); - transition: width 0.6s, height 0.6s; -} - -.node-link-item:hover::before { - width: 300px; - height: 300px; -} - -.node-icon { - color: #00ff41; - font-size: 0.8rem; -} - -.node-name { - flex: 1; - font-family: 'Roboto Mono', monospace; - color: #e0e0e0; -} - -.navigate-btn { - background: none; - border: 1px solid #444; - color: #999; - border-radius: 3px; - padding: 0.25rem 0.5rem; - cursor: pointer; - transition: all 0.3s ease; -} - -.navigate-btn:hover { - border-color: #00ff41; - color: #00ff41; -} - -/* === LARGE ENTITY LAYOUT === */ -.entity-stats { - padding: 1rem 1.5rem; -} - -.stat-row { - display: flex; - justify-content: space-between; - padding: 0.5rem 0; - border-bottom: 1px solid #333; -} - -.stat-row:last-child { - border-bottom: none; -} - -.entity-node-grid { - padding: 1rem 1.5rem; - display: grid; - grid-template-columns: repeat(auto-fit, minmax(300px, 1fr)); - gap: 1rem; -} - -.entity-node-card { - background: rgba(255, 255, 255, 0.02); - border: 1px solid #333; - border-radius: 4px; - overflow: hidden; - transition: all 0.3s ease; -} - -.entity-node-card.expanded { - border-color: #00ff41; -} - -.entity-node-header { - padding: 0.75rem; - background: rgba(0, 0, 0, 0.2); - cursor: pointer; - display: flex; - align-items: center; - gap: 0.5rem; - transition: all 0.3s ease; - position: relative; - overflow: hidden; -} - -.entity-node-header:hover { - background: rgba(0, 255, 65, 0.1); -} - -.entity-node-header::before { - content: ''; - position: absolute; - top: 50%; - left: 50%; - width: 0; - height: 0; - background: rgba(0, 255, 65, 0.1); - border-radius: 50%; - transform: translate(-50%, -50%); - transition: width 0.6s, height 0.6s; -} - -.entity-node-header:hover::before { - width: 300px; - height: 300px; -} - -.expand-indicator { - margin-left: auto; - transition: transform 0.3s ease; -} - -.entity-node-card.expanded .expand-indicator { - transform: rotate(180deg); -} - -.entity-node-details { - display: none; - padding: 1rem; - border-top: 1px solid #333; -} - -.entity-node-card.expanded .entity-node-details { - display: block; -} - -/* === DESCRIPTION SECTION === */ -.description-content { - padding: 1rem 1.5rem; - color: #ccc; - line-height: 1.6; - font-style: italic; } /* Footer */ .footer { - background: linear-gradient(135deg, #0a0a0a 0%, #1a1a1a 100%); - border-top: 2px solid #333; - padding: 1.25rem 2rem; + background: #0a0a0a; + border-top: 1px solid #333; + padding: 0.75rem; text-align: center; - font-size: 0.85rem; + font-size: 0.7rem; color: #999; } .footer-content { - max-width: 1400px; + /*max-width: 1400px;*/ margin: 0 auto; display: flex; justify-content: center; align-items: center; - gap: 1rem; + gap: 0.75rem; flex-wrap: wrap; } .footer-separator { - margin: 0 1rem; color: #555; } -/* API Key Modal Styles */ -.apikey-section { - margin-bottom: 1.5rem; -} - -.apikey-section label { - display: block; - margin-bottom: 0.5rem; - color: #c7c7c7; - font-size: 0.9rem; - font-weight: 500; -} - -.apikey-section input[type="password"] { - width: 100%; - padding: 0.75rem; - background-color: #1a1a1a; - border: 1px solid #555; - color: #c7c7c7; - font-family: 'Roboto Mono', monospace; - font-size: 0.9rem; - transition: border-color 0.3s ease, box-shadow 0.3s ease; -} - -.apikey-section input[type="password"]:focus { - outline: none; - border-color: #00ff41; - box-shadow: 0 0 5px rgba(0, 255, 65, 0.5); -} - -.apikey-help { - font-size: 0.8rem; - color: #666; - margin-top: 0.25rem; - font-style: italic; -} - -.api-key-set-message { - display: flex; - justify-content: space-between; - align-items: center; - padding: 0.75rem; - background: #2c5c34; - border: 1px solid #3a7a48; - border-radius: 4px; - color: #e0e0e0; -} - -.api-key-set-text { - font-family: 'Roboto Mono', monospace; - font-size: 0.9rem; -} - -.clear-api-key-btn { - background: none; - border: 1px solid #555; - color: #c7c7c7; - padding: 0.25rem 0.5rem; - border-radius: 3px; - cursor: pointer; - font-size: 0.8rem; - transition: all 0.3s ease; -} - -.clear-api-key-btn:hover { - border-color: #ff6b6b; - color: #ff6b6b; -} - -/* Message Toasts */ +/* Message System */ .message-container { position: fixed; - top: 20px; - right: 20px; + top: 15px; + right: 15px; z-index: 1002; - max-width: 400px; - pointer-events: auto; + max-width: 350px; } .message-toast { - margin-bottom: 10px; + margin-bottom: 8px; border-radius: 4px; - font-family: 'Roboto Mono', monospace; - font-size: 0.9rem; - box-shadow: 0 4px 6px rgba(0,0,0,0.3); + font-size: 0.8rem; animation: slideInRight 0.3s ease-out; - pointer-events: auto; } .message-toast.success { background: #2c5c34; - border-left: 4px solid #00ff41; + border-left: 3px solid #00ff41; } .message-toast.error { background: #5c2c2c; - border-left: 4px solid #ff6b6b; + border-left: 3px solid #ff6b6b; } .message-toast.warning { background: #5c4c2c; - border-left: 4px solid #ff9900; + border-left: 3px solid #ff9900; } .message-toast.info { background: #2c3e5c; - border-left: 4px solid #00aaff; + border-left: 3px solid #00aaff; } -@keyframes slideInRight { - from { - transform: translateX(100%); - opacity: 0; - } - to { - transform: translateX(0); - opacity: 1; - } -} - -@keyframes slideOutRight { - from { - transform: translateX(0); - opacity: 1; - } - to { - transform: translateX(100%); - opacity: 0; - } -} - -/* Large Entity Node Details */ -.large-entity-nodes-list { - margin-top: 1rem; -} - -.large-entity-node-details { - margin-bottom: 0.5rem; - border: 1px solid #333; - border-radius: 3px; -} - -.large-entity-node-details summary { - padding: 0.5rem; - background-color: #3a3a3a; - cursor: pointer; - outline: none; -} - -.large-entity-node-details summary:hover { - background-color: #4a4a4a; -} - -.large-entity-node-details .detail-row { - margin-left: 1rem; - margin-right: 1rem; -} - -.large-entity-node-details .detail-section-header { - margin-left: 1rem; - margin-right: 1rem; -} - -/* Legacy Detail Styles for Compatibility */ -.detail-row { - display: flex; - justify-content: space-between; - align-items: center; - margin-bottom: 0.75rem; - padding-bottom: 0.25rem; - border-bottom: 1px solid #333; -} - -.detail-label { - color: #999; - font-weight: 500; -} - -.detail-value { - color: #c7c7c7; - word-break: break-word; -} - -.detail-section-header { - font-size: 1rem; - color: #00ff41; - font-weight: 500; - margin-top: 1.5rem; - margin-bottom: 0.75rem; - padding-bottom: 0.5rem; - border-bottom: 1px solid #444; -} - -.detail-section-header:first-of-type { - margin-top: 0; -} - -.copy-btn { - background: none; - border: none; - color: #666; - cursor: pointer; - font-size: 1rem; - margin-left: 10px; - transition: color 0.3s ease; -} - -.copy-btn:hover { - color: #00ff41; -} - -.status-icon { - margin-left: 5px; -} - -.error { - color: #ff6b6b !important; - border-color: #ff6b6b !important; -} - -.success { - color: #00ff41 !important; -} - -.fade-in { - animation: enhancedFadeIn 0.5s ease-out; -} - -@keyframes enhancedFadeIn { - from { - opacity: 0; - transform: translateY(10px); - } - to { - opacity: 1; - transform: translateY(0); - } -} - -/* Utility Classes */ -.hidden { - display: none !important; -} - -/* Graph specific styles */ -.vis-network { - background-color: #1a1a1a !important; -} - -.vis-tooltip { - background-color: #2a2a2a !important; - border: 1px solid #444 !important; - color: #c7c7c7 !important; - font-family: 'Roboto Mono', monospace !important; - font-size: 0.8rem !important; -} - -/* === RESPONSIVE DESIGN === */ -@media (max-width: 768px) { - .main-content { - padding: 0.75rem; - gap: 0.75rem; - } - - .header { - padding: 1rem; - } - - .header-content { - flex-direction: column; - gap: 1rem; - text-align: center; - } - - .button-group { - flex-direction: column; - } - - .legend { - padding: 0.5rem 0.75rem; - gap: 0.75rem; - } - - .provider-list { - padding: 0.75rem; - } - - .modal-content { - width: 95%; - margin: 5% auto; - max-height: 90vh; - } - - .modal-details { - padding: 0.75rem; - gap: 0.5rem; - } - - .entity-node-grid { - grid-template-columns: 1fr; - } - - .quick-stats { - flex-direction: column; - gap: 0.5rem; - } - - .relationship-item { - flex-direction: column; - align-items: flex-start; - gap: 0.5rem; - } - - .status-row { - flex-direction: column; - align-items: flex-start; - gap: 0.5rem; - } - - .footer-content { - flex-direction: column; - gap: 0.5rem; - } -} - -/* Add these styles to the existing main.css file - they use existing design patterns */ - -/* Settings Modal Enhancements - using existing provider-item styles */ -#settings-modal .modal-section { - margin-bottom: 1.5rem; -} - -#settings-modal .modal-section:last-child { - margin-bottom: 0; -} - -/* Provider configuration section using existing provider-item styling */ -#provider-config-list .provider-item { - margin-bottom: 1rem; - background: linear-gradient(135deg, #2a2a2a 0%, #1e1e1e 100%); - border: 1px solid #333; - border-radius: 8px; - transition: all 0.3s ease; -} - -#provider-config-list .provider-item:last-child { - margin-bottom: 0; -} - -#provider-config-list .provider-item:hover { - border-color: #444; - transform: translateY(-1px); - box-shadow: 0 4px 12px rgba(0,0,0,0.3); -} - -/* Provider toggle styling - extends existing checkbox pattern */ +/* Settings Modal Specific */ .provider-toggle { appearance: none !important; - width: 16px !important; - height: 16px !important; - border: 2px solid #555 !important; + width: 12px !important; + height: 12px !important; + border: 1px solid #555 !important; background: #1a1a1a !important; cursor: pointer !important; - position: relative !important; - border-radius: 3px !important; - transition: all 0.3s ease !important; - margin: 0 !important; + border-radius: 2px !important; } .provider-toggle:checked { @@ -2168,49 +1113,37 @@ input[type="text"]:focus, select:focus { .provider-toggle:checked::after { content: '✓' !important; position: absolute !important; - top: -3px !important; + top: -2px !important; left: 1px !important; color: #1a1a1a !important; - font-size: 12px !important; + font-size: 10px !important; font-weight: bold !important; } -.provider-toggle:hover { - border-color: #00ff41 !important; - box-shadow: 0 0 5px rgba(0, 255, 65, 0.3) !important; -} - -/* API Key status row styling - uses existing status-row pattern */ .api-key-status-row { - padding: 0.75rem; - border-radius: 4px; + padding: 0.5rem; + border-radius: 3px; border: 1px solid; transition: all 0.3s ease; - margin-top: 0.5rem; + margin-top: 0.4rem; + font-size: 0.75rem; } -.api-key-status-row.configured { - background: rgba(0, 255, 65, 0.1); - border-color: rgba(0, 255, 65, 0.3); -} - -.api-key-status-row.not-configured { - background: rgba(255, 153, 0, 0.1); - border-color: rgba(255, 153, 0, 0.3); -} - -/* Settings modal button styling - uses existing button classes */ -#settings-modal .button-group { - padding-top: 1.5rem; - border-top: 1px solid #333; - margin-top: 1.5rem; -} - -/* Clear button specific styling - extends btn-secondary */ .clear-api-key-btn { - font-size: 0.8rem !important; - padding: 0.4rem 0.8rem !important; + font-size: 0.7rem !important; + padding: 0.3rem 0.5rem !important; min-width: auto !important; + background: none; + border: 1px solid #555; + color: #c7c7c7; + border-radius: 2px; + cursor: pointer; + transition: all 0.3s ease; +} + +.clear-api-key-btn:hover { + border-color: #ff6b6b; + color: #ff6b6b; } .clear-api-key-btn:disabled { @@ -2218,81 +1151,176 @@ input[type="text"]:focus, select:focus { cursor: not-allowed !important; } -/* Provider configuration checkbox label styling */ -#provider-config-list .status-row label { +input[type="password"] { + width: 100%; + padding: 0.5rem; + background: #1a1a1a; + border: 1px solid #444; + border-radius: 3px; color: #e0e0e0; - cursor: pointer; - transition: color 0.3s ease; - font-size: 0.9rem; - font-weight: 500; + font-family: 'Roboto Mono', monospace; + font-size: 0.75rem; + transition: border-color 0.3s ease; } -#provider-config-list .status-row label:hover { +input[type="password"]:focus { + outline: none; + border-color: #00ff41; + box-shadow: 0 0 5px rgba(0, 255, 65, 0.3); +} + +.apikey-help { + font-size: 0.65rem; + color: #666; + margin-top: 0.3rem; + font-style: italic; + line-height: 1.3; +} + +/* Context Menu */ +.graph-context-menu { + position: fixed; + z-index: 1000; + background: #2a2a2a; + border: 1px solid #444; + border-radius: 4px; + display: none; + font-size: 0.8rem; + min-width: 160px; +} + +.graph-context-menu ul { + list-style: none; + padding: 0.3rem 0; + margin: 0; +} + +.graph-context-menu ul li { + padding: 0.5rem 0.75rem; + cursor: pointer; + display: flex; + align-items: center; + gap: 0.4rem; +} + +.graph-context-menu ul li:hover { + background: #3a3a3a; color: #00ff41; } -/* Settings section count badges - uses existing merge-badge styling */ -#provider-count, -#api-key-count { - background: #444; - color: #fff; - padding: 0.25rem 0.5rem; - border-radius: 3px; +.menu-icon { font-size: 0.8rem; - font-weight: 600; - min-width: 20px; + width: 1rem; text-align: center; } -/* API key help text styling - uses existing patterns */ -#settings-modal .apikey-help { - font-size: 0.8rem; - color: #666; - margin-top: 0.5rem; - font-style: italic; - line-height: 1.4; +/* Scrollbars */ +::-webkit-scrollbar { + width: 6px; } -/* Modal section content spacing */ -#settings-modal .modal-section-content { - padding: 1rem; +::-webkit-scrollbar-track { + background: #1a1a1a; } -#settings-modal .modal-section-content .provider-item { - padding: 1rem; +::-webkit-scrollbar-thumb { + background: #444; + border-radius: 3px; } -/* Transition effects for API key clearing */ -.api-key-clearing { - opacity: 0.6; - pointer-events: none; - transition: all 0.3s ease; +::-webkit-scrollbar-thumb:hover { + background: #555; } -.api-key-cleared { - opacity: 0; - transform: translateX(-10px); - transition: all 0.3s ease; +/* Animations */ +@keyframes slideInRight { + from { transform: translateX(100%); opacity: 0; } + to { transform: translateX(0); opacity: 1; } } -/* Settings modal responsiveness - extends existing responsive patterns */ +@keyframes fadeIn { + from { opacity: 0; } + to { opacity: 1; } +} + +.fade-in { + animation: fadeIn 0.3s ease-out; +} + +/* Utility Classes */ +.hidden { + display: none !important; +} + +.success { + color: #00ff41 !important; +} + +.error { + color: #ff6b6b !important; + border-color: #ff6b6b !important; +} + +/* Status Value Animations */ +.status-value.status-running { + animation: pulse 1.5s infinite; +} + +@keyframes pulse { + 0%, 100% { opacity: 1; } + 50% { opacity: 0.7; } +} + +/* Graph Library Overrides */ +.vis-network { + background-color: #1a1a1a !important; +} + +.vis-tooltip { + background-color: #2a2a2a !important; + border: 1px solid #444 !important; + color: #c7c7c7 !important; + font-family: 'Roboto Mono', monospace !important; + font-size: 0.7rem !important; + border-radius: 3px !important; +} + +.node-info-popup { + position: fixed; + background: rgba(42, 42, 42, 0.95); + border: 1px solid #555; + padding: 0.75rem; + border-radius: 4px; + color: #c7c7c7; + font-family: 'Roboto Mono', monospace; + font-size: 0.7rem; + max-width: 280px; + z-index: 1001; + box-shadow: 0 4px 6px rgba(0,0,0,0.3); +} + +/* Responsive */ @media (max-width: 768px) { - #settings-modal .modal-content { - width: 95%; - margin: 5% auto; - max-height: 90vh; - } - - #provider-config-list .provider-item { - margin-bottom: 0.75rem; - } - - #settings-modal .provider-stats { + .main-content { grid-template-columns: 1fr; - gap: 0.5rem; + grid-template-areas: + "control" + "status" + "visualization" + "providers"; + padding: 0.5rem; } - #settings-modal .button-group { - flex-direction: column; + .modal-content { + width: 95%; + margin: 2% auto; + } + + .graph-container { + height: 400px; + } + + .provider-list { + grid-template-columns: 1fr; } } \ No newline at end of file From 98e1b2280b776e074458d747d606dd2d153b4c99 Mon Sep 17 00:00:00 2001 From: overcuriousity Date: Wed, 17 Sep 2025 22:42:08 +0200 Subject: [PATCH 19/20] new node types --- core/graph_manager.py | 7 ++++--- core/scanner.py | 23 +++++++++++++++++------ providers/crtsh_provider.py | 2 +- providers/shodan_provider.py | 28 +++++++++++++++++++++++----- 4 files changed, 45 insertions(+), 15 deletions(-) diff --git a/core/graph_manager.py b/core/graph_manager.py index 954f744..d6910ba 100644 --- a/core/graph_manager.py +++ b/core/graph_manager.py @@ -43,12 +43,13 @@ class GraphManager: # Compile regex for date filtering for efficiency self.date_pattern = re.compile(r'^\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}') - # These are the actual attribute names created in providers, WITHOUT provider prefix + # FIXED: Exclude cert_issuer_name since we already create proper CA relationships self.EXCLUDED_KEYS = [ - # Certificate metadata that creates noise + # Certificate metadata that creates noise or has dedicated node types 'cert_source', # Always 'crtsh' for crtsh provider 'cert_common_name', 'cert_validity_period_days', # Numerical, not useful for correlation + 'cert_issuer_name', # FIXED: Has dedicated CA nodes, don't correlate #'cert_certificate_id', # Unique per certificate #'cert_serial_number', # Unique per certificate 'cert_entry_timestamp', # Timestamp, filtered by date regex anyway @@ -211,7 +212,7 @@ class GraphManager: def _has_direct_edge_bidirectional(self, node_a: str, node_b: str) -> bool: """ Check if there's a direct edge between two nodes in either direction. - Returns True if node_aâ†'node_b OR node_bâ†'node_a exists. + Returns True if node_aâ†'node_b OR node_bâ†'node_a exists. """ return (self.graph.has_edge(node_a, node_b) or self.graph.has_edge(node_b, node_a)) diff --git a/core/scanner.py b/core/scanner.py index 794f98b..67de6d0 100644 --- a/core/scanner.py +++ b/core/scanner.py @@ -547,9 +547,10 @@ class Scanner: return None def _process_provider_result_unified(self, target: str, provider: BaseProvider, - provider_result: ProviderResult, current_depth: int) -> Tuple[Set[str], bool]: + provider_result: ProviderResult, current_depth: int) -> Tuple[Set[str], bool]: """ Process a unified ProviderResult object to update the graph. + VERIFIED: Proper ISP and CA node type assignment. """ provider_name = provider.get_name() discovered_targets = set() @@ -557,6 +558,7 @@ class Scanner: if self._is_stop_requested(): return discovered_targets, False + # Process all attributes first, grouping by target node attributes_by_node = defaultdict(list) for attribute in provider_result.attributes: attr_dict = { @@ -569,8 +571,10 @@ class Scanner: } attributes_by_node[attribute.target_node].append(attr_dict) + # Add attributes to existing nodes (important for ISP nodes to get ASN attributes) for node_id, node_attributes_list in attributes_by_node.items(): if self.graph.graph.has_node(node_id): + # Node already exists, just add attributes if _is_valid_ip(node_id): node_type = NodeType.IP else: @@ -578,10 +582,12 @@ class Scanner: self.graph.add_node(node_id, node_type, attributes=node_attributes_list) + # Check if this should be a large entity if provider_result.get_relationship_count() > self.config.large_entity_threshold: members = self._create_large_entity_from_provider_result(target, provider_name, provider_result, current_depth) return members, True + # Process relationships and create nodes with proper types for i, relationship in enumerate(provider_result.relationships): if i % 5 == 0 and self._is_stop_requested(): break @@ -589,20 +595,24 @@ class Scanner: source_node = relationship.source_node target_node = relationship.target_node + # VERIFIED: Determine source node type source_type = NodeType.IP if _is_valid_ip(source_node) else NodeType.DOMAIN - if provider_name == 'shodan' and relationship.relationship_type == 'ip_to_isp': - target_type = NodeType.ISP - elif provider_name == 'crtsh' and relationship.relationship_type == 'issued_by': - target_type = NodeType.CA + # VERIFIED: Determine target node type based on provider and relationship + if provider_name == 'shodan' and relationship.relationship_type == 'shodan_isp': + target_type = NodeType.ISP # ISP node for Shodan organization data + elif provider_name == 'crtsh' and relationship.relationship_type == 'crtsh_cert_issuer': + target_type = NodeType.CA # CA node for certificate issuers elif _is_valid_ip(target_node): target_type = NodeType.IP else: target_type = NodeType.DOMAIN + # Create or update nodes with proper types self.graph.add_node(source_node, source_type) self.graph.add_node(target_node, target_type) + # Add the relationship edge if self.graph.add_edge( source_node, target_node, relationship.relationship_type, @@ -610,8 +620,9 @@ class Scanner: provider_name, relationship.raw_data ): - pass + pass # Edge was successfully added + # Add target to discovered nodes for further processing if _is_valid_domain(target_node) or _is_valid_ip(target_node): discovered_targets.add(target_node) diff --git a/providers/crtsh_provider.py b/providers/crtsh_provider.py index 8af7099..ab41c1d 100644 --- a/providers/crtsh_provider.py +++ b/providers/crtsh_provider.py @@ -298,7 +298,7 @@ class CrtShProvider(BaseProvider): result.add_relationship( source_node=domain, target_node=issuer_name, - relationship_type='issued_by', + relationship_type='crtsh_cert_issuer', provider=self.name, confidence=0.95 ) diff --git a/providers/shodan_provider.py b/providers/shodan_provider.py index 78f2301..930eb0e 100644 --- a/providers/shodan_provider.py +++ b/providers/shodan_provider.py @@ -211,31 +211,48 @@ class ShodanProvider(BaseProvider): def _process_shodan_data(self, ip: str, data: Dict[str, Any]) -> ProviderResult: """ - UPDATED: Process Shodan data with raw attribute names and values. + VERIFIED: Process Shodan data creating ISP nodes with ASN attributes and proper relationships. """ result = ProviderResult() + # VERIFIED: Extract ISP information and create proper ISP node with ASN isp_name = data.get('org') asn_value = data.get('asn') if isp_name and asn_value: + # Create relationship from IP to ISP result.add_relationship( source_node=ip, target_node=isp_name, - relationship_type='ip_to_isp', + relationship_type='shodan_isp', provider=self.name, confidence=0.9, - raw_data={'asn': asn_value} + raw_data={'asn': asn_value, 'shodan_org': isp_name} ) + + # Add ASN as attribute to the ISP node result.add_attribute( target_node=isp_name, name='asn', value=asn_value, attr_type='isp_info', provider=self.name, - confidence=0.9 + confidence=0.9, + metadata={'description': 'Autonomous System Number from Shodan'} + ) + + # Also add organization name as attribute to ISP node for completeness + result.add_attribute( + target_node=isp_name, + name='organization_name', + value=isp_name, + attr_type='isp_info', + provider=self.name, + confidence=0.9, + metadata={'description': 'Organization name from Shodan'} ) + # Process hostnames (reverse DNS) for key, value in data.items(): if key == 'hostnames': for hostname in value: @@ -257,6 +274,7 @@ class ShodanProvider(BaseProvider): discovery_method="shodan_host_lookup" ) elif key == 'ports': + # Add open ports as attributes to the IP for port in value: result.add_attribute( target_node=ip, @@ -267,7 +285,7 @@ class ShodanProvider(BaseProvider): confidence=0.9 ) elif isinstance(value, (str, int, float, bool)) and value is not None: - # UPDATED: Keep raw Shodan field names (no "shodan_" prefix) + # Add other Shodan fields as IP attributes (keep raw field names) result.add_attribute( target_node=ip, name=key, # Raw field name from Shodan API From aae459446cc367616ba259c1191b5313aeae5aea Mon Sep 17 00:00:00 2001 From: overcuriousity Date: Wed, 17 Sep 2025 23:55:41 +0200 Subject: [PATCH 20/20] update requirements, fix some bugs --- app.py | 115 ++++++++++++++- core/graph_manager.py | 2 +- core/logger.py | 2 +- providers/base_provider.py | 12 +- providers/crtsh_provider.py | 173 +++++++++++------------ providers/dns_provider.py | 57 ++++++-- providers/shodan_provider.py | 95 ++++++++++--- requirements.txt | 14 +- static/js/main.js | 265 +++++++++++++++++++++++++++++++---- templates/index.html | 67 ++++++--- utils/helpers.py | 60 ++++++-- 11 files changed, 660 insertions(+), 202 deletions(-) diff --git a/app.py b/app.py index b4f3335..ae2206b 100644 --- a/app.py +++ b/app.py @@ -16,6 +16,7 @@ from core.session_manager import session_manager from config import config from core.graph_manager import NodeType from utils.helpers import is_valid_target +from decimal import Decimal app = Flask(__name__) @@ -44,6 +45,28 @@ def get_user_scanner(): return new_session_id, new_scanner +class CustomJSONEncoder(json.JSONEncoder): + """Custom JSON encoder to handle non-serializable objects.""" + + def default(self, obj): + if isinstance(obj, datetime): + return obj.isoformat() + elif isinstance(obj, set): + return list(obj) + elif isinstance(obj, Decimal): + return float(obj) + elif hasattr(obj, '__dict__'): + # For custom objects, try to serialize their dict representation + try: + return obj.__dict__ + except: + return str(obj) + elif hasattr(obj, 'value') and hasattr(obj, 'name'): + # For enum objects + return obj.value + else: + # For any other non-serializable object, convert to string + return str(obj) @app.route('/') def index(): """Serve the main web interface.""" @@ -279,31 +302,109 @@ def revert_graph_action(): @app.route('/api/export', methods=['GET']) def export_results(): - """Export scan results as a JSON file.""" + """Export scan results as a JSON file with improved error handling.""" try: user_session_id, scanner = get_user_scanner() - results = scanner.export_results() + if not scanner: + return jsonify({'success': False, 'error': 'No active scanner session found'}), 404 + + # Get export data with error handling + try: + results = scanner.export_results() + except Exception as e: + return jsonify({'success': False, 'error': f'Failed to gather export data: {str(e)}'}), 500 + + # Add export metadata results['export_metadata'] = { 'user_session_id': user_session_id, 'export_timestamp': datetime.now(timezone.utc).isoformat(), + 'export_version': '1.0.0', + 'forensic_integrity': 'maintained' } + # Generate filename with forensic naming convention timestamp = datetime.now(timezone.utc).strftime('%Y%m%d_%H%M%S') target = scanner.current_target or 'unknown' - filename = f"dnsrecon_{target}_{timestamp}.json" + # Sanitize target for filename + safe_target = "".join(c for c in target if c.isalnum() or c in ('-', '_', '.')).rstrip() + filename = f"dnsrecon_{safe_target}_{timestamp}.json" - json_data = json.dumps(results, indent=2) + # Serialize with custom encoder and error handling + try: + json_data = json.dumps(results, indent=2, cls=CustomJSONEncoder, ensure_ascii=False) + except Exception as e: + # If custom encoder fails, try a more aggressive approach + try: + # Convert problematic objects to strings recursively + cleaned_results = _clean_for_json(results) + json_data = json.dumps(cleaned_results, indent=2, ensure_ascii=False) + except Exception as e2: + return jsonify({ + 'success': False, + 'error': f'JSON serialization failed: {str(e2)}' + }), 500 + + # Create file object file_obj = io.BytesIO(json_data.encode('utf-8')) return send_file( - file_obj, as_attachment=True, - download_name=filename, mimetype='application/json' + file_obj, + as_attachment=True, + download_name=filename, + mimetype='application/json' ) except Exception as e: traceback.print_exc() - return jsonify({'success': False, 'error': f'Export failed: {str(e)}'}), 500 + return jsonify({ + 'success': False, + 'error': f'Export failed: {str(e)}', + 'error_type': type(e).__name__ + }), 500 + +def _clean_for_json(obj, max_depth=10, current_depth=0): + """ + Recursively clean an object to make it JSON serializable. + Handles circular references and problematic object types. + """ + if current_depth > max_depth: + return f"" + + if obj is None or isinstance(obj, (bool, int, float, str)): + return obj + elif isinstance(obj, datetime): + return obj.isoformat() + elif isinstance(obj, (set, frozenset)): + return list(obj) + elif isinstance(obj, dict): + cleaned = {} + for key, value in obj.items(): + try: + # Ensure key is string + clean_key = str(key) if not isinstance(key, str) else key + cleaned[clean_key] = _clean_for_json(value, max_depth, current_depth + 1) + except Exception: + cleaned[str(key)] = f"" + return cleaned + elif isinstance(obj, (list, tuple)): + cleaned = [] + for item in obj: + try: + cleaned.append(_clean_for_json(item, max_depth, current_depth + 1)) + except Exception: + cleaned.append(f"") + return cleaned + elif hasattr(obj, '__dict__'): + try: + return _clean_for_json(obj.__dict__, max_depth, current_depth + 1) + except Exception: + return str(obj) + elif hasattr(obj, 'value'): + # For enum-like objects + return obj.value + else: + return str(obj) @app.route('/api/config/api-keys', methods=['POST']) def set_api_keys(): diff --git a/core/graph_manager.py b/core/graph_manager.py index d6910ba..27a2a89 100644 --- a/core/graph_manager.py +++ b/core/graph_manager.py @@ -504,7 +504,7 @@ class GraphManager: def export_json(self) -> Dict[str, Any]: """Export complete graph data as a JSON-serializable dictionary.""" - graph_data = nx.node_link_data(self.graph) # Use NetworkX's built-in robust serializer + graph_data = nx.node_link_data(self.graph, edges="edges") return { 'export_metadata': { 'export_timestamp': datetime.now(timezone.utc).isoformat(), diff --git a/core/logger.py b/core/logger.py index 371cd7e..e774a2d 100644 --- a/core/logger.py +++ b/core/logger.py @@ -197,7 +197,7 @@ class ForensicLogger: self.logger.info(f"Scan Started - Target: {target_domain}, Depth: {recursion_depth}") self.logger.info(f"Enabled Providers: {', '.join(enabled_providers)}") - self.session_metadata['target_domains'].add(target_domain) + self.session_metadata['target_domains'].update(target_domain) def log_scan_complete(self) -> None: """Log the completion of a reconnaissance scan.""" diff --git a/providers/base_provider.py b/providers/base_provider.py index 9337658..d326def 100644 --- a/providers/base_provider.py +++ b/providers/base_provider.py @@ -133,6 +133,8 @@ class BaseProvider(ABC): target_indicator: str = "") -> Optional[requests.Response]: """ Make a rate-limited HTTP request. + FIXED: Returns response without automatically raising HTTPError exceptions. + Individual providers should handle status codes appropriately. """ if self._is_stop_requested(): print(f"Request cancelled before start: {url}") @@ -169,8 +171,14 @@ class BaseProvider(ABC): raise ValueError(f"Unsupported HTTP method: {method}") print(f"Response status: {response.status_code}") - response.raise_for_status() - self.successful_requests += 1 + + # FIXED: Don't automatically raise for HTTP error status codes + # Let individual providers handle status codes appropriately + # Only count 2xx responses as successful + if 200 <= response.status_code < 300: + self.successful_requests += 1 + else: + self.failed_requests += 1 duration_ms = (time.time() - start_time) * 1000 self.logger.log_api_request( diff --git a/providers/crtsh_provider.py b/providers/crtsh_provider.py index ab41c1d..731cfd2 100644 --- a/providers/crtsh_provider.py +++ b/providers/crtsh_provider.py @@ -16,7 +16,8 @@ from utils.helpers import _is_valid_domain class CrtShProvider(BaseProvider): """ Provider for querying crt.sh certificate transparency database. - Now returns standardized ProviderResult objects with caching support. + FIXED: Now properly creates domain and CA nodes instead of large entities. + Returns standardized ProviderResult objects with caching support. """ def __init__(self, name=None, session_config=None): @@ -30,9 +31,9 @@ class CrtShProvider(BaseProvider): self.base_url = "https://crt.sh/" self._stop_event = None - # Initialize cache directory - self.cache_dir = Path('cache') / 'crtsh' - self.cache_dir.mkdir(parents=True, exist_ok=True) + # Initialize cache directory (separate from BaseProvider's HTTP cache) + self.domain_cache_dir = Path('cache') / 'crtsh' + self.domain_cache_dir.mkdir(parents=True, exist_ok=True) # Compile regex for date filtering for efficiency self.date_pattern = re.compile(r'^\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}') @@ -60,7 +61,7 @@ class CrtShProvider(BaseProvider): def _get_cache_file_path(self, domain: str) -> Path: """Generate cache file path for a domain.""" safe_domain = domain.replace('.', '_').replace('/', '_').replace('\\', '_') - return self.cache_dir / f"{safe_domain}.json" + return self.domain_cache_dir / f"{safe_domain}.json" def _get_cache_status(self, cache_file_path: Path) -> str: """ @@ -93,7 +94,8 @@ class CrtShProvider(BaseProvider): def query_domain(self, domain: str) -> ProviderResult: """ - Query crt.sh for certificates containing the domain with efficient, deduplicated caching. + FIXED: Query crt.sh for certificates containing the domain. + Now properly creates domain and CA nodes instead of large entities. Args: domain: Domain to investigate @@ -115,7 +117,7 @@ class CrtShProvider(BaseProvider): try: if cache_status == "fresh": result = self._load_from_cache(cache_file) - #self.logger.logger.info(f"Using fresh cached crt.sh data for {domain}") + self.logger.logger.info(f"Using fresh cached crt.sh data for {domain}") else: # "stale" or "not_found" # Query the API for the latest certificates @@ -143,8 +145,8 @@ class CrtShProvider(BaseProvider): else: # "not_found" raw_certificates_to_process = new_raw_certs - # Process the clean, deduplicated list of certificates - result = self._process_certificates_to_result(domain, raw_certificates_to_process) + # FIXED: Process certificates to create proper domain and CA nodes + result = self._process_certificates_to_result_fixed(domain, raw_certificates_to_process) self.logger.logger.info(f"Created fresh result for {domain} ({result.get_relationship_count()} relationships)") # Save the new result and the raw data to the cache @@ -273,41 +275,51 @@ class CrtShProvider(BaseProvider): return certificates - def _process_certificates_to_result(self, domain: str, certificates: List[Dict[str, Any]]) -> ProviderResult: + def _process_certificates_to_result_fixed(self, query_domain: str, certificates: List[Dict[str, Any]]) -> ProviderResult: """ - Process certificates to create ProviderResult with relationships and attributes. + FIXED: Process certificates to create proper domain and CA nodes. + Now creates individual domain nodes instead of large entities. """ result = ProviderResult() if self._stop_event and self._stop_event.is_set(): - print(f"CrtSh processing cancelled before processing for domain: {domain}") + self.logger.logger.info(f"CrtSh processing cancelled before processing for domain: {query_domain}") return result all_discovered_domains = set() + processed_issuers = set() for i, cert_data in enumerate(certificates): - if i % 5 == 0 and self._stop_event and self._stop_event.is_set(): - print(f"CrtSh processing cancelled at certificate {i} for domain: {domain}") + if i % 10 == 0 and self._stop_event and self._stop_event.is_set(): + self.logger.logger.info(f"CrtSh processing cancelled at certificate {i} for domain: {query_domain}") break + # Extract all domains from this certificate cert_domains = self._extract_domains_from_certificate(cert_data) all_discovered_domains.update(cert_domains) + # FIXED: Create CA nodes for certificate issuers (not as domain metadata) issuer_name = self._parse_issuer_organization(cert_data.get('issuer_name', '')) - if issuer_name: + if issuer_name and issuer_name not in processed_issuers: + # Create relationship from query domain to CA result.add_relationship( - source_node=domain, + source_node=query_domain, target_node=issuer_name, relationship_type='crtsh_cert_issuer', provider=self.name, - confidence=0.95 + confidence=0.95, + raw_data={'issuer_dn': cert_data.get('issuer_name', '')} ) + processed_issuers.add(issuer_name) + # Add certificate metadata to each domain in this certificate + cert_metadata = self._extract_certificate_metadata(cert_data) for cert_domain in cert_domains: if not _is_valid_domain(cert_domain): continue - for key, value in self._extract_certificate_metadata(cert_data).items(): + # Add certificate attributes to the domain + for key, value in cert_metadata.items(): if value is not None: result.add_attribute( target_node=cert_domain, @@ -315,48 +327,72 @@ class CrtShProvider(BaseProvider): value=value, attr_type='certificate_data', provider=self.name, - confidence=0.9 + confidence=0.9, + metadata={'certificate_id': cert_data.get('id')} ) if self._stop_event and self._stop_event.is_set(): - print(f"CrtSh query cancelled before relationship creation for domain: {domain}") + self.logger.logger.info(f"CrtSh query cancelled before relationship creation for domain: {query_domain}") return result - for i, discovered_domain in enumerate(all_discovered_domains): - if discovered_domain == domain: + # FIXED: Create selective relationships to avoid large entities + # Only create relationships to domains that are closely related + for discovered_domain in all_discovered_domains: + if discovered_domain == query_domain: continue - if i % 10 == 0 and self._stop_event and self._stop_event.is_set(): - print(f"CrtSh relationship creation cancelled for domain: {domain}") - break - if not _is_valid_domain(discovered_domain): continue - confidence = self._calculate_domain_relationship_confidence( - domain, discovered_domain, [], all_discovered_domains - ) + # FIXED: Only create relationships for domains that share a meaningful connection + # This prevents creating too many relationships that trigger large entity creation + if self._should_create_relationship(query_domain, discovered_domain): + confidence = self._calculate_domain_relationship_confidence( + query_domain, discovered_domain, [], all_discovered_domains + ) - result.add_relationship( - source_node=domain, - target_node=discovered_domain, - relationship_type='crtsh_san_certificate', - provider=self.name, - confidence=confidence, - raw_data={'relationship_type': 'certificate_discovery'} - ) + result.add_relationship( + source_node=query_domain, + target_node=discovered_domain, + relationship_type='crtsh_san_certificate', + provider=self.name, + confidence=confidence, + raw_data={'relationship_type': 'certificate_discovery'} + ) - self.log_relationship_discovery( - source_node=domain, - target_node=discovered_domain, - relationship_type='crtsh_san_certificate', - confidence_score=confidence, - raw_data={'relationship_type': 'certificate_discovery'}, - discovery_method="certificate_transparency_analysis" - ) + self.log_relationship_discovery( + source_node=query_domain, + target_node=discovered_domain, + relationship_type='crtsh_san_certificate', + confidence_score=confidence, + raw_data={'relationship_type': 'certificate_discovery'}, + discovery_method="certificate_transparency_analysis" + ) + self.logger.logger.info(f"CrtSh processing completed for {query_domain}: {len(all_discovered_domains)} domains, {result.get_relationship_count()} relationships") return result + def _should_create_relationship(self, source_domain: str, target_domain: str) -> bool: + """ + FIXED: Determine if a relationship should be created between two domains. + This helps avoid creating too many relationships that trigger large entity creation. + """ + # Always create relationships for subdomains + if target_domain.endswith(f'.{source_domain}') or source_domain.endswith(f'.{target_domain}'): + return True + + # Create relationships for domains that share a common parent (up to 2 levels) + source_parts = source_domain.split('.') + target_parts = target_domain.split('.') + + # Check if they share the same root domain (last 2 parts) + if len(source_parts) >= 2 and len(target_parts) >= 2: + source_root = '.'.join(source_parts[-2:]) + target_root = '.'.join(target_parts[-2:]) + return source_root == target_root + + return False + def _extract_certificate_metadata(self, cert_data: Dict[str, Any]) -> Dict[str, Any]: """Extract comprehensive metadata from certificate data.""" raw_issuer_name = cert_data.get('issuer_name', '') @@ -383,7 +419,7 @@ class CrtShProvider(BaseProvider): metadata['is_currently_valid'] = self._is_cert_valid(cert_data) metadata['expires_soon'] = (not_after - datetime.now(timezone.utc)).days <= 30 - # UPDATED: Keep raw date format or convert to standard format + # Keep raw date format or convert to standard format metadata['not_before'] = not_before.isoformat() metadata['not_after'] = not_after.isoformat() @@ -457,7 +493,6 @@ class CrtShProvider(BaseProvider): return is_not_expired except Exception as e: - #self.logger.logger.debug(f"Certificate validity check failed: {e}") return False def _extract_domains_from_certificate(self, cert_data: Dict[str, Any]) -> Set[str]: @@ -512,20 +547,6 @@ class CrtShProvider(BaseProvider): return [d for d in final_domains if _is_valid_domain(d)] - def _get_certificate_sort_date(self, cert: Dict[str, Any]) -> datetime: - """Get a sortable date from certificate data for chronological ordering.""" - try: - if cert.get('not_before'): - return self._parse_certificate_date(cert['not_before']) - - if cert.get('entry_timestamp'): - return self._parse_certificate_date(cert['entry_timestamp']) - - return datetime(1970, 1, 1, tzinfo=timezone.utc) - - except Exception: - return datetime(1970, 1, 1, tzinfo=timezone.utc) - def _calculate_domain_relationship_confidence(self, domain1: str, domain2: str, shared_certificates: List[Dict[str, Any]], all_discovered_domains: Set[str]) -> float: @@ -544,35 +565,7 @@ class CrtShProvider(BaseProvider): else: context_bonus = 0.0 - # Adjust confidence based on shared certificates - if shared_certificates: - shared_count = len(shared_certificates) - if shared_count >= 3: - shared_bonus = 0.1 - elif shared_count >= 2: - shared_bonus = 0.05 - else: - shared_bonus = 0.02 - - valid_shared = sum(1 for cert in shared_certificates if cert.get('is_currently_valid')) - if valid_shared > 0: - validity_bonus = 0.05 - else: - validity_bonus = 0.0 - else: - shared_bonus = 0.0 - validity_bonus = 0.0 - - # Adjust confidence based on certificate issuer reputation - issuer_bonus = 0.0 - if shared_certificates: - for cert in shared_certificates: - issuer = cert.get('issuer_name', '').lower() - if any(trusted_ca in issuer for trusted_ca in ['let\'s encrypt', 'digicert', 'sectigo', 'globalsign']): - issuer_bonus = max(issuer_bonus, 0.03) - break - - final_confidence = base_confidence + context_bonus + shared_bonus + validity_bonus + issuer_bonus + final_confidence = base_confidence + context_bonus return max(0.1, min(1.0, final_confidence)) def _determine_relationship_context(self, cert_domain: str, query_domain: str) -> str: diff --git a/providers/dns_provider.py b/providers/dns_provider.py index 9ca0e35..3aef192 100644 --- a/providers/dns_provider.py +++ b/providers/dns_provider.py @@ -4,13 +4,13 @@ from dns import resolver, reversename from typing import Dict from .base_provider import BaseProvider from core.provider_result import ProviderResult -from utils.helpers import _is_valid_ip, _is_valid_domain +from utils.helpers import _is_valid_ip, _is_valid_domain, get_ip_version class DNSProvider(BaseProvider): """ Provider for standard DNS resolution and reverse DNS lookups. - Now returns standardized ProviderResult objects. + Now returns standardized ProviderResult objects with IPv4 and IPv6 support. """ def __init__(self, name=None, session_config=None): @@ -78,10 +78,10 @@ class DNSProvider(BaseProvider): def query_ip(self, ip: str) -> ProviderResult: """ - Query reverse DNS for the IP address. + Query reverse DNS for the IP address (supports both IPv4 and IPv6). Args: - ip: IP address to investigate + ip: IP address to investigate (IPv4 or IPv6) Returns: ProviderResult containing discovered relationships and attributes @@ -90,9 +90,10 @@ class DNSProvider(BaseProvider): return ProviderResult() result = ProviderResult() + ip_version = get_ip_version(ip) try: - # Perform reverse DNS lookup + # Perform reverse DNS lookup (works for both IPv4 and IPv6) self.total_requests += 1 reverse_name = reversename.from_address(ip) response = self.resolver.resolve(reverse_name, 'PTR') @@ -103,6 +104,14 @@ class DNSProvider(BaseProvider): hostname = str(ptr_record).rstrip('.') if _is_valid_domain(hostname): + # Determine appropriate forward relationship type based on IP version + if ip_version == 6: + relationship_type = 'dns_aaaa_record' + record_prefix = 'AAAA' + else: + relationship_type = 'dns_a_record' + record_prefix = 'A' + # Add the relationship result.add_relationship( source_node=ip, @@ -113,6 +122,7 @@ class DNSProvider(BaseProvider): raw_data={ 'query_type': 'PTR', 'ip_address': ip, + 'ip_version': ip_version, 'hostname': hostname, 'ttl': response.ttl } @@ -130,10 +140,11 @@ class DNSProvider(BaseProvider): raw_data={ 'query_type': 'PTR', 'ip_address': ip, + 'ip_version': ip_version, 'hostname': hostname, 'ttl': response.ttl }, - discovery_method="reverse_dns_lookup" + discovery_method=f"reverse_dns_lookup_ipv{ip_version}" ) # Add PTR records as separate attribute @@ -145,7 +156,7 @@ class DNSProvider(BaseProvider): attr_type='dns_record', provider=self.name, confidence=0.8, - metadata={'ttl': response.ttl} + metadata={'ttl': response.ttl, 'ip_version': ip_version} ) except resolver.NXDOMAIN: @@ -162,6 +173,7 @@ class DNSProvider(BaseProvider): def _query_record(self, domain: str, record_type: str, result: ProviderResult) -> None: """ FIXED: Query DNS records with unique attribute names for each record type. + Enhanced to better handle IPv6 AAAA records. """ try: self.total_requests += 1 @@ -174,6 +186,10 @@ class DNSProvider(BaseProvider): target = "" if record_type in ['A', 'AAAA']: target = str(record) + # Validate that the IP address is properly formed + if not _is_valid_ip(target): + self.logger.logger.debug(f"Invalid IP address in {record_type} record: {target}") + continue elif record_type in ['CNAME', 'NS', 'PTR']: target = str(record.target).rstrip('.') elif record_type == 'MX': @@ -196,12 +212,21 @@ class DNSProvider(BaseProvider): target = str(record) if target: + # Determine IP version for metadata if this is an IP record + ip_version = None + if record_type in ['A', 'AAAA'] and _is_valid_ip(target): + ip_version = get_ip_version(target) + raw_data = { 'query_type': record_type, 'domain': domain, 'value': target, 'ttl': response.ttl } + + if ip_version: + raw_data['ip_version'] = ip_version + relationship_type = f"dns_{record_type.lower()}_record" confidence = 0.8 @@ -218,14 +243,18 @@ class DNSProvider(BaseProvider): # Add target to records list dns_records.append(target) - # Log relationship discovery + # Log relationship discovery with IP version info + discovery_method = f"dns_{record_type.lower()}_record" + if ip_version: + discovery_method += f"_ipv{ip_version}" + self.log_relationship_discovery( source_node=domain, target_node=target, relationship_type=relationship_type, confidence_score=confidence, raw_data=raw_data, - discovery_method=f"dns_{record_type.lower()}_record" + discovery_method=discovery_method ) # FIXED: Create attribute with specific name for each record type @@ -233,6 +262,14 @@ class DNSProvider(BaseProvider): # Use record type specific attribute name (e.g., 'a_records', 'mx_records', etc.) attribute_name = f"{record_type.lower()}_records" + metadata = {'record_type': record_type, 'ttl': response.ttl} + + # Add IP version info for A/AAAA records + if record_type in ['A', 'AAAA'] and dns_records: + first_ip_version = get_ip_version(dns_records[0]) + if first_ip_version: + metadata['ip_version'] = first_ip_version + result.add_attribute( target_node=domain, name=attribute_name, # UNIQUE name for each record type! @@ -240,7 +277,7 @@ class DNSProvider(BaseProvider): attr_type='dns_record_list', provider=self.name, confidence=0.8, - metadata={'record_type': record_type, 'ttl': response.ttl} + metadata=metadata ) except Exception as e: diff --git a/providers/shodan_provider.py b/providers/shodan_provider.py index 930eb0e..f21c2dc 100644 --- a/providers/shodan_provider.py +++ b/providers/shodan_provider.py @@ -8,13 +8,13 @@ import requests from .base_provider import BaseProvider from core.provider_result import ProviderResult -from utils.helpers import _is_valid_ip, _is_valid_domain +from utils.helpers import _is_valid_ip, _is_valid_domain, get_ip_version, normalize_ip class ShodanProvider(BaseProvider): """ Provider for querying Shodan API for IP address information. - Now returns standardized ProviderResult objects with caching support. + Now returns standardized ProviderResult objects with caching support for IPv4 and IPv6. """ def __init__(self, name=None, session_config=None): @@ -53,8 +53,19 @@ class ShodanProvider(BaseProvider): return {'domains': False, 'ips': True} def _get_cache_file_path(self, ip: str) -> Path: - """Generate cache file path for an IP address.""" - safe_ip = ip.replace('.', '_').replace(':', '_') + """ + Generate cache file path for an IP address (IPv4 or IPv6). + IPv6 addresses contain colons which are replaced with underscores for filesystem safety. + """ + # Normalize the IP address first to ensure consistent caching + normalized_ip = normalize_ip(ip) + if not normalized_ip: + # Fallback for invalid IPs + safe_ip = ip.replace('.', '_').replace(':', '_') + else: + # Replace problematic characters for both IPv4 and IPv6 + safe_ip = normalized_ip.replace('.', '_').replace(':', '_') + return self.cache_dir / f"{safe_ip}.json" def _get_cache_status(self, cache_file_path: Path) -> str: @@ -99,10 +110,10 @@ class ShodanProvider(BaseProvider): def query_ip(self, ip: str) -> ProviderResult: """ - Query Shodan for information about an IP address, with caching of processed data. + Query Shodan for information about an IP address (IPv4 or IPv6), with caching of processed data. Args: - ip: IP address to investigate + ip: IP address to investigate (IPv4 or IPv6) Returns: ProviderResult containing discovered relationships and attributes @@ -110,7 +121,12 @@ class ShodanProvider(BaseProvider): if not _is_valid_ip(ip) or not self.is_available(): return ProviderResult() - cache_file = self._get_cache_file_path(ip) + # Normalize IP address for consistent processing + normalized_ip = normalize_ip(ip) + if not normalized_ip: + return ProviderResult() + + cache_file = self._get_cache_file_path(normalized_ip) cache_status = self._get_cache_status(cache_file) result = ProviderResult() @@ -118,25 +134,48 @@ class ShodanProvider(BaseProvider): try: if cache_status == "fresh": result = self._load_from_cache(cache_file) - self.logger.logger.info(f"Using cached Shodan data for {ip}") + self.logger.logger.info(f"Using cached Shodan data for {normalized_ip}") else: # "stale" or "not_found" - url = f"{self.base_url}/shodan/host/{ip}" + url = f"{self.base_url}/shodan/host/{normalized_ip}" params = {'key': self.api_key} - response = self.make_request(url, method="GET", params=params, target_indicator=ip) + response = self.make_request(url, method="GET", params=params, target_indicator=normalized_ip) if response and response.status_code == 200: data = response.json() # Process the data into ProviderResult BEFORE caching - result = self._process_shodan_data(ip, data) + result = self._process_shodan_data(normalized_ip, data) self._save_to_cache(cache_file, result, data) # Save both result and raw data + elif response and response.status_code == 404: + # Handle 404 "No information available" as successful empty result + try: + error_data = response.json() + if "No information available" in error_data.get('error', ''): + # This is a successful query - Shodan just has no data + self.logger.logger.debug(f"Shodan has no information for {normalized_ip}") + result = ProviderResult() # Empty but successful result + # Cache the empty result to avoid repeated queries + self._save_to_cache(cache_file, result, {'error': 'No information available'}) + else: + # Some other 404 error - treat as failure + raise requests.exceptions.RequestException(f"Shodan API returned 404: {error_data}") + except (ValueError, KeyError): + # Could not parse JSON response - treat as failure + raise requests.exceptions.RequestException(f"Shodan API returned 404 with unparseable response") elif cache_status == "stale": # If API fails on a stale cache, use the old data result = self._load_from_cache(cache_file) + else: + # Other HTTP error codes should be treated as failures + status_code = response.status_code if response else "No response" + raise requests.exceptions.RequestException(f"Shodan API returned HTTP {status_code}") except requests.exceptions.RequestException as e: - self.logger.logger.error(f"Shodan API query failed for {ip}: {e}") + self.logger.logger.info(f"Shodan API query returned no info for {normalized_ip}: {e}") if cache_status == "stale": result = self._load_from_cache(cache_file) + else: + # Re-raise for retry scheduling - but only for actual failures + raise e return result @@ -212,8 +251,12 @@ class ShodanProvider(BaseProvider): def _process_shodan_data(self, ip: str, data: Dict[str, Any]) -> ProviderResult: """ VERIFIED: Process Shodan data creating ISP nodes with ASN attributes and proper relationships. + Enhanced to include IP version information for IPv6 addresses. """ result = ProviderResult() + + # Determine IP version for metadata + ip_version = get_ip_version(ip) # VERIFIED: Extract ISP information and create proper ISP node with ASN isp_name = data.get('org') @@ -227,7 +270,7 @@ class ShodanProvider(BaseProvider): relationship_type='shodan_isp', provider=self.name, confidence=0.9, - raw_data={'asn': asn_value, 'shodan_org': isp_name} + raw_data={'asn': asn_value, 'shodan_org': isp_name, 'ip_version': ip_version} ) # Add ASN as attribute to the ISP node @@ -238,7 +281,7 @@ class ShodanProvider(BaseProvider): attr_type='isp_info', provider=self.name, confidence=0.9, - metadata={'description': 'Autonomous System Number from Shodan'} + metadata={'description': 'Autonomous System Number from Shodan', 'ip_version': ip_version} ) # Also add organization name as attribute to ISP node for completeness @@ -249,7 +292,7 @@ class ShodanProvider(BaseProvider): attr_type='isp_info', provider=self.name, confidence=0.9, - metadata={'description': 'Organization name from Shodan'} + metadata={'description': 'Organization name from Shodan', 'ip_version': ip_version} ) # Process hostnames (reverse DNS) @@ -257,21 +300,27 @@ class ShodanProvider(BaseProvider): if key == 'hostnames': for hostname in value: if _is_valid_domain(hostname): + # Use appropriate relationship type based on IP version + if ip_version == 6: + relationship_type = 'shodan_aaaa_record' + else: + relationship_type = 'shodan_a_record' + result.add_relationship( source_node=ip, target_node=hostname, - relationship_type='shodan_a_record', + relationship_type=relationship_type, provider=self.name, confidence=0.8, - raw_data=data + raw_data={**data, 'ip_version': ip_version} ) self.log_relationship_discovery( source_node=ip, target_node=hostname, - relationship_type='shodan_a_record', + relationship_type=relationship_type, confidence_score=0.8, - raw_data=data, - discovery_method="shodan_host_lookup" + raw_data={**data, 'ip_version': ip_version}, + discovery_method=f"shodan_host_lookup_ipv{ip_version}" ) elif key == 'ports': # Add open ports as attributes to the IP @@ -282,7 +331,8 @@ class ShodanProvider(BaseProvider): value=port, attr_type='shodan_network_info', provider=self.name, - confidence=0.9 + confidence=0.9, + metadata={'ip_version': ip_version} ) elif isinstance(value, (str, int, float, bool)) and value is not None: # Add other Shodan fields as IP attributes (keep raw field names) @@ -292,7 +342,8 @@ class ShodanProvider(BaseProvider): value=value, attr_type='shodan_info', provider=self.name, - confidence=0.9 + confidence=0.9, + metadata={'ip_version': ip_version} ) return result \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 0e37daa..d46c0bc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,10 @@ -Flask>=2.3.3 -networkx>=3.1 -requests>=2.31.0 -python-dateutil>=2.8.2 -Werkzeug>=2.3.7 -urllib3>=2.0.0 -dnspython>=2.4.2 +Flask +networkx +requests +python-dateutil +Werkzeug +urllib3 +dnspython gunicorn redis python-dotenv \ No newline at end of file diff --git a/static/js/main.js b/static/js/main.js index cfb3c25..ee56fe7 100644 --- a/static/js/main.js +++ b/static/js/main.js @@ -58,7 +58,10 @@ class DNSReconApp { startScan: document.getElementById('start-scan'), addToGraph: document.getElementById('add-to-graph'), stopScan: document.getElementById('stop-scan'), - exportResults: document.getElementById('export-results'), + exportOptions: document.getElementById('export-options'), + exportModal: document.getElementById('export-modal'), + exportModalClose: document.getElementById('export-modal-close'), + exportGraphJson: document.getElementById('export-graph-json'), configureSettings: document.getElementById('configure-settings'), // Status elements @@ -146,11 +149,24 @@ class DNSReconApp { this.stopScan(); }); - this.elements.exportResults.addEventListener('click', (e) => { + this.elements.exportOptions.addEventListener('click', (e) => { e.preventDefault(); - this.exportResults(); + this.showExportModal(); }); + if (this.elements.exportModalClose) { + this.elements.exportModalClose.addEventListener('click', () => this.hideExportModal()); + } + if (this.elements.exportModal) { + this.elements.exportModal.addEventListener('click', (e) => { + if (e.target === this.elements.exportModal) this.hideExportModal(); + }); + } + if (this.elements.exportGraphJson) { + this.elements.exportGraphJson.addEventListener('click', () => this.exportGraphJson()); + } + + this.elements.configureSettings.addEventListener('click', () => this.showSettingsModal()); // Enter key support for target domain input @@ -219,6 +235,7 @@ class DNSReconApp { if (e.key === 'Escape') { this.hideModal(); this.hideSettingsModal(); + this.hideExportModal(); // Add this line } }); @@ -376,26 +393,96 @@ class DNSReconApp { } /** - * Export scan results + * Show Export modal */ - async exportResults() { + showExportModal() { + if (this.elements.exportModal) { + this.elements.exportModal.style.display = 'block'; + } + } + + /** + * Hide Export modal + */ + hideExportModal() { + if (this.elements.exportModal) { + this.elements.exportModal.style.display = 'none'; + } + } + + /** + * Export graph data as JSON with proper error handling + */ + async exportGraphJson() { try { - console.log('Exporting results...'); + console.log('Exporting graph data as JSON...'); - // Create a temporary link to trigger download + // Show loading state + if (this.elements.exportGraphJson) { + const originalContent = this.elements.exportGraphJson.innerHTML; + this.elements.exportGraphJson.innerHTML = '[...]Exporting...'; + this.elements.exportGraphJson.disabled = true; + + // Store original content for restoration + this.elements.exportGraphJson._originalContent = originalContent; + } + + // Make API call to get export data + const response = await fetch('/api/export', { + method: 'GET', + headers: { + 'Content-Type': 'application/json' + } + }); + + if (!response.ok) { + const errorData = await response.json().catch(() => ({})); + throw new Error(errorData.error || `HTTP ${response.status}: ${response.statusText}`); + } + + // Check if response is JSON or file download + const contentType = response.headers.get('content-type'); + if (contentType && contentType.includes('application/json') && !response.headers.get('content-disposition')) { + // This is an error response in JSON format + const errorData = await response.json(); + throw new Error(errorData.error || 'Export failed'); + } + + // Get the filename from headers or create one + const contentDisposition = response.headers.get('content-disposition'); + let filename = 'dnsrecon_export.json'; + if (contentDisposition) { + const filenameMatch = contentDisposition.match(/filename[^;=\n]*=((['"]).*?\2|[^;\n]*)/); + if (filenameMatch) { + filename = filenameMatch[1].replace(/['"]/g, ''); + } + } + + // Create blob and download + const blob = await response.blob(); + const url = window.URL.createObjectURL(blob); const link = document.createElement('a'); - link.href = '/api/export'; - link.download = ''; // Let server determine filename + link.href = url; + link.download = filename; document.body.appendChild(link); link.click(); document.body.removeChild(link); + window.URL.revokeObjectURL(url); - this.showSuccess('Results export initiated'); - console.log('Results export initiated'); + this.showSuccess('Graph data exported successfully'); + this.hideExportModal(); } catch (error) { - console.error('Failed to export results:', error); - this.showError(`Failed to export results: ${error.message}`); + console.error('Failed to export graph data:', error); + this.showError(`Export failed: ${error.message}`); + } finally { + // Restore button state + if (this.elements.exportGraphJson) { + const originalContent = this.elements.exportGraphJson._originalContent || + '[JSON]Export Graph Data'; + this.elements.exportGraphJson.innerHTML = originalContent; + this.elements.exportGraphJson.disabled = false; + } } } @@ -2116,14 +2203,7 @@ class DNSReconApp { } } - /** - * Validate target (domain or IP) - * @param {string} target - Target to validate - * @returns {boolean} True if valid - */ - isValidTarget(target) { - return this.isValidDomain(target) || this.isValidIp(target); - } + /** * Validate domain name @@ -2143,20 +2223,149 @@ class DNSReconApp { } /** - * Validate IP address + * Validate target (domain or IP) - UPDATED for IPv6 support + * @param {string} target - Target to validate + * @returns {boolean} True if valid + */ + isValidTarget(target) { + return this.isValidDomain(target) || this.isValidIp(target); + } + + /** + * Validate IP address (IPv4 or IPv6) * @param {string} ip - IP to validate * @returns {boolean} True if valid */ isValidIp(ip) { console.log(`Validating IP: "${ip}"`); - const parts = ip.split('.'); - if (parts.length !== 4) { + + if (!ip || typeof ip !== 'string') { return false; } - return parts.every(part => { - const num = parseInt(part, 10); - return !isNaN(num) && num >= 0 && num <= 255 && String(num) === part; - }); + + ip = ip.trim(); + + // IPv4 validation + if (this.isValidIPv4(ip)) { + return true; + } + + // IPv6 validation + if (this.isValidIPv6(ip)) { + return true; + } + + return false; + } + + /** + * Validate IPv4 address + * @param {string} ip - IP to validate + * @returns {boolean} True if valid IPv4 + */ + isValidIPv4(ip) { + const ipv4Pattern = /^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/; + const match = ip.match(ipv4Pattern); + + if (!match) { + return false; + } + + // Check each octet is between 0-255 + for (let i = 1; i <= 4; i++) { + const octet = parseInt(match[i], 10); + if (octet < 0 || octet > 255) { + return false; + } + // Check for leading zeros (except for '0' itself) + if (match[i].length > 1 && match[i][0] === '0') { + return false; + } + } + + return true; + } + + /** + * Validate IPv6 address + * @param {string} ip - IP to validate + * @returns {boolean} True if valid IPv6 + */ + isValidIPv6(ip) { + // Handle IPv6 with embedded IPv4 (e.g., ::ffff:192.168.1.1) + if (ip.includes('.')) { + const lastColon = ip.lastIndexOf(':'); + if (lastColon !== -1) { + const ipv6Part = ip.substring(0, lastColon + 1); + const ipv4Part = ip.substring(lastColon + 1); + + if (this.isValidIPv4(ipv4Part)) { + // Validate the IPv6 part (should end with ::) + return this.isValidIPv6Pure(ipv6Part + '0:0'); + } + } + } + + return this.isValidIPv6Pure(ip); + } + + /** + * Validate pure IPv6 address (no embedded IPv4) + * @param {string} ip - IPv6 address to validate + * @returns {boolean} True if valid IPv6 + */ + isValidIPv6Pure(ip) { + // Basic format check + if (!ip || ip.length < 2 || ip.length > 39) { + return false; + } + + // Check for invalid characters + if (!/^[0-9a-fA-F:]+$/.test(ip)) { + return false; + } + + // Handle double colon (::) for zero compression + const doubleColonCount = (ip.match(/::/g) || []).length; + if (doubleColonCount > 1) { + return false; // Only one :: allowed + } + + let parts; + if (doubleColonCount === 1) { + // Expand the :: notation + const [before, after] = ip.split('::'); + const beforeParts = before ? before.split(':') : []; + const afterParts = after ? after.split(':') : []; + + // Calculate how many zero groups the :: represents + const totalParts = beforeParts.length + afterParts.length; + const zeroGroups = 8 - totalParts; + + if (zeroGroups < 1) { + return false; // :: must represent at least one zero group + } + + // Build the full address + parts = beforeParts.concat(Array(zeroGroups).fill('0')).concat(afterParts); + } else { + // No :: notation, split normally + parts = ip.split(':'); + } + + // IPv6 should have exactly 8 groups + if (parts.length !== 8) { + return false; + } + + // Validate each group (1-4 hex digits) + for (const part of parts) { + if (!part || part.length > 4 || !/^[0-9a-fA-F]+$/.test(part)) { + return false; + } + } + + return true; } /** diff --git a/templates/index.html b/templates/index.html index 0c7a4f9..f32d02c 100644 --- a/templates/index.html +++ b/templates/index.html @@ -53,9 +53,9 @@ [STOP] Terminate Scan -
- + + + +
diff --git a/utils/helpers.py b/utils/helpers.py index 2d17717..8849790 100644 --- a/utils/helpers.py +++ b/utils/helpers.py @@ -1,3 +1,8 @@ +# dnsrecon-reduced/utils/helpers.py + +import ipaddress +from typing import Union + def _is_valid_domain(domain: str) -> bool: """ Basic domain validation. @@ -26,32 +31,27 @@ def _is_valid_domain(domain: str) -> bool: def _is_valid_ip(ip: str) -> bool: """ - Basic IP address validation. + IP address validation supporting both IPv4 and IPv6. Args: ip: IP address string to validate Returns: - True if IP appears valid + True if IP appears valid (IPv4 or IPv6) """ + if not ip: + return False + try: - parts = ip.split('.') - if len(parts) != 4: - return False - - for part in parts: - num = int(part) - if not 0 <= num <= 255: - return False - + # This handles both IPv4 and IPv6 validation + ipaddress.ip_address(ip.strip()) return True - except (ValueError, AttributeError): return False def is_valid_target(target: str) -> bool: """ - Checks if the target is a valid domain or IP address. + Checks if the target is a valid domain or IP address (IPv4/IPv6). Args: target: The target string to validate. @@ -59,4 +59,36 @@ def is_valid_target(target: str) -> bool: Returns: True if the target is a valid domain or IP, False otherwise. """ - return _is_valid_domain(target) or _is_valid_ip(target) \ No newline at end of file + return _is_valid_domain(target) or _is_valid_ip(target) + +def get_ip_version(ip: str) -> Union[int, None]: + """ + Get the IP version (4 or 6) of a valid IP address. + + Args: + ip: IP address string + + Returns: + 4 for IPv4, 6 for IPv6, None if invalid + """ + try: + addr = ipaddress.ip_address(ip.strip()) + return addr.version + except (ValueError, AttributeError): + return None + +def normalize_ip(ip: str) -> Union[str, None]: + """ + Normalize an IP address to its canonical form. + + Args: + ip: IP address string + + Returns: + Normalized IP address string, None if invalid + """ + try: + addr = ipaddress.ip_address(ip.strip()) + return str(addr) + except (ValueError, AttributeError): + return None \ No newline at end of file