From 949fbdbb456e4af4e0e9e4cb94ca6675ebf81a79 Mon Sep 17 00:00:00 2001 From: overcuriousity Date: Sun, 14 Sep 2025 17:18:56 +0200 Subject: [PATCH] itteration --- config.py | 4 ++-- core/graph_manager.py | 21 +++++++++++++-------- core/logger.py | 2 +- core/scanner.py | 15 ++++++++------- providers/base_provider.py | 2 +- providers/crtsh_provider.py | 2 +- providers/dns_provider.py | 9 ++++----- providers/shodan_provider.py | 2 +- static/js/graph.js | 2 +- 9 files changed, 32 insertions(+), 27 deletions(-) diff --git a/config.py b/config.py index 01cd30b..6122157 100644 --- a/config.py +++ b/config.py @@ -19,7 +19,7 @@ class Config: # --- General Settings --- self.default_recursion_depth = 2 - self.default_timeout = 10 + self.default_timeout = 15 self.max_concurrent_requests = 5 self.large_entity_threshold = 100 self.max_retries_per_target = 3 @@ -27,7 +27,7 @@ class Config: # --- Rate Limiting (requests per minute) --- self.rate_limits = { - 'crtsh': 60, + 'crtsh': 30, 'shodan': 60, 'dns': 100 } diff --git a/core/graph_manager.py b/core/graph_manager.py index b1f0a4c..43853c2 100644 --- a/core/graph_manager.py +++ b/core/graph_manager.py @@ -80,8 +80,8 @@ class GraphManager: return if len(value) < 4 or value.lower() in ['true', 'false', 'unknown', 'none', 'crt.sh']: return - elif isinstance(value, int) and abs(value) < 9999: - return # Ignore small integers + elif isinstance(value, int) and (abs(value) < 1024 or abs(value) > 65535): + return # Ignore small integers and common port numbers elif isinstance(value, bool): return # Ignore boolean values @@ -421,10 +421,14 @@ class GraphManager: def _get_confidence_distribution(self) -> Dict[str, int]: """Get distribution of edge confidence scores.""" distribution = {'high': 0, 'medium': 0, 'low': 0} - for _, _, confidence in self.graph.edges(data='confidence_score', default=0): - if confidence >= 0.8: distribution['high'] += 1 - elif confidence >= 0.6: distribution['medium'] += 1 - else: distribution['low'] += 1 + for _, _, data in self.graph.edges(data=True): + confidence = data.get('confidence_score', 0) + if confidence >= 0.8: + distribution['high'] += 1 + elif confidence >= 0.6: + distribution['medium'] += 1 + else: + distribution['low'] += 1 return distribution def get_statistics(self) -> Dict[str, Any]: @@ -439,9 +443,10 @@ class GraphManager: # Calculate distributions for node_type in NodeType: stats['node_type_distribution'][node_type.value] = self.get_nodes_by_type(node_type).__len__() - for _, _, rel_type in self.graph.edges(data='relationship_type', default='unknown'): + for _, _, data in self.graph.edges(data=True): + rel_type = data.get('relationship_type', 'unknown') stats['relationship_type_distribution'][rel_type] = stats['relationship_type_distribution'].get(rel_type, 0) + 1 - for _, _, provider in self.graph.edges(data='source_provider', default='unknown'): + provider = data.get('source_provider', 'unknown') stats['provider_distribution'][provider] = stats['provider_distribution'].get(provider, 0) + 1 return stats diff --git a/core/logger.py b/core/logger.py index 845c595..a916853 100644 --- a/core/logger.py +++ b/core/logger.py @@ -42,7 +42,7 @@ class ForensicLogger: Maintains detailed audit trail of all reconnaissance activities. """ - def __init__(self, session_id: str = None): + def __init__(self, session_id: str = ""): """ Initialize forensic logger. diff --git a/core/scanner.py b/core/scanner.py index d300bc2..fc25620 100644 --- a/core/scanner.py +++ b/core/scanner.py @@ -5,7 +5,7 @@ import traceback import time import os import importlib -from typing import List, Set, Dict, Any, Tuple +from typing import List, Set, Dict, Any, Tuple, Optional from concurrent.futures import ThreadPoolExecutor, as_completed, CancelledError, Future from collections import defaultdict, deque from datetime import datetime, timezone @@ -49,7 +49,7 @@ class Scanner: self.max_depth = 2 self.stop_event = threading.Event() self.scan_thread = None - self.session_id = None # Will be set by session manager + self.session_id: Optional[str] = None # Will be set by session manager self.task_queue = deque([]) self.target_retries = defaultdict(int) self.scan_failed_due_to_retries = False @@ -170,9 +170,10 @@ class Scanner: attribute = getattr(module, attribute_name) if isinstance(attribute, type) and issubclass(attribute, BaseProvider) and attribute is not BaseProvider: provider_class = attribute - provider_name = provider_class(session_config=self.config).get_name() + provider = provider_class(name=attribute_name, session_config=self.config) + provider_name = provider.get_name() + if self.config.is_provider_enabled(provider_name): - provider = provider_class(session_config=self.config) if provider.is_available(): provider.set_stop_event(self.stop_event) self.providers.append(provider) @@ -459,7 +460,7 @@ class Scanner: provider_states = node_data.get('metadata', {}).get('provider_states', {}) return provider_name in provider_states - def _query_single_provider_forensic(self, provider, target: str, is_ip: bool, current_depth: int) -> List: + def _query_single_provider_forensic(self, provider, target: str, is_ip: bool, current_depth: int) -> Optional[List]: """Query a single provider with stop signal checking.""" provider_name = provider.get_name() start_time = datetime.now(timezone.utc) @@ -493,7 +494,7 @@ class Scanner: return None def _update_provider_state(self, target: str, provider_name: str, status: str, - results_count: int, error: str, start_time: datetime) -> None: + results_count: int, error: Optional[str], start_time: datetime) -> None: """Update provider state in node metadata for forensic tracking.""" if not self.graph.graph.has_node(target): return @@ -785,7 +786,7 @@ class Scanner: if isinstance(attribute, type) and issubclass(attribute, BaseProvider) and attribute is not BaseProvider: provider_class = attribute # Instantiate to get metadata, even if not fully configured - temp_provider = provider_class(session_config=self.config) + temp_provider = provider_class(name=attribute_name, session_config=self.config) provider_name = temp_provider.get_name() # Find the actual provider instance if it exists, to get live stats diff --git a/providers/base_provider.py b/providers/base_provider.py index f8b66a7..2bb086e 100644 --- a/providers/base_provider.py +++ b/providers/base_provider.py @@ -228,7 +228,7 @@ class BaseProvider(ABC): self.total_requests += 1 # Prepare request - request_headers = self.session.headers.copy() + request_headers = dict(self.session.headers).copy() if headers: request_headers.update(headers) diff --git a/providers/crtsh_provider.py b/providers/crtsh_provider.py index d09ed28..1ff248c 100644 --- a/providers/crtsh_provider.py +++ b/providers/crtsh_provider.py @@ -21,7 +21,7 @@ class CrtShProvider(BaseProvider): Now uses session-specific configuration and caching. """ - def __init__(self, session_config=None): + def __init__(self, name=None, session_config=None): """Initialize CrtSh provider with session-specific configuration.""" super().__init__( name="crtsh", diff --git a/providers/dns_provider.py b/providers/dns_provider.py index 82260ac..5691a15 100644 --- a/providers/dns_provider.py +++ b/providers/dns_provider.py @@ -1,7 +1,6 @@ # dnsrecon/providers/dns_provider.py -import dns.resolver -import dns.reversename +from dns import resolver, reversename from typing import List, Dict, Any, Tuple from .base_provider import BaseProvider from utils.helpers import _is_valid_ip, _is_valid_domain @@ -13,7 +12,7 @@ class DNSProvider(BaseProvider): Now uses session-specific configuration. """ - def __init__(self, session_config=None): + def __init__(self, name=None, session_config=None): """Initialize DNS provider with session-specific configuration.""" super().__init__( name="dns", @@ -23,7 +22,7 @@ class DNSProvider(BaseProvider): ) # Configure DNS resolver - self.resolver = dns.resolver.Resolver() + self.resolver = resolver.Resolver() self.resolver.timeout = 5 self.resolver.lifetime = 10 #self.resolver.nameservers = ['127.0.0.1'] @@ -87,7 +86,7 @@ class DNSProvider(BaseProvider): try: # Perform reverse DNS lookup self.total_requests += 1 - reverse_name = dns.reversename.from_address(ip) + reverse_name = reversename.from_address(ip) response = self.resolver.resolve(reverse_name, 'PTR') self.successful_requests += 1 diff --git a/providers/shodan_provider.py b/providers/shodan_provider.py index 439214a..7b80d3c 100644 --- a/providers/shodan_provider.py +++ b/providers/shodan_provider.py @@ -15,7 +15,7 @@ class ShodanProvider(BaseProvider): Now uses session-specific API keys. """ - def __init__(self, session_config=None): + def __init__(self, name=None, session_config=None): """Initialize Shodan provider with session-specific configuration.""" super().__init__( name="shodan", diff --git a/static/js/graph.js b/static/js/graph.js index d5a0a12..c2337c3 100644 --- a/static/js/graph.js +++ b/static/js/graph.js @@ -376,7 +376,7 @@ class GraphManager { // Single correlation value const value = Array.isArray(values) && values.length > 0 ? values[0] : (metadata.value || 'Unknown'); const displayValue = typeof value === 'string' && value.length > 20 ? value.substring(0, 17) + '...' : value; - processedNode.label = `Corr: ${displayValue}`; + processedNode.label = `${displayValue}`; processedNode.title = `Correlation: ${value}`; } }