renaming

.env.example

@@ -1,5 +1,5 @@
 # ===============================================
-# DNSRecon Environment Variables
+# DNScope Environment Variables
 # ===============================================
 # Copy this file to .env and fill in your values.
 
@@ -32,3 +32,5 @@ LARGE_ENTITY_THRESHOLD=100
 MAX_RETRIES_PER_TARGET=8
 # How long cached provider responses are stored (in hours).
 CACHE_TIMEOUT_HOURS=12
+
+GRAPH_POLLING_NODE_THRESHOLD=100

README.md

@@ -1,16 +1,18 @@
-# DNSRecon - Passive Infrastructure Reconnaissance Tool
+# DNScope - Passive Infrastructure Reconnaissance Tool
 
-DNSRecon is an interactive, passive reconnaissance tool designed to map adversary infrastructure. It operates on a "free-by-default" model, ensuring core functionality without subscriptions, while allowing power users to enhance its capabilities with paid API keys. It is aimed at cybersecurity researchers, pentesters, and administrators who want to understand the public footprint of a target domain.
+DNScope is an interactive, passive reconnaissance tool designed to map adversary infrastructure. It operates on a "free-by-default" model, ensuring core functionality without subscriptions, while allowing power users to enhance its capabilities with paid API keys. It is aimed at cybersecurity researchers, pentesters, and administrators who want to understand the public footprint of a target domain.
 
-**Repo Link:** [https://git.cc24.dev/mstoeck3/dnsrecon](https://git.cc24.dev/mstoeck3/dnsrecon)
+**Repo Link:** [https://github.com/overcuriousity/DNScope](https://github.com/overcuriousity/DNScope)
 
 -----
 
 ## Concept and Philosophy
 
-The core philosophy of DNSRecon is to provide a comprehensive and accurate map of a target's infrastructure using only **passive data sources** by default. This means that, out of the box, DNSRecon will not send any traffic to the target's servers. Instead, it queries public and historical data sources to build a picture of the target's online presence. This approach is ideal for researchers and pentesters who want to gather intelligence without alerting the target, and for administrators who want to see what information about their own infrastructure is publicly available.
+The core philosophy of DNScope is to provide a comprehensive and accurate map of a target's infrastructure using only **passive data sources** by default. This means that, out of the box, DNScope will not send any traffic to the target's servers. Instead, it queries public and historical data sources to build a picture of the target's online presence. This approach is ideal for researchers and pentesters who want to gather intelligence without alerting the target, and for administrators who want to see what information about their own infrastructure is publicly available.
 
-For power users who require more in-depth information, DNSRecon can be configured to use API keys for services like Shodan, which provides a wealth of information about internet-connected devices. However, this is an optional feature, and the core functionality of the tool will always remain free and passive.
+For power users who require more in-depth information, DNScope can be configured to use API keys for services like Shodan, which provides a wealth of information about internet-connected devices. However, this is an optional feature, and the core functionality of the tool will always remain free and passive.
+
+-----
 
 -----
 
@@ -24,12 +26,15 @@ For power users who require more in-depth information, DNSRecon can be configure
   * **Session Management**: Supports concurrent user sessions with isolated scanner instances.
   * **Extensible Provider Architecture**: Easily add new data sources to expand the tool's capabilities.
   * **Web-Based UI**: An intuitive and interactive web interface for managing scans and visualizing results.
+  * **Export Options**: Export scan results to JSON, a list of targets to a text file, or an executive summary.
+  * **API Key Management**: Securely manage API keys for various providers through the web interface.
+  * **Provider Management**: Enable or disable providers for the current session.
 
 -----
 
 ## Technical Architecture
 
-DNSRecon is a web-based application built with a modern technology stack:
+DNScope is a web-based application built with a modern technology stack:
 
   * **Backend**: The backend is a **Flask** application that provides a REST API for the frontend and manages the scanning process.
   * **Scanning Engine**: The core scanning engine is a multi-threaded Python application that uses a provider-based architecture to query different data sources.
@@ -41,7 +46,7 @@ DNSRecon is a web-based application built with a modern technology stack:
 
 ## Data Sources
 
-DNSRecon queries the following data sources:
+DNScope queries the following data sources:
 
   * **DNS**: Standard DNS lookups (A, AAAA, CNAME, MX, NS, SOA, TXT).
   * **crt.sh**: A certificate transparency log that provides information about SSL/TLS certificates.
@@ -56,15 +61,43 @@ DNSRecon queries the following data sources:
   * Python 3.8 or higher
   * A modern web browser with JavaScript enabled
   * A Linux host for running the application
+  * Redis Server
 
-### 1\. Clone the Project
+### 1\. Install Redis
+
+It is recommended to install Redis from the official repositories.
+
+**On Debian/Ubuntu:**
 
 ```bash
-git clone https://git.cc24.dev/mstoeck3/dnsrecon
+sudo apt-get update
-cd dnsrecon
+sudo apt-get install redis-server
 ```
 
-### 2\. Install Python Dependencies
+**On CentOS/RHEL:**
+
+```bash
+sudo yum install redis
+sudo systemctl start redis
+sudo systemctl enable redis
+```
+
+You can verify that Redis is running with the following command:
+
+```bash
+redis-cli ping
+```
+
+You should see `PONG` as the response.
+
+### 2\. Clone the Project
+
+```bash
+git clone https://github.com/overcuriousity/DNScope
+cd DNScope
+```
+
+### 3\. Install Python Dependencies
 
 It is highly recommended to use a virtual environment:
 
@@ -86,10 +119,11 @@ The `requirements.txt` file contains the following dependencies:
   * gunicorn
   * redis
   * python-dotenv
+  * psycopg2-binary
 
-### 3\. Configure the Application
+### 4\. Configure the Application
 
-DNSRecon is configured using a `.env` file. You can copy the provided example file and edit it to suit your needs:
+DNScope is configured using a `.env` file. You can copy the provided example file and edit it to suit your needs:
 
 ```bash
 cp .env.example .env
@@ -133,30 +167,30 @@ gunicorn --workers 4 --bind 0.0.0.0:5000 app:app
 
 ## Systemd Service
 
-To run DNSRecon as a service that starts automatically on boot, you can use `systemd`.
+To run DNScope as a service that starts automatically on boot, you can use `systemd`.
 
 ### 1\. Create a `.service` file
 
 Create a new service file in `/etc/systemd/system/`:
 
 ```bash
-sudo nano /etc/systemd/system/dnsrecon.service
+sudo nano /etc/systemd/system/DNScope.service
 ```
 
 ### 2\. Add the Service Configuration
 
-Paste the following configuration into the file. **Remember to replace `/path/to/your/dnsrecon` and `your_user` with your actual project path and username.**
+Paste the following configuration into the file. **Remember to replace `/path/to/your/DNScope` and `your_user` with your actual project path and username.**
 
 ```ini
 [Unit]
-Description=DNSRecon Application
+Description=DNScope Application
 After=network.target
 
 [Service]
 User=your_user
 Group=your_user
-WorkingDirectory=/path/to/your/dnsrecon
+WorkingDirectory=/path/to/your/DNScope
-ExecStart=/path/to/your/dnsrecon/venv/bin/gunicorn --workers 4 --bind 0.0.0.0:5000 app:app
+ExecStart=/path/to/your/DNScope/venv/bin/gunicorn --workers 4 --bind 0.0.0.0:5000 app:app
 Restart=always
 Environment="SECRET_KEY=your-super-secret-and-random-key"
 Environment="FLASK_ENV=production"
@@ -173,14 +207,14 @@ Reload the `systemd` daemon, enable the service to start on boot, and then start
 
 ```bash
 sudo systemctl daemon-reload
-sudo systemctl enable dnsrecon.service
+sudo systemctl enable DNScope.service
-sudo systemctl start dnsrecon.service
+sudo systemctl start DNScope.service
 ```
 
 You can check the status of the service at any time with:
 
 ```bash
-sudo systemctl status dnsrecon.service
+sudo systemctl status DNScope.service
 ```
 
 -----
@@ -210,14 +244,14 @@ rm -rf cache/*
 ### 4\. Restart the Service
 
 ```bash
-sudo systemctl restart dnsrecon.service
+sudo systemctl restart DNScope.service
 ```
 
 -----
 
 ## Extensibility
 
-DNSRecon is designed to be extensible, and adding new providers is a straightforward process. To add a new provider, you will need to create a new Python file in the `providers` directory that inherits from the `BaseProvider` class. The new provider will need to implement the following methods:
+DNScope is designed to be extensible, and adding new providers is a straightforward process. To add a new provider, you will need to create a new Python file in the `providers` directory that inherits from the `BaseProvider` class. The new provider will need to implement the following methods:
 
   * `get_name()`: Return the name of the provider.
   * `get_display_name()`: Return a display-friendly name for the provider.

app.py

@@ -1,8 +1,9 @@
-# dnsrecon-reduced/app.py
+# DNScope-reduced/app.py
 
 """
-Flask application entry point for DNSRecon web interface.
+Flask application entry point for DNScope web interface.
 Provides REST API endpoints and serves the web interface with user session support.
+UPDATED: Added /api/config endpoint for graph polling optimization settings.
 """
 
 import json
@@ -28,7 +29,7 @@ def get_user_scanner():
     """
     Retrieves the scanner for the current session, or creates a new one if none exists.
     """
-    current_flask_session_id = session.get('dnsrecon_session_id')
+    current_flask_session_id = session.get('DNScope_session_id')
     
     if current_flask_session_id:
         existing_scanner = session_manager.get_session(current_flask_session_id)
@@ -41,7 +42,7 @@ def get_user_scanner():
     if not new_scanner:
         raise Exception("Failed to create new scanner session")
     
-    session['dnsrecon_session_id'] = new_session_id
+    session['DNScope_session_id'] = new_session_id
     session.permanent = True
     
     return new_session_id, new_scanner
@@ -53,6 +54,21 @@ def index():
     return render_template('index.html')
 
 
+@app.route('/api/config', methods=['GET'])
+def get_config():
+    """Get configuration settings for frontend."""
+    try:
+        return jsonify({
+            'success': True,
+            'config': {
+                'graph_polling_node_threshold': config.graph_polling_node_threshold
+            }
+        })
+    except Exception as e:
+        traceback.print_exc()
+        return jsonify({'success': False, 'error': f'Internal server error: {str(e)}'}), 500
+
+
 @app.route('/api/scan/start', methods=['POST'])
 def start_scan():
     """
@@ -187,7 +203,9 @@ def get_graph_data():
 
 @app.route('/api/graph/large-entity/extract', methods=['POST'])
 def extract_from_large_entity():
-    """Extract a node from a large entity."""
+    """
+    FIXED: Extract a node from a large entity with proper error handling.
+    """
     try:
         data = request.get_json()
         large_entity_id = data.get('large_entity_id')
@@ -200,17 +218,66 @@ def extract_from_large_entity():
         if not scanner:
             return jsonify({'success': False, 'error': 'No active session found'}), 404
 
+        # FIXED: Check if node exists and provide better error messages
+        if not scanner.graph.graph.has_node(node_id):
+            return jsonify({
+                'success': False, 
+                'error': f'Node {node_id} not found in graph'
+            }), 404
+
+        # FIXED: Check if node is actually part of the large entity
+        node_data = scanner.graph.graph.nodes[node_id]
+        metadata = node_data.get('metadata', {})
+        current_large_entity = metadata.get('large_entity_id')
+        
+        if not current_large_entity:
+            return jsonify({
+                'success': False, 
+                'error': f'Node {node_id} is not part of any large entity'
+            }), 400
+            
+        if current_large_entity != large_entity_id:
+            return jsonify({
+                'success': False, 
+                'error': f'Node {node_id} belongs to large entity {current_large_entity}, not {large_entity_id}'
+            }), 400
+
+        # FIXED: Check if large entity exists
+        if not scanner.graph.graph.has_node(large_entity_id):
+            return jsonify({
+                'success': False, 
+                'error': f'Large entity {large_entity_id} not found'
+            }), 404
+
+        # Perform the extraction
         success = scanner.extract_node_from_large_entity(large_entity_id, node_id)
         
         if success:
+            # Force immediate session state update
             session_manager.update_session_scanner(user_session_id, scanner)
-            return jsonify({'success': True, 'message': f'Node {node_id} extracted successfully.'})
+            
+            return jsonify({
+                'success': True, 
+                'message': f'Node {node_id} extracted successfully from {large_entity_id}.',
+                'extracted_node': node_id,
+                'large_entity': large_entity_id
+            })
         else:
-            return jsonify({'success': False, 'error': f'Failed to extract node {node_id}.'}), 500
+            # This should not happen with the improved checks above, but handle it gracefully
+            return jsonify({
+                'success': False, 
+                'error': f'Failed to extract node {node_id} from {large_entity_id}. Node may have already been extracted.'
+            }), 409
 
+    except json.JSONDecodeError:
+        return jsonify({'success': False, 'error': 'Invalid JSON in request body'}), 400
     except Exception as e:
         traceback.print_exc()
-        return jsonify({'success': False, 'error': f'Internal server error: {str(e)}'}), 500
+        return jsonify({
+            'success': False, 
+            'error': f'Internal server error: {str(e)}',
+            'error_type': type(e).__name__
+        }), 500
 
 @app.route('/api/graph/node/<node_id>', methods=['DELETE'])
 def delete_graph_node(node_id):

config.py

@@ -1,7 +1,7 @@
-# dnsrecon-reduced/config.py
+# DNScope-reduced/config.py
 
 """
-Configuration management for DNSRecon tool.
+Configuration management for DNScope tool.
 Handles API key storage, rate limiting, and default settings.
 """
 
@@ -13,7 +13,7 @@ from dotenv import load_dotenv
 load_dotenv()
 
 class Config:
-    """Configuration manager for DNSRecon application."""
+    """Configuration manager for DNScope application."""
     
     def __init__(self):
         """Initialize configuration with default values."""
@@ -26,6 +26,9 @@ class Config:
         self.large_entity_threshold = 100
         self.max_retries_per_target = 8
         
+        # --- Graph Polling Performance Settings ---
+        self.graph_polling_node_threshold = 100  # Stop graph auto-polling above this many nodes
+        
         # --- Provider Caching Settings ---
         self.cache_timeout_hours = 6  # Provider-specific cache timeout
         
@@ -72,6 +75,9 @@ class Config:
         self.max_retries_per_target = int(os.getenv('MAX_RETRIES_PER_TARGET', self.max_retries_per_target))
         self.cache_timeout_hours = int(os.getenv('CACHE_TIMEOUT_HOURS', self.cache_timeout_hours))
         
+        # Override graph polling threshold from environment
+        self.graph_polling_node_threshold = int(os.getenv('GRAPH_POLLING_NODE_THRESHOLD', self.graph_polling_node_threshold))
+        
         # Override Flask and session settings
         self.flask_host = os.getenv('FLASK_HOST', self.flask_host)
         self.flask_port = int(os.getenv('FLASK_PORT', self.flask_port))

core/__init__.py

@@ -1,5 +1,5 @@
 """
-Core modules for DNSRecon passive reconnaissance tool.
+Core modules for DNScope passive reconnaissance tool.
 Contains graph management, scanning orchestration, and forensic logging.
 """
 

core/graph_manager.py

@@ -1,7 +1,7 @@
-# dnsrecon-reduced/core/graph_manager.py
+# DNScope-reduced/core/graph_manager.py
 
 """
-Graph data model for DNSRecon using NetworkX.
+Graph data model for DNScope using NetworkX.
 Manages in-memory graph storage with confidence scoring and forensic metadata.
 Now fully compatible with the unified ProviderResult data model.
 UPDATED: Fixed correlation exclusion keys to match actual attribute names.
@@ -30,7 +30,7 @@ class NodeType(Enum):
 
 class GraphManager:
     """
-    Thread-safe graph manager for DNSRecon infrastructure mapping.
+    Thread-safe graph manager for DNScope infrastructure mapping.
     Uses NetworkX for in-memory graph storage with confidence scoring.
     Compatible with unified ProviderResult data model.
     """

core/logger.py

@@ -1,4 +1,4 @@
-# dnsrecon/core/logger.py
+# DNScope/core/logger.py
 
 import logging
 import threading
@@ -38,7 +38,7 @@ class RelationshipDiscovery:
 
 class ForensicLogger:
     """
-    Thread-safe forensic logging system for DNSRecon.
+    Thread-safe forensic logging system for DNScope.
     Maintains detailed audit trail of all reconnaissance activities.
     """
     
@@ -66,7 +66,7 @@ class ForensicLogger:
         }
         
         # Configure standard logger
-        self.logger = logging.getLogger(f'dnsrecon.{self.session_id}')
+        self.logger = logging.getLogger(f'DNScope.{self.session_id}')
         self.logger.setLevel(logging.INFO)
         
         # Create formatter for structured logging
@@ -94,7 +94,7 @@ class ForensicLogger:
         """Restore ForensicLogger after unpickling by reconstructing logger."""
         self.__dict__.update(state)
         # Re-initialize the 'logger' attribute
-        self.logger = logging.getLogger(f'dnsrecon.{self.session_id}')
+        self.logger = logging.getLogger(f'DNScope.{self.session_id}')
         self.logger.setLevel(logging.INFO)
         formatter = logging.Formatter(
             '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
@@ -107,7 +107,7 @@ class ForensicLogger:
 
     def _generate_session_id(self) -> str:
         """Generate unique session identifier."""
-        return f"dnsrecon_{datetime.now(timezone.utc).strftime('%Y%m%d_%H%M%S')}"
+        return f"DNScope_{datetime.now(timezone.utc).strftime('%Y%m%d_%H%M%S')}"
     
     def log_api_request(self, provider: str, url: str, method: str = "GET", 
                        status_code: Optional[int] = None, 

core/provider_result.py

@@ -1,7 +1,7 @@
-# dnsrecon-reduced/core/provider_result.py
+# DNScope-reduced/core/provider_result.py
 
 """
-Unified data model for DNSRecon passive reconnaissance.
+Unified data model for DNScope passive reconnaissance.
 Standardizes the data structure across all providers to ensure consistent processing.
 """
 

core/rate_limiter.py

@@ -1,28 +1,145 @@
-# dnsrecon-reduced/core/rate_limiter.py
+# DNScope-reduced/core/rate_limiter.py
 
 import time
+import logging
 
 class GlobalRateLimiter:
+    """
+    FIXED: Improved rate limiter with better cleanup and error handling.
+    Prevents accumulation of stale entries that cause infinite retry loops.
+    """
+    
     def __init__(self, redis_client):
         self.redis = redis_client
+        self.logger = logging.getLogger('DNScope.rate_limiter')
+        # Track last cleanup times to avoid excessive Redis operations
+        self._last_cleanup = {}
 
     def is_rate_limited(self, key, limit, period):
         """
-        Check if a key is rate-limited.
+        FIXED: Check if a key is rate-limited with improved cleanup and error handling.
+        
+        Args:
+            key: Rate limit key (e.g., provider name)
+            limit: Maximum requests allowed
+            period: Time period in seconds (60 for per-minute)
+            
+        Returns:
+            bool: True if rate limited, False otherwise
+        """
+        if limit <= 0:
+            # Rate limit of 0 or negative means no limiting
+            return False
+            
+        now = time.time()
+        rate_key = f"rate_limit:{key}"
+        
+        try:
+            # FIXED: More aggressive cleanup to prevent accumulation
+            # Only clean up if we haven't cleaned recently (every 10 seconds max)
+            should_cleanup = (
+                rate_key not in self._last_cleanup or 
+                now - self._last_cleanup.get(rate_key, 0) > 10
+            )
+            
+            if should_cleanup:
+                # Remove entries older than the period
+                removed_count = self.redis.zremrangebyscore(rate_key, 0, now - period)
+                self._last_cleanup[rate_key] = now
+                
+                if removed_count > 0:
+                    self.logger.debug(f"Rate limiter cleaned up {removed_count} old entries for {key}")
+            
+            # Get current count
+            current_count = self.redis.zcard(rate_key)
+            
+            if current_count >= limit:
+                self.logger.debug(f"Rate limited: {key} has {current_count}/{limit} requests in period")
+                return True
+            
+            # Add new timestamp with error handling
+            try:
+                # Use pipeline for atomic operations
+                pipe = self.redis.pipeline()
+                pipe.zadd(rate_key, {str(now): now})
+                pipe.expire(rate_key, int(period * 2))  # Set TTL to 2x period for safety
+                pipe.execute()
+            except Exception as e:
+                self.logger.warning(f"Failed to record rate limit entry for {key}: {e}")
+                # Don't block the request if we can't record it
+                return False
+            
+            return False
+            
+        except Exception as e:
+            self.logger.error(f"Rate limiter error for {key}: {e}")
+            # FIXED: On Redis errors, don't block requests to avoid infinite loops
+            return False
+    
+    def get_rate_limit_status(self, key, limit, period):
+        """
+        Get detailed rate limit status for debugging.
+        
+        Returns:
+            dict: Status information including current count, limit, and time to reset
         """
         now = time.time()
-        key = f"rate_limit:{key}"
+        rate_key = f"rate_limit:{key}"
-
+        
-        # Remove old timestamps
+        try:
-        self.redis.zremrangebyscore(key, 0, now - period)
+            current_count = self.redis.zcard(rate_key)
-
+            
-        # Check the count
+            # Get oldest entry to calculate reset time
-        count = self.redis.zcard(key)
+            oldest_entries = self.redis.zrange(rate_key, 0, 0, withscores=True)
-        if count >= limit:
+            time_to_reset = 0
+            if oldest_entries:
+                oldest_time = oldest_entries[0][1]
+                time_to_reset = max(0, period - (now - oldest_time))
+            
+            return {
+                'key': key,
+                'current_count': current_count,
+                'limit': limit,
+                'period': period,
+                'is_limited': current_count >= limit,
+                'time_to_reset': time_to_reset
+            }
+        except Exception as e:
+            self.logger.error(f"Failed to get rate limit status for {key}: {e}")
+            return {
+                'key': key,
+                'current_count': 0,
+                'limit': limit,
+                'period': period,
+                'is_limited': False,
+                'time_to_reset': 0,
+                'error': str(e)
+            }
+    
+    def reset_rate_limit(self, key):
+        """
+        ADDED: Reset rate limit for a specific key (useful for debugging).
+        """
+        rate_key = f"rate_limit:{key}"
+        try:
+            deleted = self.redis.delete(rate_key)
+            self.logger.info(f"Reset rate limit for {key} (deleted: {deleted})")
             return True
-
+        except Exception as e:
-        # Add new timestamp
+            self.logger.error(f"Failed to reset rate limit for {key}: {e}")
-        self.redis.zadd(key, {now: now})
+            return False
-        self.redis.expire(key, period)
+    
-
+    def cleanup_all_rate_limits(self):
-        return False
+        """
+        ADDED: Clean up all rate limit entries (useful for maintenance).
+        """
+        try:
+            keys = self.redis.keys("rate_limit:*")
+            if keys:
+                deleted = self.redis.delete(*keys)
+                self.logger.info(f"Cleaned up {deleted} rate limit keys")
+                return deleted
+            return 0
+        except Exception as e:
+            self.logger.error(f"Failed to cleanup rate limits: {e}")
+            return 0

core/scanner.py

@@ -1,4 +1,4 @@
-# dnsrecon-reduced/core/scanner.py
+# DNScope-reduced/core/scanner.py
 
 import threading
 import traceback
@@ -27,6 +27,7 @@ class ScanStatus:
     """Enumeration of scan statuses."""
     IDLE = "idle"
     RUNNING = "running"
+    FINALIZING = "finalizing"  # New state for post-scan analysis
     COMPLETED = "completed"
     FAILED = "failed"
     STOPPED = "stopped"
@@ -34,7 +35,7 @@ class ScanStatus:
 
 class Scanner:
     """
-    Main scanning orchestrator for DNSRecon passive reconnaissance.
+    Main scanning orchestrator for DNScope passive reconnaissance.
     UNIFIED: Combines comprehensive features with improved display formatting.
     """
 
@@ -192,6 +193,8 @@ class Scanner:
         
         print(f"=== INITIALIZING PROVIDERS FROM {provider_dir} ===")
         
+        correlation_provider_instance = None
+        
         for filename in os.listdir(provider_dir):
             if filename.endswith('_provider.py') and not filename.startswith('base'):
                 module_name = f"providers.{filename[:-3]}"
@@ -202,7 +205,6 @@ class Scanner:
                         attribute = getattr(module, attribute_name)
                         if isinstance(attribute, type) and issubclass(attribute, BaseProvider) and attribute is not BaseProvider:
                             provider_class = attribute
-                            # FIXED: Pass the 'name' argument during initialization
                             provider = provider_class(name=attribute_name, session_config=self.config)
                             provider_name = provider.get_name()
 
@@ -223,8 +225,13 @@ class Scanner:
                                 
                                 if is_available:
                                     provider.set_stop_event(self.stop_event)
+                                    
+                                    # Special handling for correlation provider
                                     if isinstance(provider, CorrelationProvider):
                                         provider.set_graph_manager(self.graph)
+                                        correlation_provider_instance = provider
+                                        print(f"    ✓ Correlation provider configured with graph manager")
+                                    
                                     self.providers.append(provider)
                                     print(f"    ✓ Added to scanner")
                                 else:
@@ -239,6 +246,11 @@ class Scanner:
         print(f"=== PROVIDER INITIALIZATION COMPLETE ===")
         print(f"Active providers: {[p.get_name() for p in self.providers]}")
         print(f"Provider count: {len(self.providers)}")
+        
+        # Verify correlation provider is properly configured
+        if correlation_provider_instance:
+            print(f"Correlation provider configured: {correlation_provider_instance.graph is not None}")
+        
         print("=" * 50)
 
     def _status_logger_thread(self):
@@ -450,12 +462,10 @@ class Scanner:
 
     def _execute_scan(self, target: str, max_depth: int) -> None:
         self.executor = ThreadPoolExecutor(max_workers=self.max_workers)
-        processed_tasks = set()  # FIXED: Now includes depth to avoid incorrect skipping
+        processed_tasks = set()
 
         is_ip = _is_valid_ip(target)
-        initial_providers = self._get_eligible_providers(target, is_ip, False)
+        initial_providers = [p for p in self._get_eligible_providers(target, is_ip, False) if not isinstance(p, CorrelationProvider)]
-        # FIXED: Filter out correlation provider from initial providers
-        initial_providers = [p for p in initial_providers if not isinstance(p, CorrelationProvider)]
         
         for provider in initial_providers:
             provider_name = provider.get_name()
@@ -474,9 +484,8 @@ class Scanner:
             self.graph.add_node(target, node_type)
             self._initialize_provider_states(target)
             consecutive_empty_iterations = 0
-            max_empty_iterations = 50  # Allow 5 seconds of empty queue before considering completion
+            max_empty_iterations = 50
 
-            # PHASE 1: Run all non-correlation providers
             print(f"\n=== PHASE 1: Running non-correlation providers ===")
             while not self._is_stop_requested():
                 queue_empty = self.task_queue.empty()
@@ -486,57 +495,39 @@ class Scanner:
                 if queue_empty and no_active_processing:
                     consecutive_empty_iterations += 1
                     if consecutive_empty_iterations >= max_empty_iterations:
-                        break  # Phase 1 complete
+                        break
                     time.sleep(0.1)
                     continue
                 else:
                     consecutive_empty_iterations = 0
 
-                # Process tasks (same logic as before, but correlations are filtered out)
                 try:
                     run_at, priority, (provider_name, target_item, depth) = self.task_queue.get(timeout=0.1)
-                    
+                    if provider_name == 'correlation': continue
-                    # Skip correlation tasks during Phase 1
-                    if provider_name == 'correlation':
-                        continue
-                    
-                    # Check if task is ready to run
                     current_time = time.time()
                     if run_at > current_time:
                         self.task_queue.put((run_at, priority, (provider_name, target_item, depth)))
                         time.sleep(min(0.5, run_at - current_time))
                         continue
-                            
+                except:
-                except:  # Queue is empty or timeout occurred
                     time.sleep(0.1)
                     continue
 
                 self.last_task_from_queue = (run_at, priority, (provider_name, target_item, depth))
-
-                # Skip if already processed
                 task_tuple = (provider_name, target_item, depth)
-                if task_tuple in processed_tasks:
+                if task_tuple in processed_tasks or depth > max_depth:
-                    self.tasks_skipped += 1
-                    self.indicators_completed += 1
-                    continue
-
-                # Skip if depth exceeded
-                if depth > max_depth:
                     self.tasks_skipped += 1
                     self.indicators_completed += 1
                     continue
                 
-                # Rate limiting with proper time-based deferral
                 if self.rate_limiter.is_rate_limited(provider_name, self.config.get_rate_limit(provider_name), 60):
                     defer_until = time.time() + 60
                     self.task_queue.put((defer_until, priority, (provider_name, target_item, depth)))
                     self.tasks_re_enqueued += 1
                     continue
 
-                # Thread-safe processing state management
                 with self.processing_lock:
-                    if self._is_stop_requested(): 
+                    if self._is_stop_requested(): break
-                        break
                     processing_key = (provider_name, target_item)
                     if processing_key in self.currently_processing:
                         self.tasks_skipped += 1
@@ -548,29 +539,21 @@ class Scanner:
                     self.current_depth = depth
                     self.current_indicator = target_item
                     self._update_session_state()
-
+                    if self._is_stop_requested(): break
-                    if self._is_stop_requested(): 
-                        break
                     
                     provider = next((p for p in self.providers if p.get_name() == provider_name), None)
-
                     if provider and not isinstance(provider, CorrelationProvider):
                         new_targets, _, success = self._process_provider_task(provider, target_item, depth)
-
+                        if self._is_stop_requested(): break
-                        if self._is_stop_requested(): 
-                            break
 
                         if not success:
                             retry_key = (provider_name, target_item, depth)
                             self.target_retries[retry_key] += 1
-                            
                             if self.target_retries[retry_key] <= self.config.max_retries_per_target:
                                 retry_count = self.target_retries[retry_key]
                                 backoff_delay = min(300, (2 ** retry_count) + random.uniform(0, 1))
-                                retry_at = time.time() + backoff_delay
+                                self.task_queue.put((time.time() + backoff_delay, priority, (provider_name, target_item, depth)))
-                                self.task_queue.put((retry_at, priority, (provider_name, target_item, depth)))
                                 self.tasks_re_enqueued += 1
-                                self.logger.logger.debug(f"Retrying {provider_name}:{target_item} in {backoff_delay:.1f}s (attempt {retry_count})")
                             else:
                                 self.scan_failed_due_to_retries = True
                                 self._log_target_processing_error(str(task_tuple), f"Max retries ({self.config.max_retries_per_target}) exceeded")
@@ -578,54 +561,33 @@ class Scanner:
                             processed_tasks.add(task_tuple)
                             self.indicators_completed += 1
 
-                        # Enqueue new targets with proper depth tracking
                         if not self._is_stop_requested():
                             for new_target in new_targets:
                                 is_ip_new = _is_valid_ip(new_target)
-                                eligible_providers_new = self._get_eligible_providers(new_target, is_ip_new, False)
+                                eligible_providers_new = [p for p in self._get_eligible_providers(new_target, is_ip_new, False) if not isinstance(p, CorrelationProvider)]
-                                # FIXED: Filter out correlation providers in Phase 1
-                                eligible_providers_new = [p for p in eligible_providers_new if not isinstance(p, CorrelationProvider)]
-                                
                                 for p_new in eligible_providers_new:
                                     p_name_new = p_new.get_name()
                                     new_depth = depth + 1
-                                    new_task_tuple = (p_name_new, new_target, new_depth)
+                                    if (p_name_new, new_target, new_depth) not in processed_tasks and new_depth <= max_depth:
-                                    
+                                        self.task_queue.put((time.time(), self._get_priority(p_name_new), (p_name_new, new_target, new_depth)))
-                                    if new_task_tuple not in processed_tasks and new_depth <= max_depth:
-                                        new_priority = self._get_priority(p_name_new)
-                                        self.task_queue.put((time.time(), new_priority, (p_name_new, new_target, new_depth)))
                                         self.total_tasks_ever_enqueued += 1
                     else:
-                        self.logger.logger.warning(f"Provider {provider_name} not found in active providers")
                         self.tasks_skipped += 1
                         self.indicators_completed += 1
-                        
                 finally:
                     with self.processing_lock:
-                        processing_key = (provider_name, target_item)
+                        self.currently_processing.discard((provider_name, target_item))
-                        self.currently_processing.discard(processing_key)
+            
+            # This code runs after the main loop finishes or is stopped.
+            self.status = ScanStatus.FINALIZING
+            self._update_session_state()
+            self.logger.logger.info("Scan stopped or completed. Entering finalization phase.")
 
-            # PHASE 2: Run correlations on all discovered nodes
+            if self.status in [ScanStatus.FINALIZING, ScanStatus.COMPLETED, ScanStatus.STOPPED]:
-            if not self._is_stop_requested():
                 print(f"\n=== PHASE 2: Running correlation analysis ===")
                 self._run_correlation_phase(max_depth, processed_tasks)
 
-        except Exception as e:
+            # Determine the final status *after* finalization.
-            traceback.print_exc()
-            self.status = ScanStatus.FAILED
-            self.logger.logger.error(f"Scan failed: {e}")
-        finally:
-            # Comprehensive cleanup (same as before)
-            with self.processing_lock:
-                self.currently_processing.clear()
-            self.currently_processing_display = []
-
-            while not self.task_queue.empty():
-                try:
-                    self.task_queue.get_nowait()
-                except:
-                    break
-
             if self._is_stop_requested():
                 self.status = ScanStatus.STOPPED
             elif self.scan_failed_due_to_retries:
@@ -633,13 +595,25 @@ class Scanner:
             else:
                 self.status = ScanStatus.COMPLETED
 
+        except Exception as e:
+            traceback.print_exc()
+            self.status = ScanStatus.FAILED
+            self.logger.logger.error(f"Scan failed: {e}")
+        finally:
+            # The 'finally' block is now only for guaranteed cleanup.
+            with self.processing_lock:
+                self.currently_processing.clear()
+            self.currently_processing_display = []
+
+            while not self.task_queue.empty():
+                try: self.task_queue.get_nowait()
+                except: break
+
             self.status_logger_stop_event.set()
             if self.status_logger_thread and self.status_logger_thread.is_alive():
                 self.status_logger_thread.join(timeout=2.0)
-
-            self._update_session_state()
-            self.logger.log_scan_complete()
             
+            # The executor shutdown now happens *after* the correlation phase has run.
             if self.executor:
                 try:
                     self.executor.shutdown(wait=False, cancel_futures=True)
@@ -647,27 +621,32 @@ class Scanner:
                     self.logger.logger.warning(f"Error shutting down executor: {e}")
                 finally:
                     self.executor = None
+            
+            self._update_session_state()
+            self.logger.log_scan_complete()
 
     def _run_correlation_phase(self, max_depth: int, processed_tasks: set) -> None:
         """
         PHASE 2: Run correlation analysis on all discovered nodes.
-        This ensures correlations run after all other providers have completed.
+        Enhanced with better error handling and progress tracking.
         """
         correlation_provider = next((p for p in self.providers if isinstance(p, CorrelationProvider)), None)
         if not correlation_provider:
             print("No correlation provider found - skipping correlation phase")
             return
 
+        # Ensure correlation provider has access to current graph state
+        correlation_provider.set_graph_manager(self.graph)
+        print(f"Correlation provider configured with graph containing {self.graph.get_node_count()} nodes")
+
         # Get all nodes from the graph for correlation analysis
         all_nodes = list(self.graph.graph.nodes())
         correlation_tasks = []
+        correlation_tasks_enqueued = 0
         
         print(f"Enqueueing correlation tasks for {len(all_nodes)} nodes")
         
         for node_id in all_nodes:
-            if self._is_stop_requested():
-                break
-                
             # Determine appropriate depth for correlation (use 0 for simplicity)
             correlation_depth = 0
             task_tuple = ('correlation', node_id, correlation_depth)
@@ -677,15 +656,22 @@ class Scanner:
                 priority = self._get_priority('correlation')
                 self.task_queue.put((time.time(), priority, ('correlation', node_id, correlation_depth)))
                 correlation_tasks.append(task_tuple)
+                correlation_tasks_enqueued += 1
                 self.total_tasks_ever_enqueued += 1
         
-        print(f"Enqueued {len(correlation_tasks)} correlation tasks")
+        print(f"Enqueued {correlation_tasks_enqueued} new correlation tasks")
         
-        # Process correlation tasks
+        # Force session state update to reflect new task count
+        self._update_session_state()
+        
+        # Process correlation tasks with enhanced tracking
         consecutive_empty_iterations = 0
-        max_empty_iterations = 20  # Shorter timeout for correlation phase
+        max_empty_iterations = 20
+        correlation_completed = 0
+        correlation_errors = 0
         
-        while not self._is_stop_requested() and correlation_tasks:
+        while correlation_tasks:
+            # Check if we should continue processing
             queue_empty = self.task_queue.empty()
             with self.processing_lock:
                 no_active_processing = len(self.currently_processing) == 0
@@ -693,6 +679,7 @@ class Scanner:
             if queue_empty and no_active_processing:
                 consecutive_empty_iterations += 1
                 if consecutive_empty_iterations >= max_empty_iterations:
+                    print(f"Correlation phase timeout - {len(correlation_tasks)} tasks remaining")
                     break
                 time.sleep(0.1)
                 continue
@@ -721,8 +708,6 @@ class Scanner:
                 continue
 
             with self.processing_lock:
-                if self._is_stop_requested():
-                    break
                 processing_key = (provider_name, target_item)
                 if processing_key in self.currently_processing:
                     self.tasks_skipped += 1
@@ -734,36 +719,52 @@ class Scanner:
                 self.current_indicator = target_item
                 self._update_session_state()
                 
-                if self._is_stop_requested():
+                # Process correlation task with enhanced error handling
-                    break
+                try:
-                
+                    new_targets, _, success = self._process_provider_task(correlation_provider, target_item, depth)
-                # Process correlation task
-                new_targets, _, success = self._process_provider_task(correlation_provider, target_item, depth)
-                
-                if success:
-                    processed_tasks.add(task_tuple)
-                    self.indicators_completed += 1
-                    if task_tuple in correlation_tasks:
-                        correlation_tasks.remove(task_tuple)
-                else:
-                    # For correlations, don't retry - just mark as completed
-                    self.indicators_completed += 1
-                    if task_tuple in correlation_tasks:
-                        correlation_tasks.remove(task_tuple)
                     
+                    if success:
+                        processed_tasks.add(task_tuple)
+                        correlation_completed += 1
+                        self.indicators_completed += 1
+                        if task_tuple in correlation_tasks:
+                            correlation_tasks.remove(task_tuple)
+                    else:
+                        # For correlations, don't retry - just mark as completed
+                        correlation_errors += 1
+                        self.indicators_completed += 1
+                        if task_tuple in correlation_tasks:
+                            correlation_tasks.remove(task_tuple)
+                            
+                except Exception as e:
+                    correlation_errors += 1
+                    self.indicators_completed += 1
+                    if task_tuple in correlation_tasks:
+                        correlation_tasks.remove(task_tuple)
+                    self.logger.logger.warning(f"Correlation task failed for {target_item}: {e}")
+                        
             finally:
                 with self.processing_lock:
                     processing_key = (provider_name, target_item)
                     self.currently_processing.discard(processing_key)
 
-        print(f"Correlation phase complete. Remaining tasks: {len(correlation_tasks)}")
+            # Periodic progress update during correlation phase
+            if correlation_completed % 10 == 0 and correlation_completed > 0:
+                remaining = len(correlation_tasks)
+                print(f"Correlation progress: {correlation_completed} completed, {remaining} remaining")
+
+        print(f"Correlation phase complete:")
+        print(f"  - Successfully processed: {correlation_completed}")
+        print(f"  - Errors encountered: {correlation_errors}")
+        print(f"  - Tasks remaining: {len(correlation_tasks)}")
+
 
     def _process_provider_task(self, provider: BaseProvider, target: str, depth: int) -> Tuple[Set[str], Set[str], bool]:
         """
         Manages the entire process for a given target and provider.
         This version is generalized to handle all relationships dynamically.
         """
-        if self._is_stop_requested():
+        if self._is_stop_requested() and not isinstance(provider, CorrelationProvider):
             return set(), set(), False
     
         is_ip = _is_valid_ip(target)
@@ -780,7 +781,8 @@ class Scanner:
     
             if provider_result is None:
                 provider_successful = False
-            elif not self._is_stop_requested():
+            # Allow correlation provider to process results even if scan is stopped
+            elif not self._is_stop_requested() or isinstance(provider, CorrelationProvider):
                 # Pass all relationships to be processed
                 discovered, is_large_entity = self._process_provider_result_unified(
                     target, provider, provider_result, depth
@@ -800,7 +802,7 @@ class Scanner:
         provider_name = provider.get_name()
         start_time = datetime.now(timezone.utc)
         
-        if self._is_stop_requested():
+        if self._is_stop_requested() and not isinstance(provider, CorrelationProvider):
             return None
         
         try:
@@ -809,7 +811,7 @@ class Scanner:
             else:
                 result = provider.query_domain(target)
             
-            if self._is_stop_requested():
+            if self._is_stop_requested() and not isinstance(provider, CorrelationProvider):
                 return None
             
             relationship_count = result.get_relationship_count() if result else 0
@@ -822,18 +824,34 @@ class Scanner:
             return None
 
     def _create_large_entity_from_result(self, source_node: str, provider_name: str,
-                                         provider_result: ProviderResult, depth: int) -> Tuple[str, Set[str]]:
+                                        provider_result: ProviderResult, depth: int) -> Tuple[str, Set[str]]:
         """
-        Creates a large entity node, tags all member nodes, and returns its ID and members.
+        Creates a large entity node, tags all member nodes, and stores original relationships.
+        FIXED: Now stores original relationships for later restoration during extraction.
         """
         members = {rel.target_node for rel in provider_result.relationships
-                   if _is_valid_domain(rel.target_node) or _is_valid_ip(rel.target_node)}
+                if _is_valid_domain(rel.target_node) or _is_valid_ip(rel.target_node)}
         
         if not members:
             return "", set()
 
         large_entity_id = f"le_{provider_name}_{source_node}"
         
+        # FIXED: Store original relationships for each member
+        member_relationships = {}
+        for rel in provider_result.relationships:
+            if rel.target_node in members:
+                if rel.target_node not in member_relationships:
+                    member_relationships[rel.target_node] = []
+                member_relationships[rel.target_node].append({
+                    'source_node': rel.source_node,
+                    'target_node': rel.target_node,
+                    'relationship_type': rel.relationship_type,
+                    'confidence': rel.confidence,
+                    'provider': rel.provider,
+                    'raw_data': rel.raw_data
+                })
+        
         self.graph.add_node(
             node_id=large_entity_id,
             node_type=NodeType.LARGE_ENTITY,
@@ -841,7 +859,8 @@ class Scanner:
                 {"name": "count", "value": len(members), "type": "statistic"},
                 {"name": "source_provider", "value": provider_name, "type": "metadata"},
                 {"name": "discovery_depth", "value": depth, "type": "metadata"},
-                {"name": "nodes", "value": list(members), "type": "metadata"}
+                {"name": "nodes", "value": list(members), "type": "metadata"},
+                {"name": "original_relationships", "value": member_relationships, "type": "metadata"}  # FIXED: Store original relationships
             ],
             description=f"A collection of {len(members)} nodes discovered from {source_node} via {provider_name}."
         )
@@ -858,7 +877,8 @@ class Scanner:
 
     def extract_node_from_large_entity(self, large_entity_id: str, node_id: str) -> bool:
         """
-        Removes a node from a large entity, allowing it to be processed normally.
+        Removes a node from a large entity and restores its original relationships.
+        FIXED: Now restores original relationships to make the node reachable.
         """
         if not self.graph.graph.has_node(node_id):
             return False
@@ -866,31 +886,67 @@ class Scanner:
         node_data = self.graph.graph.nodes[node_id]
         metadata = node_data.get('metadata', {})
         
-        if metadata.get('large_entity_id') == large_entity_id:
+        if metadata.get('large_entity_id') != large_entity_id:
-            # Remove the large entity tag
+            return False
-            del metadata['large_entity_id']
+        
-            self.graph.add_node(node_id, NodeType(node_data['type']), metadata=metadata)
+        # Remove the large entity tag
+        del metadata['large_entity_id']
+        self.graph.add_node(node_id, NodeType(node_data['type']), metadata=metadata)
+        
+        # FIXED: Restore original relationships if they exist
+        if self.graph.graph.has_node(large_entity_id):
+            le_attrs = self.graph.graph.nodes[large_entity_id].get('attributes', [])
+            original_relationships_attr = next((a for a in le_attrs if a['name'] == 'original_relationships'), None)
             
-            # Re-enqueue the node for full processing
+            if original_relationships_attr and node_id in original_relationships_attr['value']:
-            is_ip = _is_valid_ip(node_id)
+                # Restore all original relationships for this node
-            eligible_providers = self._get_eligible_providers(node_id, is_ip, False)
+                for rel_data in original_relationships_attr['value'][node_id]:
-            for provider in eligible_providers:
+                    self.graph.add_edge(
-                provider_name = provider.get_name()
+                        source_id=rel_data['source_node'],
-                priority = self._get_priority(provider_name)
+                        target_id=rel_data['target_node'],
-                # Use current depth of the large entity if available, else 0
+                        relationship_type=rel_data['relationship_type'],
-                depth = 0
+                        confidence_score=rel_data['confidence'],
-                if self.graph.graph.has_node(large_entity_id):
+                        source_provider=rel_data['provider'],
-                    le_attrs = self.graph.graph.nodes[large_entity_id].get('attributes', [])
+                        raw_data=rel_data['raw_data']
-                    depth_attr = next((a for a in le_attrs if a['name'] == 'discovery_depth'), None)
+                    )
-                    if depth_attr:
+                    
-                        depth = depth_attr['value']
+                    # Ensure both nodes exist in the graph
+                    source_type = NodeType.IP if _is_valid_ip(rel_data['source_node']) else NodeType.DOMAIN
+                    target_type = NodeType.IP if _is_valid_ip(rel_data['target_node']) else NodeType.DOMAIN
+                    self.graph.add_node(rel_data['source_node'], source_type)
+                    self.graph.add_node(rel_data['target_node'], target_type)
+                
+                # Update the large entity to remove this node from its list
+                nodes_attr = next((a for a in le_attrs if a['name'] == 'nodes'), None)
+                if nodes_attr and node_id in nodes_attr['value']:
+                    nodes_attr['value'].remove(node_id)
+                    
+                count_attr = next((a for a in le_attrs if a['name'] == 'count'), None)
+                if count_attr:
+                    count_attr['value'] = max(0, count_attr['value'] - 1)
+                
+                # Remove from original relationships tracking
+                if node_id in original_relationships_attr['value']:
+                    del original_relationships_attr['value'][node_id]
+        
+        # Re-enqueue the node for full processing
+        is_ip = _is_valid_ip(node_id)
+        eligible_providers = self._get_eligible_providers(node_id, is_ip, False)
+        for provider in eligible_providers:
+            provider_name = provider.get_name()
+            priority = self._get_priority(provider_name)
+            # Use current depth of the large entity if available, else 0
+            depth = 0
+            if self.graph.graph.has_node(large_entity_id):
+                le_attrs = self.graph.graph.nodes[large_entity_id].get('attributes', [])
+                depth_attr = next((a for a in le_attrs if a['name'] == 'discovery_depth'), None)
+                if depth_attr:
+                    depth = depth_attr['value']
 
-                self.task_queue.put((time.time(), priority, (provider_name, node_id, depth)))
+            self.task_queue.put((time.time(), priority, (provider_name, node_id, depth)))
-                self.total_tasks_ever_enqueued += 1
+            self.total_tasks_ever_enqueued += 1
-            
+        
-            return True
+        return True
-            
-        return False
 
     def _process_provider_result_unified(self, target: str, provider: BaseProvider,
                                         provider_result: ProviderResult, current_depth: int) -> Tuple[Set[str], bool]:
@@ -903,7 +959,8 @@ class Scanner:
         large_entity_id = ""
         large_entity_members = set()
 
-        if self._is_stop_requested():
+        # Stop processing for non-correlation providers if requested
+        if self._is_stop_requested() and not isinstance(provider, CorrelationProvider):
             return discovered_targets, False
 
         eligible_rel_count = sum(
@@ -917,7 +974,8 @@ class Scanner:
             )
 
         for i, relationship in enumerate(provider_result.relationships):
-            if i % 5 == 0 and self._is_stop_requested():
+            # Stop processing for non-correlation providers if requested
+            if i % 5 == 0 and self._is_stop_requested() and not isinstance(provider, CorrelationProvider):
                 break
 
             source_node_id = relationship.source_node
@@ -1182,6 +1240,10 @@ class Scanner:
         self.logger.logger.error(f"Provider {provider_name} failed for {target}: {error}")
 
     def _calculate_progress(self) -> float:
+        """
+        Enhanced progress calculation that properly accounts for correlation tasks
+        added during the correlation phase.
+        """
         try:
             if self.total_tasks_ever_enqueued == 0:
                 return 0.0
@@ -1191,7 +1253,18 @@ class Scanner:
             with self.processing_lock:
                 active_tasks = len(self.currently_processing)
             
-            # Adjust total to account for remaining work
+            # For correlation phase, be more conservative about progress calculation
+            if self.status == ScanStatus.FINALIZING:
+                # During correlation phase, show progress more conservatively
+                base_progress = (self.indicators_completed / max(self.total_tasks_ever_enqueued, 1)) * 100
+                
+                # If we have active correlation tasks, cap progress at 95% until done
+                if queue_size > 0 or active_tasks > 0:
+                    return min(95.0, base_progress)
+                else:
+                    return min(100.0, base_progress)
+            
+            # Normal phase progress calculation
             adjusted_total = max(self.total_tasks_ever_enqueued, 
                             self.indicators_completed + queue_size + active_tasks)
             

core/session_config.py

@@ -1,5 +1,5 @@
 """
-Per-session configuration management for DNSRecon.
+Per-session configuration management for DNScope.
 Provides isolated configuration instances for each user session.
 """
 

core/session_manager.py

@@ -1,4 +1,4 @@
-# dnsrecon/core/session_manager.py
+# DNScope/core/session_manager.py
 
 import threading
 import time
@@ -58,11 +58,11 @@ class SessionManager:
 
     def _get_session_key(self, session_id: str) -> str:
         """Generates the Redis key for a session."""
-        return f"dnsrecon:session:{session_id}"
+        return f"DNScope:session:{session_id}"
 
     def _get_stop_signal_key(self, session_id: str) -> str:
         """Generates the Redis key for a session's stop signal."""
-        return f"dnsrecon:stop:{session_id}"
+        return f"DNScope:stop:{session_id}"
 
     def create_session(self) -> str:
         """
@@ -353,7 +353,7 @@ class SessionManager:
         while True:
             try:
                 # Clean up orphaned stop signals
-                stop_keys = self.redis_client.keys("dnsrecon:stop:*")
+                stop_keys = self.redis_client.keys("DNScope:stop:*")
                 for stop_key in stop_keys:
                     # Extract session ID from stop key
                     session_id = stop_key.decode('utf-8').split(':')[-1]
@@ -372,8 +372,8 @@ class SessionManager:
     def get_statistics(self) -> Dict[str, Any]:
         """Get session manager statistics."""
         try:
-            session_keys = self.redis_client.keys("dnsrecon:session:*")
+            session_keys = self.redis_client.keys("DNScope:session:*")
-            stop_keys = self.redis_client.keys("dnsrecon:stop:*")
+            stop_keys = self.redis_client.keys("DNScope:stop:*")
             
             active_sessions = len(session_keys)
             running_scans = 0

providers/__init__.py

@@ -1,5 +1,5 @@
 """
-Data provider modules for DNSRecon.
+Data provider modules for DNScope.
 Contains implementations for various reconnaissance data sources.
 """
 

providers/base_provider.py

@@ -1,4 +1,4 @@
-# dnsrecon/providers/base_provider.py
+# DNScope/providers/base_provider.py
 
 import time
 import requests
@@ -13,7 +13,7 @@ from core.provider_result import ProviderResult
 
 class BaseProvider(ABC):
     """
-    Abstract base class for all DNSRecon data providers.
+    Abstract base class for all DNScope data providers.
     Now supports session-specific configuration and returns standardized ProviderResult objects.
     """
 
@@ -72,7 +72,7 @@ class BaseProvider(ABC):
         if not hasattr(self._local, 'session'):
             self._local.session = requests.Session()
             self._local.session.headers.update({
-                'User-Agent': 'DNSRecon/1.0 (Passive Reconnaissance Tool)'
+                'User-Agent': 'DNScope/1.0 (Passive Reconnaissance Tool)'
             })
         return self._local.session
 

providers/correlation_provider.py

@@ -1,4 +1,4 @@
-# dnsrecon/providers/correlation_provider.py
+# DNScope/providers/correlation_provider.py
 
 import re
 from typing import Dict, Any, List
@@ -27,6 +27,7 @@ class CorrelationProvider(BaseProvider):
             'cert_validity_period_days',
             'cert_issuer_name',
             'cert_entry_timestamp',
+            'cert_serial_number', # useless
             'cert_not_before',
             'cert_not_after',
             'dns_ttl',
@@ -77,90 +78,166 @@ class CorrelationProvider(BaseProvider):
 
     def _find_correlations(self, node_id: str) -> ProviderResult:
         """
-        Find correlations for a given node.
+        Find correlations for a given node with enhanced filtering and error handling.
         """
         result = ProviderResult()
-        # FIXED: Ensure self.graph is not None before proceeding.
+        
+        # Enhanced safety checks
         if not self.graph or not self.graph.graph.has_node(node_id):
             return result
 
-        node_attributes = self.graph.graph.nodes[node_id].get('attributes', [])
+        try:
+            node_attributes = self.graph.graph.nodes[node_id].get('attributes', [])
+            
+            # Ensure attributes is a list (handle legacy data)
+            if not isinstance(node_attributes, list):
+                return result
+                
+            correlations_found = 0
+            
+            for attr in node_attributes:
+                if not isinstance(attr, dict):
+                    continue
+                    
+                attr_name = attr.get('name', '')
+                attr_value = attr.get('value')
+                attr_provider = attr.get('provider', 'unknown')
 
-        for attr in node_attributes:
+                # Enhanced filtering logic
-            attr_name = attr.get('name')
+                should_exclude = self._should_exclude_attribute(attr_name, attr_value)
-            attr_value = attr.get('value')
+                
-            attr_provider = attr.get('provider', 'unknown')
+                if should_exclude:
+                    continue
 
-            should_exclude = (
+                # Build correlation index
-                any(excluded_key in attr_name or attr_name == excluded_key for excluded_key in self.EXCLUDED_KEYS) or
+                if attr_value not in self.correlation_index:
-                not isinstance(attr_value, (str, int, float, bool)) or
+                    self.correlation_index[attr_value] = {
-                attr_value is None or
+                        'nodes': set(),
-                isinstance(attr_value, bool) or
+                        'sources': []
-                (isinstance(attr_value, str) and (
+                    }
-                    len(attr_value) < 4 or
-                    self.date_pattern.match(attr_value) or
-                    attr_value.lower() in ['unknown', 'none', 'null', 'n/a', 'true', 'false', '0', '1']
-                )) or
-                (isinstance(attr_value, (int, float)) and (
-                    attr_value == 0 or
-                    attr_value == 1 or
-                    abs(attr_value) > 1000000
-                ))
-            )
 
-            if should_exclude:
+                self.correlation_index[attr_value]['nodes'].add(node_id)
-                continue
 
-            if attr_value not in self.correlation_index:
+                source_info = {
-                self.correlation_index[attr_value] = {
+                    'node_id': node_id,
-                    'nodes': set(),
+                    'provider': attr_provider,
-                    'sources': []
+                    'attribute': attr_name,
+                    'path': f"{attr_provider}_{attr_name}"
                 }
 
-            self.correlation_index[attr_value]['nodes'].add(node_id)
+                # Avoid duplicate sources
+                existing_sources = [s for s in self.correlation_index[attr_value]['sources']
+                                if s['node_id'] == node_id and s['path'] == source_info['path']]
+                if not existing_sources:
+                    self.correlation_index[attr_value]['sources'].append(source_info)
 
-            source_info = {
+                # Create correlation if we have multiple nodes with this value
-                'node_id': node_id,
+                if len(self.correlation_index[attr_value]['nodes']) > 1:
-                'provider': attr_provider,
+                    self._create_correlation_relationships(attr_value, self.correlation_index[attr_value], result)
-                'attribute': attr_name,
+                    correlations_found += 1
-                'path': f"{attr_provider}_{attr_name}"
+                    
-            }
+            # Log correlation results
-
+            if correlations_found > 0:
-            existing_sources = [s for s in self.correlation_index[attr_value]['sources']
+                self.logger.logger.info(f"Found {correlations_found} correlations for node {node_id}")
-                              if s['node_id'] == node_id and s['path'] == source_info['path']]
+                
-            if not existing_sources:
+        except Exception as e:
-                self.correlation_index[attr_value]['sources'].append(source_info)
+            self.logger.logger.error(f"Error finding correlations for {node_id}: {e}")
-
+            
-            if len(self.correlation_index[attr_value]['nodes']) > 1:
-                self._create_correlation_relationships(attr_value, self.correlation_index[attr_value], result)
         return result
-
+    
+    def _should_exclude_attribute(self, attr_name: str, attr_value: Any) -> bool:
+        """
+        Enhanced logic to determine if an attribute should be excluded from correlation.
+        """
+        # Check against excluded keys (exact match or substring)
+        if any(excluded_key in attr_name or attr_name == excluded_key for excluded_key in self.EXCLUDED_KEYS):
+            return True
+        
+        # Value type filtering
+        if not isinstance(attr_value, (str, int, float, bool)) or attr_value is None:
+            return True
+        
+        # Boolean values are not useful for correlation
+        if isinstance(attr_value, bool):
+            return True
+            
+        # String value filtering
+        if isinstance(attr_value, str):                
+            # Date/timestamp strings
+            if self.date_pattern.match(attr_value):
+                return True
+                
+            # Common non-useful values
+            if attr_value.lower() in ['unknown', 'none', 'null', 'n/a', 'true', 'false', '0', '1']:
+                return True
+                
+            # Very long strings that are likely unique (> 100 chars)
+            if len(attr_value) > 100:
+                return True
+        
+        # Numeric value filtering  
+        if isinstance(attr_value, (int, float)):
+            # Very common values
+            if attr_value in [0, 1]:
+                return True
+                
+            # Very large numbers (likely timestamps or unique IDs)
+            if abs(attr_value) > 1000000:
+                return True
+        
+        return False
+    
     def _create_correlation_relationships(self, value: Any, correlation_data: Dict[str, Any], result: ProviderResult):
         """
-        Create correlation relationships and add them to the provider result.
+        Create correlation relationships with enhanced deduplication and validation.
         """
         correlation_node_id = f"corr_{hash(str(value)) & 0x7FFFFFFF}"
         nodes = correlation_data['nodes']
         sources = correlation_data['sources']
+        
+        # Only create correlations if we have meaningful nodes (more than 1)
+        if len(nodes) < 2:
+            return
+            
+        # Limit correlation size to prevent overly large correlation objects
+        MAX_CORRELATION_SIZE = 50
+        if len(nodes) > MAX_CORRELATION_SIZE:
+            # Sample the nodes to keep correlation manageable
+            import random
+            sampled_nodes = random.sample(list(nodes), MAX_CORRELATION_SIZE)
+            nodes = set(sampled_nodes)
+            # Filter sources to match sampled nodes
+            sources = [s for s in sources if s['node_id'] in nodes]
 
         # Add the correlation node as an attribute to the result
         result.add_attribute(
             target_node=correlation_node_id,
             name="correlation_value",
             value=value,
-            attr_type=str(type(value)),
+            attr_type=str(type(value).__name__),
             provider=self.name,
             confidence=0.9,
             metadata={
                 'correlated_nodes': list(nodes),
                 'sources': sources,
+                'correlation_size': len(nodes),
+                'value_type': type(value).__name__
             }
         )
 
+        # Create relationships with source validation
+        created_relationships = set()
+        
         for source in sources:
             node_id = source['node_id']
             provider = source['provider']
             attribute = source['attribute']
+            
+            # Skip if we've already created this relationship
+            relationship_key = (node_id, correlation_node_id)
+            if relationship_key in created_relationships:
+                continue
+                
             relationship_label = f"corr_{provider}_{attribute}"
 
             # Add the relationship to the result
@@ -173,6 +250,9 @@ class CorrelationProvider(BaseProvider):
                 raw_data={
                     'correlation_value': value,
                     'original_attribute': attribute,
-                    'correlation_type': 'attribute_matching'
+                    'correlation_type': 'attribute_matching',
+                    'correlation_size': len(nodes)
                 }
-            )
+            )
+            
+            created_relationships.add(relationship_key)

providers/crtsh_provider.py

@@ -1,44 +1,22 @@
-# dnsrecon/providers/crtsh_provider.py
+# DNScope/providers/crtsh_provider.py
 
 import json
 import re
-import psycopg2
 from pathlib import Path
 from typing import List, Dict, Any, Set, Optional
 from urllib.parse import quote
 from datetime import datetime, timezone
 import requests
-from psycopg2 import pool
 
 from .base_provider import BaseProvider
 from core.provider_result import ProviderResult
 from utils.helpers import _is_valid_domain
 from core.logger import get_forensic_logger
 
-# --- Global Instance for PostgreSQL Connection Pool ---
-# This pool will be created once per worker process and is not part of the
-# CrtShProvider instance, thus avoiding pickling errors.
-db_pool = None
-try:
-    db_pool = psycopg2.pool.SimpleConnectionPool(
-        1, 5,
-        host='crt.sh',
-        port=5432,
-        user='guest',
-        dbname='certwatch',
-        sslmode='prefer',
-        connect_timeout=60
-    )
-    # Use a generic logger here as this is at the module level
-    get_forensic_logger().logger.info("crt.sh: Global PostgreSQL connection pool created successfully.")
-except Exception as e:
-    get_forensic_logger().logger.warning(f"crt.sh: Failed to create global DB connection pool: {e}. Will fall back to HTTP API.")
-
-
 class CrtShProvider(BaseProvider):
     """
     Provider for querying crt.sh certificate transparency database.
-    FIXED: Now properly creates domain and CA nodes instead of large entities.
+    FIXED: Improved caching logic and error handling to prevent infinite retry loops.
     Returns standardized ProviderResult objects with caching support.
     """
     
@@ -53,7 +31,7 @@ class CrtShProvider(BaseProvider):
         self.base_url = "https://crt.sh/"
         self._stop_event = None
         
-        # Initialize cache directory (separate from BaseProvider's HTTP cache)
+        # Initialize cache directory
         self.domain_cache_dir = Path('cache') / 'crtsh'
         self.domain_cache_dir.mkdir(parents=True, exist_ok=True)
         
@@ -87,43 +65,72 @@ class CrtShProvider(BaseProvider):
     
     def _get_cache_status(self, cache_file_path: Path) -> str:
         """
-        Check cache status for a domain.
+        FIXED: More robust cache status checking with better error handling.
         Returns: 'not_found', 'fresh', or 'stale'
         """
         if not cache_file_path.exists():
             return "not_found"
         
         try:
-            with open(cache_file_path, 'r') as f:
+            # Check if file is readable and not corrupted
+            if cache_file_path.stat().st_size == 0:
+                self.logger.logger.warning(f"Empty cache file: {cache_file_path}")
+                return "stale"
+                
+            with open(cache_file_path, 'r', encoding='utf-8') as f:
                 cache_data = json.load(f)
             
-            last_query_str = cache_data.get("last_upstream_query")
+            # Validate cache structure
-            if not last_query_str:
+            if not isinstance(cache_data, dict):
+                self.logger.logger.warning(f"Invalid cache structure: {cache_file_path}")
                 return "stale"
             
-            last_query = datetime.fromisoformat(last_query_str.replace('Z', '+00:00'))
+            last_query_str = cache_data.get("last_upstream_query")
-            hours_since_query = (datetime.now(timezone.utc) - last_query).total_seconds() / 3600
+            if not last_query_str or not isinstance(last_query_str, str):
+                self.logger.logger.warning(f"Missing or invalid last_upstream_query: {cache_file_path}")
+                return "stale"
             
+            try:
+                # More robust datetime parsing
+                if last_query_str.endswith('Z'):
+                    last_query = datetime.fromisoformat(last_query_str.replace('Z', '+00:00'))
+                elif '+' in last_query_str or last_query_str.endswith('UTC'):
+                    # Handle various timezone formats
+                    clean_time = last_query_str.replace('UTC', '').strip()
+                    if '+' in clean_time:
+                        clean_time = clean_time.split('+')[0]
+                    last_query = datetime.fromisoformat(clean_time).replace(tzinfo=timezone.utc)
+                else:
+                    last_query = datetime.fromisoformat(last_query_str).replace(tzinfo=timezone.utc)
+                    
+            except (ValueError, AttributeError) as e:
+                self.logger.logger.warning(f"Failed to parse timestamp in cache {cache_file_path}: {e}")
+                return "stale"
+            
+            hours_since_query = (datetime.now(timezone.utc) - last_query).total_seconds() / 3600
             cache_timeout = self.config.cache_timeout_hours
+            
             if hours_since_query < cache_timeout:
                 return "fresh"
             else:
                 return "stale"
                 
-        except (json.JSONDecodeError, ValueError, KeyError) as e:
+        except (json.JSONDecodeError, OSError, PermissionError) as e:
-            self.logger.logger.warning(f"Invalid cache file format for {cache_file_path}: {e}")
+            self.logger.logger.warning(f"Cache file error for {cache_file_path}: {e}")
+            # FIXED: Try to remove corrupted cache file
+            try:
+                cache_file_path.unlink()
+                self.logger.logger.info(f"Removed corrupted cache file: {cache_file_path}")
+            except Exception:
+                pass
+            return "not_found"
+        except Exception as e:
+            self.logger.logger.error(f"Unexpected error checking cache status for {cache_file_path}: {e}")
             return "stale"
 
     def query_domain(self, domain: str) -> ProviderResult:
         """
-        FIXED: Query crt.sh for certificates containing the domain.
+        FIXED: Simplified and more robust domain querying with better error handling.
-        Now properly creates domain and CA nodes instead of large entities.
-        
-        Args:
-            domain: Domain to investigate
-            
-        Returns:
-            ProviderResult containing discovered relationships and attributes
         """
         if not _is_valid_domain(domain):
             return ProviderResult()
@@ -132,124 +139,155 @@ class CrtShProvider(BaseProvider):
             return ProviderResult()
 
         cache_file = self._get_cache_file_path(domain)
-        cache_status = self._get_cache_status(cache_file)
-        
         result = ProviderResult()
 
         try:
-            if cache_status == "fresh":
+            cache_status = self._get_cache_status(cache_file)
-                result = self._load_from_cache(cache_file)
-                self.logger.logger.info(f"Using fresh cached crt.sh data for {domain}")
             
-            else:  # "stale" or "not_found"
+            if cache_status == "fresh":
-                # Query the API for the latest certificates
+                # Load from cache
-                new_raw_certs = self._query_crtsh(domain)
-                
-                if self._stop_event and self._stop_event.is_set():
-                    return ProviderResult()
-
-                # Combine with old data if cache is stale
-                if cache_status == "stale":
-                    old_raw_certs = self._load_raw_data_from_cache(cache_file)
-                    combined_certs = old_raw_certs + new_raw_certs
-                    
-                    # Deduplicate the combined list
-                    seen_ids = set()
-                    unique_certs = []
-                    for cert in combined_certs:
-                        cert_id = cert.get('id')
-                        if cert_id not in seen_ids:
-                            unique_certs.append(cert)
-                            seen_ids.add(cert_id)
-                    
-                    raw_certificates_to_process = unique_certs
-                    self.logger.logger.info(f"Refreshed and merged cache for {domain}. Total unique certs: {len(raw_certificates_to_process)}")
-                else:  # "not_found"
-                    raw_certificates_to_process = new_raw_certs
-                
-                # FIXED: Process certificates to create proper domain and CA nodes
-                result = self._process_certificates_to_result_fixed(domain, raw_certificates_to_process)
-                self.logger.logger.info(f"Created fresh result for {domain} ({result.get_relationship_count()} relationships)")
-
-                # Save the new result and the raw data to the cache
-                self._save_result_to_cache(cache_file, result, raw_certificates_to_process, domain)
-
-        except (requests.exceptions.RequestException, psycopg2.Error) as e:
-            self.logger.logger.error(f"Upstream query failed for {domain}: {e}")
-            if cache_status != "not_found":
                 result = self._load_from_cache(cache_file)
-                self.logger.logger.warning(f"Using stale cache for {domain} due to API failure.")
+                if result and (result.relationships or result.attributes):
-            else:
+                    self.logger.logger.debug(f"Using fresh cached crt.sh data for {domain}")
-                raise e  # Re-raise if there's no cache to fall back on
+                    return result
+                else:
+                    # Cache exists but is empty, treat as stale
+                    cache_status = "stale"
+            
+            # Need to query API (either no cache, stale cache, or empty cache)
+            self.logger.logger.debug(f"Querying crt.sh API for {domain} (cache status: {cache_status})")
+            new_raw_certs = self._query_crtsh_api(domain)
+            
+            if self._stop_event and self._stop_event.is_set():
+                return ProviderResult()
 
-        return result
+            # FIXED: Simplified processing - just process the new data
+            # Don't try to merge with stale cache as it can cause corruption
+            raw_certificates_to_process = new_raw_certs
+            
+            if cache_status == "stale":
+                self.logger.logger.info(f"Refreshed stale cache for {domain} with {len(raw_certificates_to_process)} certs")
+            else:
+                self.logger.logger.info(f"Created fresh cache for {domain} with {len(raw_certificates_to_process)} certs")
+            
+            result = self._process_certificates_to_result_fixed(domain, raw_certificates_to_process)
+            
+            # Save the result to cache
+            self._save_result_to_cache(cache_file, result, raw_certificates_to_process, domain)
+            
+            return result
+
+        except requests.exceptions.RequestException as e:
+            # FIXED: Don't re-raise network errors after long idle periods
+            # Instead return empty result and log the issue
+            self.logger.logger.warning(f"Network error querying crt.sh for {domain}: {e}")
+            
+            # Try to use stale cache if available
+            if cache_status == "stale":
+                try:
+                    stale_result = self._load_from_cache(cache_file)
+                    if stale_result and (stale_result.relationships or stale_result.attributes):
+                        self.logger.logger.info(f"Using stale cache for {domain} due to network error")
+                        return stale_result
+                except Exception as cache_error:
+                    self.logger.logger.warning(f"Failed to load stale cache for {domain}: {cache_error}")
+            
+            # Return empty result instead of raising - this prevents infinite retries
+            return ProviderResult()
+            
+        except Exception as e:
+            # FIXED: Handle any other exceptions gracefully
+            self.logger.logger.error(f"Unexpected error querying crt.sh for {domain}: {e}")
+            
+            # Try stale cache as fallback
+            try:
+                if cache_file.exists():
+                    fallback_result = self._load_from_cache(cache_file)
+                    if fallback_result and (fallback_result.relationships or fallback_result.attributes):
+                        self.logger.logger.info(f"Using cached data for {domain} due to processing error")
+                        return fallback_result
+            except Exception:
+                pass
+            
+            # Return empty result to prevent retries
+            return ProviderResult()
 
     def query_ip(self, ip: str) -> ProviderResult:
         """
-        Query crt.sh for certificates containing the IP address.
+        crt.sh does not support IP-based certificate queries effectively via its API.
-        Note: crt.sh doesn't typically index by IP, so this returns empty results.
-        
-        Args:
-            ip: IP address to investigate
-            
-        Returns:
-            Empty ProviderResult (crt.sh doesn't support IP-based certificate queries effectively)
         """
         return ProviderResult()
 
     def _load_from_cache(self, cache_file_path: Path) -> ProviderResult:
-        """Load processed crt.sh data from a cache file."""
+        """FIXED: More robust cache loading with better validation."""
         try:
-            with open(cache_file_path, 'r') as f:
+            if not cache_file_path.exists() or cache_file_path.stat().st_size == 0:
+                return ProviderResult()
+                
+            with open(cache_file_path, 'r', encoding='utf-8') as f:
                 cache_content = json.load(f)
             
+            if not isinstance(cache_content, dict):
+                self.logger.logger.warning(f"Invalid cache format in {cache_file_path}")
+                return ProviderResult()
+            
             result = ProviderResult()
             
-            # Reconstruct relationships
+            # Reconstruct relationships with validation
-            for rel_data in cache_content.get("relationships", []):
+            relationships = cache_content.get("relationships", [])
-                result.add_relationship(
+            if isinstance(relationships, list):
-                    source_node=rel_data["source_node"],
+                for rel_data in relationships:
-                    target_node=rel_data["target_node"],
+                    if not isinstance(rel_data, dict):
-                    relationship_type=rel_data["relationship_type"],
+                        continue
-                    provider=rel_data["provider"],
+                    try:
-                    confidence=rel_data["confidence"],
+                        result.add_relationship(
-                    raw_data=rel_data.get("raw_data", {})
+                            source_node=rel_data.get("source_node", ""),
-                )
+                            target_node=rel_data.get("target_node", ""),
+                            relationship_type=rel_data.get("relationship_type", ""),
+                            provider=rel_data.get("provider", self.name),
+                            confidence=float(rel_data.get("confidence", 0.8)),
+                            raw_data=rel_data.get("raw_data", {})
+                        )
+                    except (ValueError, TypeError) as e:
+                        self.logger.logger.warning(f"Skipping invalid relationship in cache: {e}")
+                        continue
             
-            # Reconstruct attributes
+            # Reconstruct attributes with validation
-            for attr_data in cache_content.get("attributes", []):
+            attributes = cache_content.get("attributes", [])
-                result.add_attribute(
+            if isinstance(attributes, list):
-                    target_node=attr_data["target_node"],
+                for attr_data in attributes:
-                    name=attr_data["name"],
+                    if not isinstance(attr_data, dict):
-                    value=attr_data["value"],
+                        continue
-                    attr_type=attr_data["type"],
+                    try:
-                    provider=attr_data["provider"],
+                        result.add_attribute(
-                    confidence=attr_data["confidence"],
+                            target_node=attr_data.get("target_node", ""),
-                    metadata=attr_data.get("metadata", {})
+                            name=attr_data.get("name", ""),
-                )
+                            value=attr_data.get("value"),
+                            attr_type=attr_data.get("type", "unknown"),
+                            provider=attr_data.get("provider", self.name),
+                            confidence=float(attr_data.get("confidence", 0.9)),
+                            metadata=attr_data.get("metadata", {})
+                        )
+                    except (ValueError, TypeError) as e:
+                        self.logger.logger.warning(f"Skipping invalid attribute in cache: {e}")
+                        continue
             
             return result
             
-        except (json.JSONDecodeError, FileNotFoundError, KeyError) as e:
+        except (json.JSONDecodeError, OSError, PermissionError) as e:
-            self.logger.logger.error(f"Failed to load cached certificates from {cache_file_path}: {e}")
+            self.logger.logger.warning(f"Failed to load cache from {cache_file_path}: {e}")
+            return ProviderResult()
+        except Exception as e:
+            self.logger.logger.error(f"Unexpected error loading cache from {cache_file_path}: {e}")
             return ProviderResult()
 
-    def _load_raw_data_from_cache(self, cache_file_path: Path) -> List[Dict[str, Any]]:
-        """Load only the raw certificate data from a cache file."""
-        try:
-            with open(cache_file_path, 'r') as f:
-                cache_content = json.load(f)
-            return cache_content.get("raw_certificates", [])
-        except (json.JSONDecodeError, FileNotFoundError):
-            return []
-
     def _save_result_to_cache(self, cache_file_path: Path, result: ProviderResult, raw_certificates: List[Dict[str, Any]], domain: str) -> None:
-        """Save processed crt.sh result and raw data to a cache file."""
+        """FIXED: More robust cache saving with atomic writes."""
         try:
             cache_data = {
                 "domain": domain,
                 "last_upstream_query": datetime.now(timezone.utc).isoformat(),
-                "raw_certificates": raw_certificates, # Store the raw data for deduplication
+                "raw_certificates": raw_certificates,
                 "relationships": [
                     {
                         "source_node": rel.source_node,
@@ -272,87 +310,68 @@ class CrtShProvider(BaseProvider):
                     } for attr in result.attributes
                 ]
             }
+            
             cache_file_path.parent.mkdir(parents=True, exist_ok=True)
-            with open(cache_file_path, 'w') as f:
+            
-                json.dump(cache_data, f, separators=(',', ':'), default=str)
+            # FIXED: Atomic write using temporary file
+            temp_file = cache_file_path.with_suffix('.tmp')
+            try:
+                with open(temp_file, 'w', encoding='utf-8') as f:
+                    json.dump(cache_data, f, separators=(',', ':'), default=str, ensure_ascii=False)
+                
+                # Atomic rename
+                temp_file.replace(cache_file_path)
+                self.logger.logger.debug(f"Saved cache for {domain} ({len(result.relationships)} relationships)")
+                
+            except Exception as e:
+                # Clean up temp file on error
+                if temp_file.exists():
+                    try:
+                        temp_file.unlink()
+                    except Exception:
+                        pass
+                raise e
+                
         except Exception as e:
             self.logger.logger.warning(f"Failed to save cache file for {domain}: {e}")
-            
+    
-    def _query_crtsh(self, domain: str) -> List[Dict[str, Any]]:
-        """Query crt.sh, trying the database first and falling back to the API."""
-        global db_pool
-        if db_pool:
-            try:
-                self.logger.logger.info(f"crt.sh: Attempting DB query for {domain}")
-                return self._query_crtsh_db(domain)
-            except psycopg2.Error as e:
-                self.logger.logger.warning(f"crt.sh: DB query failed for {domain}: {e}. Falling back to HTTP API.")
-                return self._query_crtsh_api(domain)
-        else:
-            self.logger.logger.info(f"crt.sh: No DB connection pool. Using HTTP API for {domain}")
-            return self._query_crtsh_api(domain)
-
-    def _query_crtsh_db(self, domain: str) -> List[Dict[str, Any]]:
-        """Query crt.sh database for raw certificate data."""
-        global db_pool
-        conn = db_pool.getconn()
-        try:
-            with conn.cursor() as cursor:
-                query = """
-                SELECT
-                    c.id,
-                    x509_serialnumber(c.certificate) as serial_number,
-                    x509_notbefore(c.certificate) as not_before,
-                    x509_notafter(c.certificate) as not_after,
-                    c.issuer_ca_id,
-                    ca.name as issuer_name,
-                    x509_commonname(c.certificate) as common_name,
-                    identities(c.certificate)::text as name_value
-                FROM certificate c
-                LEFT JOIN ca ON c.issuer_ca_id = ca.id
-                WHERE identities(c.certificate) @@ plainto_tsquery(%s)
-                ORDER BY c.id DESC
-                LIMIT 5000;
-                """
-                cursor.execute(query, (domain,))
-                
-                results = []
-                columns = [desc[0] for desc in cursor.description]
-                for row in cursor.fetchall():
-                    row_dict = dict(zip(columns, row))
-                    if row_dict.get('not_before'):
-                        row_dict['not_before'] = row_dict['not_before'].isoformat()
-                    if row_dict.get('not_after'):
-                        row_dict['not_after'] = row_dict['not_after'].isoformat()
-                    results.append(row_dict)
-                self.logger.logger.info(f"crt.sh: DB query for {domain} returned {len(results)} records.")
-                return results
-        finally:
-            db_pool.putconn(conn)
-
     def _query_crtsh_api(self, domain: str) -> List[Dict[str, Any]]:
-        """Query crt.sh API for raw certificate data."""
+        """FIXED: More robust API querying with better error handling."""
         url = f"{self.base_url}?q={quote(domain)}&output=json"
-        response = self.make_request(url, target_indicator=domain)
-        
-        if not response or response.status_code != 200:
-            raise requests.exceptions.RequestException(f"crt.sh API returned status {response.status_code if response else 'None'}")
         
         try:
-            certificates = response.json()
+            response = self.make_request(url, target_indicator=domain)
-        except json.JSONDecodeError:
+            
-            self.logger.logger.error(f"crt.sh returned invalid JSON for {domain}")
+            if not response:
-            return []
+                self.logger.logger.warning(f"No response from crt.sh for {domain}")
+                return []
+            
+            if response.status_code != 200:
+                self.logger.logger.warning(f"crt.sh returned status {response.status_code} for {domain}")
+                return []
+            
+            # FIXED: Better JSON parsing with error handling
+            try:
+                certificates = response.json()
+            except json.JSONDecodeError as e:
+                self.logger.logger.error(f"crt.sh returned invalid JSON for {domain}: {e}")
+                return []
 
-        if not certificates:
+            if not certificates or not isinstance(certificates, list):
-            return []
+                self.logger.logger.debug(f"crt.sh returned no certificates for {domain}")
-        
+                return []
-        return certificates
+            
+            self.logger.logger.debug(f"crt.sh returned {len(certificates)} certificates for {domain}")
+            return certificates
+            
+        except Exception as e:
+            self.logger.logger.error(f"Error querying crt.sh API for {domain}: {e}")
+            raise e
 
     def _process_certificates_to_result_fixed(self, query_domain: str, certificates: List[Dict[str, Any]]) -> ProviderResult:
         """
-        FIXED: Process certificates to create proper domain and CA nodes.
+        Process certificates to create proper domain and CA nodes.
-        Now creates individual domain nodes instead of large entities.
+        FIXED: Better error handling and progress tracking.
         """
         result = ProviderResult()
 
@@ -360,6 +379,11 @@ class CrtShProvider(BaseProvider):
             self.logger.logger.info(f"CrtSh processing cancelled before processing for domain: {query_domain}")
             return result
         
+        if not certificates:
+            self.logger.logger.debug(f"No certificates to process for {query_domain}")
+            return result
+        
+        # Check for incomplete data warning
         incompleteness_warning = self._check_for_incomplete_data(query_domain, certificates)
         if incompleteness_warning:
             result.add_attribute(
@@ -373,55 +397,68 @@ class CrtShProvider(BaseProvider):
 
         all_discovered_domains = set()
         processed_issuers = set()
+        processed_certs = 0
 
         for i, cert_data in enumerate(certificates):
-            if i % 10 == 0 and self._stop_event and self._stop_event.is_set():
+            # FIXED: More frequent stop checks and progress logging
-                self.logger.logger.info(f"CrtSh processing cancelled at certificate {i} for domain: {query_domain}")
+            if i % 5 == 0:
-                break
+                if self._stop_event and self._stop_event.is_set():
+                    self.logger.logger.info(f"CrtSh processing cancelled at certificate {i}/{len(certificates)} for domain: {query_domain}")
+                    break
+                
+                if i > 0 and i % 100 == 0:
+                    self.logger.logger.debug(f"Processed {i}/{len(certificates)} certificates for {query_domain}")
 
-            # Extract all domains from this certificate
+            try:
-            cert_domains = self._extract_domains_from_certificate(cert_data)
+                # Extract all domains from this certificate
-            all_discovered_domains.update(cert_domains)
+                cert_domains = self._extract_domains_from_certificate(cert_data)
+                if cert_domains:
+                    all_discovered_domains.update(cert_domains)
 
-            # FIXED: Create CA nodes for certificate issuers (not as domain metadata)
+                # Create CA nodes for certificate issuers
-            issuer_name = self._parse_issuer_organization(cert_data.get('issuer_name', ''))
+                issuer_name = self._parse_issuer_organization(cert_data.get('issuer_name', ''))
-            if issuer_name and issuer_name not in processed_issuers:
+                if issuer_name and issuer_name not in processed_issuers:
-                # Create relationship from query domain to CA
+                    result.add_relationship(
-                result.add_relationship(
+                        source_node=query_domain,
-                    source_node=query_domain,
+                        target_node=issuer_name,
-                    target_node=issuer_name,
+                        relationship_type='crtsh_cert_issuer',
-                    relationship_type='crtsh_cert_issuer',
+                        provider=self.name,
-                    provider=self.name,
+                        confidence=0.95,
-                    confidence=0.95,
+                        raw_data={'issuer_dn': cert_data.get('issuer_name', '')}
-                    raw_data={'issuer_dn': cert_data.get('issuer_name', '')}
+                    )
-                )
+                    processed_issuers.add(issuer_name)
-                processed_issuers.add(issuer_name)
 
-            # Add certificate metadata to each domain in this certificate
+                # Add certificate metadata to each domain in this certificate
-            cert_metadata = self._extract_certificate_metadata(cert_data)
+                cert_metadata = self._extract_certificate_metadata(cert_data)
-            for cert_domain in cert_domains:
+                for cert_domain in cert_domains:
-                if not _is_valid_domain(cert_domain):
+                    if not _is_valid_domain(cert_domain):
-                    continue
+                        continue
-
-                # Add certificate attributes to the domain
-                for key, value in cert_metadata.items():
-                    if value is not None:
-                        result.add_attribute(
-                            target_node=cert_domain,
-                            name=f"cert_{key}",
-                            value=value,
-                            attr_type='certificate_data',
-                            provider=self.name,
-                            confidence=0.9,
-                            metadata={'certificate_id': cert_data.get('id')}
-                        )
 
+                    for key, value in cert_metadata.items():
+                        if value is not None:
+                            result.add_attribute(
+                                target_node=cert_domain,
+                                name=f"cert_{key}",
+                                value=value,
+                                attr_type='certificate_data',
+                                provider=self.name,
+                                confidence=0.9,
+                                metadata={'certificate_id': cert_data.get('id')}
+                            )
+                
+                processed_certs += 1
+                
+            except Exception as e:
+                self.logger.logger.warning(f"Error processing certificate {i} for {query_domain}: {e}")
+                continue
+        
+        # Check for stop event before creating final relationships
         if self._stop_event and self._stop_event.is_set():
             self.logger.logger.info(f"CrtSh query cancelled before relationship creation for domain: {query_domain}")
             return result
 
-        # FIXED: Create selective relationships to avoid large entities
+        # Create selective relationships to avoid large entities
-        # Only create relationships to domains that are closely related
+        relationships_created = 0
         for discovered_domain in all_discovered_domains:
             if discovered_domain == query_domain:
                 continue
@@ -429,8 +466,6 @@ class CrtShProvider(BaseProvider):
             if not _is_valid_domain(discovered_domain):
                 continue
 
-            # FIXED: Only create relationships for domains that share a meaningful connection
-            # This prevents creating too many relationships that trigger large entity creation
             if self._should_create_relationship(query_domain, discovered_domain):
                 confidence = self._calculate_domain_relationship_confidence(
                     query_domain, discovered_domain, [], all_discovered_domains
@@ -453,24 +488,22 @@ class CrtShProvider(BaseProvider):
                     raw_data={'relationship_type': 'certificate_discovery'},
                     discovery_method="certificate_transparency_analysis"
                 )
+                relationships_created += 1
 
-        self.logger.logger.info(f"CrtSh processing completed for {query_domain}: {len(all_discovered_domains)} domains, {result.get_relationship_count()} relationships")
+        self.logger.logger.info(f"CrtSh processing completed for {query_domain}: processed {processed_certs}/{len(certificates)} certificates, {len(all_discovered_domains)} domains, {relationships_created} relationships")
         return result
 
+    # [Rest of the methods remain the same as in the original file]
     def _should_create_relationship(self, source_domain: str, target_domain: str) -> bool:
         """
-        FIXED: Determine if a relationship should be created between two domains.
+        Determine if a relationship should be created between two domains.
-        This helps avoid creating too many relationships that trigger large entity creation.
         """
-        # Always create relationships for subdomains
         if target_domain.endswith(f'.{source_domain}') or source_domain.endswith(f'.{target_domain}'):
             return True
         
-        # Create relationships for domains that share a common parent (up to 2 levels)
         source_parts = source_domain.split('.')
         target_parts = target_domain.split('.')
         
-        # Check if they share the same root domain (last 2 parts)
         if len(source_parts) >= 2 and len(target_parts) >= 2:
             source_root = '.'.join(source_parts[-2:])
             target_root = '.'.join(target_parts[-2:])
@@ -504,7 +537,6 @@ class CrtShProvider(BaseProvider):
                 metadata['is_currently_valid'] = self._is_cert_valid(cert_data)
                 metadata['expires_soon'] = (not_after - datetime.now(timezone.utc)).days <= 30
                 
-                # Keep raw date format or convert to standard format
                 metadata['not_before'] = not_before.isoformat()
                 metadata['not_after'] = not_after.isoformat()
                 
@@ -586,14 +618,12 @@ class CrtShProvider(BaseProvider):
         """Extract all domains from certificate data."""
         domains = set()
         
-        # Extract from common name
         common_name = cert_data.get('common_name', '')
         if common_name:
             cleaned_cn = self._clean_domain_name(common_name)
             if cleaned_cn:
                 domains.update(cleaned_cn)
         
-        # Extract from name_value field (contains SANs)
         name_value = cert_data.get('name_value', '')
         if name_value:
             for line in name_value.split('\n'):
@@ -640,7 +670,6 @@ class CrtShProvider(BaseProvider):
         """Calculate confidence score for domain relationship based on various factors."""
         base_confidence = 0.9
         
-        # Adjust confidence based on domain relationship context
         relationship_context = self._determine_relationship_context(domain2, domain1)
         
         if relationship_context == 'exact_match':
@@ -672,12 +701,10 @@ class CrtShProvider(BaseProvider):
         """
         cert_count = len(certificates)
 
-        # Heuristic 1: Check if the number of certs hits a known hard limit.
         if cert_count >= 10000:
             return f"Result likely truncated; received {cert_count} certificates, which may be the maximum limit."
 
-        # Heuristic 2: Check if all returned certificates are old.
+        if cert_count > 1000: 
-        if cert_count > 1000: # Only apply this for a reasonable number of certs
             latest_expiry = None
             for cert in certificates:
                 try:

providers/dns_provider.py

@@ -1,4 +1,4 @@
-# dnsrecon/providers/dns_provider.py
+# DNScope/providers/dns_provider.py
 
 from dns import resolver, reversename
 from typing import Dict

providers/shodan_provider.py

@@ -1,4 +1,4 @@
-# dnsrecon/providers/shodan_provider.py
+# DNScope/providers/shodan_provider.py
 
 import json
 from pathlib import Path

static/css/main.css

@@ -1,4 +1,4 @@
-/* DNSRecon - Optimized Compact Theme */
+/* DNScope - Optimized Compact Theme */
 
 /* Reset and Base */
 * {
@@ -474,6 +474,7 @@ input[type="text"]:focus, select:focus {
     flex-wrap: wrap;
     gap: 0.75rem;
     align-items: center;
+    max-height: 3rem;
 }
 
 .legend-item {
@@ -1323,4 +1324,16 @@ input[type="password"]:focus {
     .provider-list {
         grid-template-columns: 1fr;
     }
+}
+
+.manual-refresh-btn {
+    background: rgba(92, 76, 44, 0.9) !important; /* Orange/amber background */
+    border: 1px solid #7a6a3a !important;
+    color: #ffcc00 !important; /* Bright yellow text */
+}
+
+.manual-refresh-btn:hover {
+    border-color: #ffcc00 !important;
+    color: #fff !important;
+    background: rgba(112, 96, 54, 0.9) !important;
 }

static/js/graph.js

@@ -1,8 +1,8 @@
-// dnsrecon-reduced/static/js/graph.js
+// DNScope-reduced/static/js/graph.js
 /**
- * Graph visualization module for DNSRecon
+ * Graph visualization module for DNScope
  * Handles network graph rendering using vis.js with proper large entity node hiding
- * UPDATED: Now compatible with a strictly flat, unified data model for attributes.
+ * UPDATED: Added manual refresh button for polling optimization when graph becomes large
  */
 const contextMenuCSS = `
 .graph-context-menu {
@@ -71,6 +71,10 @@ class GraphManager {
         // Track large entity members for proper hiding
         this.largeEntityMembers = new Set();
         this.isScanning = false;
+        
+        // Manual refresh button for polling optimization
+        this.manualRefreshButton = null;
+        this.manualRefreshHandler = null; // Store the handler
 
         this.options = {
             nodes: {
@@ -254,6 +258,7 @@ class GraphManager {
 
     /**
      * Add interactive graph controls
+     * UPDATED: Added manual refresh button for polling optimization
      */
     addGraphControls() {
         const controlsContainer = document.createElement('div');
@@ -264,6 +269,9 @@ class GraphManager {
             <button class="graph-control-btn" id="graph-cluster" title="Cluster Nodes">[CLUSTER]</button>
             <button class="graph-control-btn" id="graph-unhide" title="Unhide All">[UNHIDE]</button>
             <button class="graph-control-btn" id="graph-revert" title="Revert Last Action">[REVERT]</button>
+            <button class="graph-control-btn manual-refresh-btn" id="graph-manual-refresh" 
+                    title="Manual Refresh - Auto-refresh disabled due to large graph" 
+                    style="display: none;">[REFRESH]</button>
         `;
 
         this.container.appendChild(controlsContainer);
@@ -274,6 +282,35 @@ class GraphManager {
         document.getElementById('graph-cluster').addEventListener('click', () => this.toggleClustering());
         document.getElementById('graph-unhide').addEventListener('click', () => this.unhideAll());
         document.getElementById('graph-revert').addEventListener('click', () => this.revertLastAction());
+        
+        // Manual refresh button - handler will be set by main app
+        this.manualRefreshButton = document.getElementById('graph-manual-refresh');
+        // If a handler was set before the button existed, attach it now
+        if (this.manualRefreshButton && this.manualRefreshHandler) {
+            this.manualRefreshButton.addEventListener('click', this.manualRefreshHandler);
+        }
+    }
+    
+    /**
+     * Set the manual refresh button click handler
+     * @param {Function} handler - Function to call when manual refresh is clicked
+     */
+    setManualRefreshHandler(handler) {
+        this.manualRefreshHandler = handler;
+        // If the button already exists, attach the handler
+        if (this.manualRefreshButton && typeof handler === 'function') {
+            this.manualRefreshButton.addEventListener('click', handler);
+        }
+    }
+    
+    /**
+     * Show or hide the manual refresh button
+     * @param {boolean} show - Whether to show the button
+     */
+    showManualRefreshButton(show) {
+        if (this.manualRefreshButton) {
+            this.manualRefreshButton.style.display = show ? 'inline-block' : 'none';
+        }
     }
     
     addFilterPanel() {
@@ -353,9 +390,6 @@ class GraphManager {
         });
     }
 
-    /**
-     * @param {Object} graphData - Graph data from backend
-     */
     updateGraph(graphData) {
         if (!graphData || !graphData.nodes || !graphData.edges) {
             console.warn('Invalid graph data received');
@@ -382,16 +416,18 @@ class GraphManager {
 
             const nodeMap = new Map(graphData.nodes.map(node => [node.id, node]));
 
-            // Filter out hidden nodes before processing for rendering
+            // FIXED: Process all nodes first, then apply hiding logic correctly
-            const filteredNodes = graphData.nodes.filter(node => 
-                !(node.metadata && node.metadata.large_entity_id)
-            );
-
             const processedNodes = graphData.nodes.map(node => {
                 const processed = this.processNode(node);
+                
+                // FIXED: Only hide if node is still a large entity member
                 if (node.metadata && node.metadata.large_entity_id) {
                     processed.hidden = true;
+                } else {
+                    // FIXED: Ensure extracted nodes are visible
+                    processed.hidden = false;
                 }
+                
                 return processed;
             });
             
@@ -401,6 +437,7 @@ class GraphManager {
                 let fromId = edge.from;
                 let toId = edge.to;
 
+                // FIXED: Only re-route if nodes are STILL in large entities
                 if (fromNode && fromNode.metadata && fromNode.metadata.large_entity_id) {
                     fromId = fromNode.metadata.large_entity_id;
                 }
@@ -423,6 +460,7 @@ class GraphManager {
             const newNodes = processedNodes.filter(node => !existingNodeIds.includes(node.id));
             const newEdges = processedEdges.filter(edge => !existingEdgeIds.includes(edge.id));
 
+            // FIXED: Update all nodes to ensure extracted nodes become visible
             this.nodes.update(processedNodes);
             this.edges.update(processedEdges);
             
@@ -606,7 +644,7 @@ class GraphManager {
     formatEdgeLabel(relationshipType, confidence) {
         if (!relationshipType) return '';
 
-        const confidenceText = confidence >= 0.8 ? '●' : confidence >= 0.6 ? '◐' : '○';
+        const confidenceText = confidence >= 0.8 ? '●' : confidence >= 0.6 ? '●' : '○';
         return `${relationshipType} ${confidenceText}`;
     }
     
@@ -1416,7 +1454,7 @@ class GraphManager {
 
         menuItems += `
                 <li data-action="hide" data-node-id="${nodeId}">
-                    <span class="menu-icon">👁️‍🗨️</span>
+                    <span class="menu-icon">👻</span>
                     <span>Hide Node</span>
                 </li>
                 <li data-action="delete" data-node-id="${nodeId}">

static/js/main.js

@@ -1,15 +1,16 @@
 /**
- * Main application logic for DNSRecon web interface
+ * Main application logic for DNScope web interface
  * Handles UI interactions, API communication, and data flow
  * UPDATED: Now compatible with a strictly flat, unified data model for attributes.
  */
 
-class DNSReconApp {
+class DNScopeApp {
     constructor() {
-        console.log('DNSReconApp constructor called');
+        console.log('DNScopeApp constructor called');
         this.graphManager = null;
         this.scanStatus = 'idle';
-        this.pollInterval = null;
+        this.statusPollInterval = null;      // Separate status polling
+        this.graphPollInterval = null;       // Separate graph polling
         this.currentSessionId = null;
         
         this.elements = {};
@@ -17,6 +18,10 @@ class DNSReconApp {
         this.isScanning = false;
         this.lastGraphUpdate = null;
         
+        // Graph polling optimization
+        this.graphPollingNodeThreshold = 500;  // Default, will be loaded from config
+        this.graphPollingEnabled = true;
+        
         this.init();
     }
     
@@ -24,7 +29,7 @@ class DNSReconApp {
      * Initialize the application
      */
     init() {
-        console.log('DNSReconApp init called');
+        console.log('DNScopeApp init called');
         document.addEventListener('DOMContentLoaded', () => {
             console.log('DOM loaded, initializing application...');
             try {
@@ -35,16 +40,32 @@ class DNSReconApp {
                 this.loadProviders();
                 this.initializeEnhancedModals();
                 this.addCheckboxStyling();
+                this.loadConfig();  // Load configuration including threshold
                 
                 this.updateGraph();
                 
-                console.log('DNSRecon application initialized successfully');
+                console.log('DNScope application initialized successfully');
             } catch (error) {
-                console.error('Failed to initialize DNSRecon application:', error);
+                console.error('Failed to initialize DNScope application:', error);
                 this.showError(`Initialization failed: ${error.message}`);
             }
         });
     }
+
+    /**
+     * Load configuration from backend
+     */
+    async loadConfig() {
+        try {
+            const response = await this.apiCall('/api/config');
+            if (response.success) {
+                this.graphPollingNodeThreshold = response.config.graph_polling_node_threshold;
+                console.log(`Graph polling threshold set to: ${this.graphPollingNodeThreshold} nodes`);
+            }
+        } catch (error) {
+            console.warn('Failed to load config, using defaults:', error);
+        }
+    }
         
     /**
      * Initialize DOM element references
@@ -263,18 +284,53 @@ class DNSReconApp {
     }
     
     /**
-     * Initialize graph visualization
+     * Initialize graph visualization with manual refresh button
      */
     initializeGraph() {
         try {
             console.log('Initializing graph manager...');
             this.graphManager = new GraphManager('network-graph');
+            
+            // Set up manual refresh handler
+            this.graphManager.setManualRefreshHandler(() => {
+                console.log('Manual graph refresh requested');
+                this.updateGraph();
+            });
+            
             console.log('Graph manager initialized successfully');
         } catch (error) {
             console.error('Failed to initialize graph manager:', error);
             this.showError('Failed to initialize graph visualization');
         }
     }
+
+    /**
+     * Check if graph polling should be enabled based on node count
+     */
+    shouldEnableGraphPolling() {
+        if (!this.graphManager || !this.graphManager.nodes) {
+            return true;
+        }
+        
+        const nodeCount = this.graphManager.nodes.length;
+        return nodeCount <= this.graphPollingNodeThreshold;
+    }
+    
+    /**
+     * Update manual refresh button visibility and state.
+     * The button will be visible whenever auto-polling is disabled,
+     * and enabled only when a scan is in progress.
+     */
+    updateManualRefreshButton() {
+        if (!this.graphManager || !this.graphManager.manualRefreshButton) return;
+
+        const shouldShow = !this.graphPollingEnabled;
+        this.graphManager.showManualRefreshButton(shouldShow);
+
+        if (shouldShow) {
+            this.graphManager.manualRefreshButton.disabled = false;
+        }
+    }
     
     /**
      * Start scan with error handling
@@ -324,18 +380,21 @@ class DNSReconApp {
 
                 if (clearGraph) {
                     this.graphManager.clear();
+                    this.graphPollingEnabled = true; // Reset polling when starting fresh
                 }
                 
                 console.log(`Scan started for ${target} with depth ${maxDepth}`);
                 
-                // Start polling immediately with faster interval for responsiveness
+                // Start optimized polling
-                this.startPolling(1000);
+                this.startOptimizedPolling();
                 
                 // Force an immediate status update
                 console.log('Forcing immediate status update...');
                 setTimeout(() => {
                     this.updateStatus();
-                    this.updateGraph();
+                    if (this.graphPollingEnabled) {
+                        this.updateGraph();
+                    }
                 }, 100);
                 
             } else {
@@ -348,6 +407,35 @@ class DNSReconApp {
             this.setUIState('idle');
         }
     }
+
+    /**
+     * Start optimized polling with separate status and graph intervals
+     */
+    startOptimizedPolling() {
+        console.log('=== STARTING OPTIMIZED POLLING ===');
+        
+        this.stopPolling(); // Stop any existing polling
+        
+        // Always poll status for progress bar
+        this.statusPollInterval = setInterval(() => {
+            this.updateStatus();
+            this.loadProviders();
+        }, 2000);
+        
+        // Only poll graph if enabled
+        if (this.graphPollingEnabled) {
+            this.graphPollInterval = setInterval(() => {
+                this.updateGraph();
+            }, 2000);
+            console.log('Graph polling started');
+        } else {
+            console.log('Graph polling disabled due to node count');
+        }
+        
+        this.updateManualRefreshButton();
+        console.log(`Optimized polling started - Status: enabled, Graph: ${this.graphPollingEnabled ? 'enabled' : 'disabled'}`);
+    }
+
     /**
      * Scan stop with immediate UI feedback
      */
@@ -374,15 +462,8 @@ class DNSReconApp {
                     this.updateStatus();
                 }, 100);
                 
-                // Continue polling for a bit to catch the status change
+                // Continue status polling for a bit to catch the status change
-                this.startPolling(500); // Fast polling to catch status change
+                // No need to resume graph polling
-                
-                // Stop fast polling after 10 seconds
-                setTimeout(() => {
-                    if (this.scanStatus === 'stopped' || this.scanStatus === 'idle') {
-                        this.stopPolling();
-                    }
-                }, 10000);
                 
             } else {
                 throw new Error(response.error || 'Failed to stop scan');
@@ -458,7 +539,7 @@ class DNSReconApp {
             
             // Get the filename from headers or create one
             const contentDisposition = response.headers.get('content-disposition');
-            let filename = 'dnsrecon_export.json';
+            let filename = 'DNScope_export.json';
             if (contentDisposition) {
                 const filenameMatch = contentDisposition.match(/filename[^;=\n]*=((['"]).*?\2|[^;\n]*)/);
                 if (filenameMatch) {
@@ -573,12 +654,20 @@ class DNSReconApp {
      */
     stopPolling() {
         console.log('=== STOPPING POLLING ===');
-        if (this.pollInterval) {
+        
-            clearInterval(this.pollInterval);
+        if (this.statusPollInterval) {
-            this.pollInterval = null;
+            clearInterval(this.statusPollInterval);
+            this.statusPollInterval = null;
         }
+        
+        if (this.graphPollInterval) {
+            clearInterval(this.graphPollInterval);
+            this.graphPollInterval = null;
+        }
+        
+        this.updateManualRefreshButton();
     }
-    
+
     /**
      * Status update with better error handling
      */
@@ -611,7 +700,7 @@ class DNSReconApp {
     }
     
     /**
-     * Update graph from server
+     * Update graph from server with polling optimization
      */
     async updateGraph() {
         try {
@@ -626,11 +715,29 @@ class DNSReconApp {
                 console.log('- Nodes:', graphData.nodes ? graphData.nodes.length : 0);
                 console.log('- Edges:', graphData.edges ? graphData.edges.length : 0);
                 
-                // FIXED: Always update graph, even if empty - let GraphManager handle placeholder
+                // Always update graph, even if empty - let GraphManager handle placeholder
                 if (this.graphManager) {
                     this.graphManager.updateGraph(graphData);
                     this.lastGraphUpdate = Date.now();
                     
+                    // Check if we should disable graph polling after this update
+                    const nodeCount = graphData.nodes ? graphData.nodes.length : 0;
+                    const shouldEnablePolling = nodeCount <= this.graphPollingNodeThreshold;
+                    
+                    if (this.graphPollingEnabled && !shouldEnablePolling) {
+                        console.log(`Graph polling disabled: ${nodeCount} nodes exceeds threshold of ${this.graphPollingNodeThreshold}`);
+                        this.graphPollingEnabled = false;
+                        this.showWarning(`Graph auto-refresh disabled: ${nodeCount} nodes exceed threshold of ${this.graphPollingNodeThreshold}. Use manual refresh button.`);
+                        
+                        // Stop graph polling but keep status polling
+                        if (this.graphPollInterval) {
+                            clearInterval(this.graphPollInterval);
+                            this.graphPollInterval = null;
+                        }
+                        
+                        this.updateManualRefreshButton();
+                    }
+                    
                     // Update relationship count in status
                     const edgeCount = graphData.edges ? graphData.edges.length : 0;
                     if (this.elements.relationshipsDisplay) {
@@ -639,7 +746,7 @@ class DNSReconApp {
                 }
             } else {
                 console.error('Graph update failed:', response);
-                // FIXED: Show placeholder when graph update fails
+                // Show placeholder when graph update fails
                 if (this.graphManager && this.graphManager.container) {
                     const placeholder = this.graphManager.container.querySelector('.graph-placeholder');
                     if (placeholder) {
@@ -650,7 +757,7 @@ class DNSReconApp {
             
         } catch (error) {
             console.error('Failed to update graph:', error);
-            // FIXED: Show placeholder on error
+            // Show placeholder on error
             if (this.graphManager && this.graphManager.container) {
                 const placeholder = this.graphManager.container.querySelector('.graph-placeholder');
                 if (placeholder) {
@@ -659,7 +766,6 @@ class DNSReconApp {
             }
         }
     }
-
     
     /**
      * Update status display elements
@@ -737,8 +843,6 @@ class DNSReconApp {
             case 'running':
                 this.setUIState('scanning', task_queue_size);
                 this.showSuccess('Scan is running');
-                // Increase polling frequency for active scans
-                this.startPolling(1000); // Poll every 1 second for running scans
                 this.updateConnectionStatus('active');
                 break;
                 
@@ -748,8 +852,9 @@ class DNSReconApp {
                 this.showSuccess('Scan completed successfully');
                 this.updateConnectionStatus('completed');
                 this.loadProviders();
-                // Force a final graph update
+                
-                console.log('Scan completed - forcing final graph update');
+                // Do final graph update when scan completes
+                console.log('Scan completed - performing final graph update');
                 setTimeout(() => this.updateGraph(), 100);
                 break;
                 
@@ -820,6 +925,7 @@ class DNSReconApp {
 
         switch (state) {
             case 'scanning':
+            case 'running':
                 this.isScanning = true;
                 if (this.graphManager) {
                     this.graphManager.isScanning = true;
@@ -852,12 +958,12 @@ class DNSReconApp {
                     this.graphManager.isScanning = false;
                 }
                 if (this.elements.startScan) {
-                    this.elements.startScan.disabled = !isQueueEmpty;
+                    this.elements.startScan.disabled = false;
                     this.elements.startScan.classList.remove('loading');
                     this.elements.startScan.innerHTML = '<span class="btn-icon">[RUN]</span><span>Start Reconnaissance</span>';
                 }
                 if (this.elements.addToGraph) {
-                    this.elements.addToGraph.disabled = !isQueueEmpty;
+                    this.elements.addToGraph.disabled = false;
                     this.elements.addToGraph.classList.remove('loading');
                 }
                 if (this.elements.stopScan) {
@@ -867,6 +973,9 @@ class DNSReconApp {
                 if (this.elements.targetInput) this.elements.targetInput.disabled = false;
                 if (this.elements.maxDepth) this.elements.maxDepth.disabled = false;
                 if (this.elements.configureSettings) this.elements.configureSettings.disabled = false;
+                
+                // Update manual refresh button visibility
+                this.updateManualRefreshButton();
                 break;
         }
     }
@@ -2023,6 +2132,16 @@ class DNSReconApp {
 
     async extractNode(largeEntityId, nodeId) {
         try {
+            console.log(`Extracting node ${nodeId} from large entity ${largeEntityId}`);
+            
+            // Show immediate feedback
+            const button = document.querySelector(`[data-node-id="${nodeId}"][data-large-entity-id="${largeEntityId}"]`);
+            if (button) {
+                const originalContent = button.innerHTML;
+                button.innerHTML = '[...]';
+                button.disabled = true;
+            }
+
             const response = await this.apiCall('/api/graph/large-entity/extract', 'POST', {
                 large_entity_id: largeEntityId,
                 node_id: nodeId,
@@ -2030,14 +2149,33 @@ class DNSReconApp {
 
             if (response.success) {
                 this.showSuccess(response.message);
+
+                // FIXED: Don't update local modal data - let backend be source of truth
+                // Force immediate graph update to get fresh backend data
+                console.log('Extraction successful, updating graph with fresh backend data');
+                await this.updateGraph();
                 
-                // If the scanner was idle, it's now running. Start polling to see the new node appear.
+                // FIXED: Re-fetch graph data instead of manipulating local state
-                if (this.scanStatus === 'idle') {
+                setTimeout(async () => {
-                    this.startPolling(1000);
+                    try {
-                } else {
+                        const graphResponse = await this.apiCall('/api/graph');
-                    // If already scanning, force a quick graph update to see the change sooner.
+                        if (graphResponse.success) {
-                    setTimeout(() => this.updateGraph(), 500);
+                            this.graphManager.updateGraph(graphResponse.graph);
-                }
+                            
+                            // Update modal with fresh data if still open
+                            if (this.elements.nodeModal && this.elements.nodeModal.style.display === 'block') {
+                                if (this.graphManager.nodes) {
+                                    const updatedLargeEntity = this.graphManager.nodes.get(largeEntityId);
+                                    if (updatedLargeEntity) {
+                                        this.showNodeModal(updatedLargeEntity);
+                                    }
+                                }
+                            }
+                        }
+                    } catch (error) {
+                        console.error('Error refreshing graph after extraction:', error);
+                    }
+                }, 100);
 
             } else {
                 throw new Error(response.error || 'Extraction failed on the server.');
@@ -2045,6 +2183,13 @@ class DNSReconApp {
         } catch (error) {
             console.error('Failed to extract node:', error);
             this.showError(`Extraction failed: ${error.message}`);
+            
+            // Restore button state on error
+            const button = document.querySelector(`[data-node-id="${nodeId}"][data-large-entity-id="${largeEntityId}"]`);
+            if (button) {
+                button.innerHTML = '[+]';
+                button.disabled = false;
+            }
         }
     }
 
@@ -2644,5 +2789,5 @@ style.textContent = `
 document.head.appendChild(style);
 
 // Initialize application when page loads
-console.log('Creating DNSReconApp instance...');
+console.log('Creating DNScopeApp instance...');
-const app = new DNSReconApp();
+const app = new DNScopeApp();

templates/index.html

@@ -4,7 +4,7 @@
 <head>
     <meta charset="UTF-8">
     <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>DNSRecon - Infrastructure Reconnaissance</title>
+    <title>DNScope - Infrastructure Reconnaissance</title>
     <link rel="stylesheet" href="{{ url_for('static', filename='css/main.css') }}">
     <script src="https://cdnjs.cloudflare.com/ajax/libs/vis/4.21.0/vis.min.js"></script>
     <link href="https://cdnjs.cloudflare.com/ajax/libs/vis/4.21.0/vis.min.css" rel="stylesheet" type="text/css">
@@ -18,8 +18,8 @@
         <header class="header">
             <div class="header-content">
                 <div class="logo">
-                    <span class="logo-icon">[DNS]</span>
+                    <span class="logo-icon">[DN]</span>
-                    <span class="logo-text">RECON</span>
+                    <span class="logo-text">Scope</span>
                 </div>
                 <div class="status-indicator">
                     <span id="connection-status" class="status-dot"></span>

utils/__init__.py

@@ -1,7 +1,7 @@
-# dnsrecon-reduced/utils/__init__.py
+# DNScope-reduced/utils/__init__.py
 
 """
-Utility modules for DNSRecon.
+Utility modules for DNScope.
 Contains helper functions, export management, and supporting utilities.
 """
 

utils/export_manager.py

@@ -1,7 +1,7 @@
-# dnsrecon-reduced/utils/export_manager.py
+# DNScope-reduced/utils/export_manager.py
 
 """
-Centralized export functionality for DNSRecon.
+Centralized export functionality for DNScope.
 Handles all data export operations with forensic integrity and proper formatting.
 ENHANCED: Professional forensic executive summary generation for court-ready documentation.
 """
@@ -18,7 +18,7 @@ from utils.helpers import _is_valid_domain, _is_valid_ip
 
 class ExportManager:
     """
-    Centralized manager for all DNSRecon export operations.
+    Centralized manager for all DNScope export operations.
     Maintains forensic integrity and provides consistent export formats.
     ENHANCED: Advanced forensic analysis and professional reporting capabilities.
     """
@@ -324,7 +324,7 @@ class ExportManager:
             "a complete audit trail maintained for forensic integrity.",
             "",
             f"Investigation completed: {now}",
-            f"Report authenticated by: DNSRecon v{self._get_version()}",
+            f"Report authenticated by: DNScope v{self._get_version()}",
             "",
             "=" * 80,
             "END OF REPORT",
@@ -694,7 +694,7 @@ class ExportManager:
                 if centrality >= threshold]
     
     def _get_version(self) -> str:
-        """Get DNSRecon version for report authentication."""
+        """Get DNScope version for report authentication."""
         return "1.0.0-forensic"
     
     def export_graph_json(self, graph_manager) -> Dict[str, Any]:
@@ -717,7 +717,7 @@ class ExportManager:
                 'last_modified': graph_manager.last_modified,
                 'total_nodes': graph_manager.get_node_count(),
                 'total_edges': graph_manager.get_edge_count(),
-                'graph_format': 'dnsrecon_v1_unified_model'
+                'graph_format': 'DNScope_v1_unified_model'
             },
             'graph': graph_data,
             'statistics': graph_manager.get_statistics()
@@ -818,7 +818,7 @@ class ExportManager:
         }
         
         extension = extension_map.get(export_type, 'txt')
-        return f"dnsrecon_{export_type}_{safe_target}_{timestamp_str}.{extension}"
+        return f"DNScope_{export_type}_{safe_target}_{timestamp_str}.{extension}"
 
 
 class CustomJSONEncoder(json.JSONEncoder):

utils/helpers.py

@@ -1,4 +1,4 @@
-# dnsrecon-reduced/utils/helpers.py
+# DNScope-reduced/utils/helpers.py
 
 import ipaddress
 from typing import Union