attempt bugfix

Reviewed-on: mstoeck3/dnsrecon#4

.env.example

@@ -0,0 +1,36 @@
+# ===============================================
+# DNScope Environment Variables
+# ===============================================
+# Copy this file to .env and fill in your values.
+
+# --- API Keys ---
+# Add your Shodan API key for the Shodan provider to be enabled.
+SHODAN_API_KEY=
+
+# --- Flask & Session Settings ---
+# A strong, random secret key is crucial for session security.
+FLASK_SECRET_KEY=your-very-secret-and-random-key-here
+FLASK_HOST=127.0.0.1
+FLASK_PORT=5000
+FLASK_DEBUG=True
+# How long a user's session in the browser lasts (in hours).
+FLASK_PERMANENT_SESSION_LIFETIME_HOURS=2
+# How long inactive scanner data is stored in Redis (in minutes).
+SESSION_TIMEOUT_MINUTES=60
+
+
+# --- Application Core Settings ---
+# The default number of levels to recurse when scanning.
+DEFAULT_RECURSION_DEPTH=2
+# Default timeout for provider API requests in seconds.
+DEFAULT_TIMEOUT=30
+# The number of concurrent provider requests to make.
+MAX_CONCURRENT_REQUESTS=1
+# The number of results from a provider that triggers the "large entity" grouping.
+LARGE_ENTITY_THRESHOLD=100
+# The number of times to retry a target if a provider fails.
+MAX_RETRIES_PER_TARGET=8
+# How long cached provider responses are stored (in hours).
+CACHE_TIMEOUT_HOURS=12
+
+GRAPH_POLLING_NODE_THRESHOLD=100

.gitignore

@@ -169,4 +169,4 @@ cython_debug/
 #.idea/
 
 dump.rdb
-.vscode
+cache/

README.md

@@ -1,40 +1,102 @@
-# DNSRecon - Passive Infrastructure Reconnaissance Tool
+# DNScope - Passive Infrastructure Reconnaissance Tool
 
-DNSRecon is an interactive, passive reconnaissance tool designed to map adversary infrastructure. It operates on a "free-by-default" model, ensuring core functionality without subscriptions, while allowing power users to enhance its capabilities with paid API keys.
+DNScope is an interactive, passive reconnaissance tool designed to map adversary infrastructure. It operates on a "free-by-default" model, ensuring core functionality without subscriptions, while allowing power users to enhance its capabilities with paid API keys. It is aimed at cybersecurity researchers, pentesters, and administrators who want to understand the public footprint of a target domain.
 
-**Current Status: Phase 2 Implementation**
+**Repo Link:** [https://github.com/overcuriousity/DNScope](https://github.com/overcuriousity/DNScope)
 
-  - ✅ Core infrastructure and graph engine
+-----
-  - ✅ Multi-provider support (crt.sh, DNS, Shodan)
+
-  - ✅ Session-based multi-user support
+## Concept and Philosophy
-  - ✅ Real-time web interface with interactive visualization
+
-  - ✅ Forensic logging system and JSON export
+The core philosophy of DNScope is to provide a comprehensive and accurate map of a target's infrastructure using only **passive data sources** by default. This means that, out of the box, DNScope will not send any traffic to the target's servers. Instead, it queries public and historical data sources to build a picture of the target's online presence. This approach is ideal for researchers and pentesters who want to gather intelligence without alerting the target, and for administrators who want to see what information about their own infrastructure is publicly available.
+
+For power users who require more in-depth information, DNScope can be configured to use API keys for services like Shodan, which provides a wealth of information about internet-connected devices. However, this is an optional feature, and the core functionality of the tool will always remain free and passive.
+
+-----
+
+-----
 
 ## Features
 
-  - **Passive Reconnaissance**: Gathers data without direct contact with target infrastructure.
+  * **Passive Reconnaissance**: Gathers data without direct contact with target infrastructure.
-  - **In-Memory Graph Analysis**: Uses NetworkX for efficient relationship mapping.
+  * **In-Memory Graph Analysis**: Uses NetworkX for efficient relationship mapping.
-  - **Real-Time Visualization**: The graph updates dynamically as the scan progresses.
+  * **Real-Time Visualization**: The graph updates dynamically as the scan progresses.
-  - **Forensic Logging**: A complete audit trail of all reconnaissance activities is maintained.
+  * **Forensic Logging**: A complete audit trail of all reconnaissance activities is maintained.
-  - **Confidence Scoring**: Relationships are weighted based on the reliability of the data source.
+  * **Session Management**: Supports concurrent user sessions with isolated scanner instances.
-  - **Session Management**: Supports concurrent user sessions with isolated scanner instances.
+  * **Extensible Provider Architecture**: Easily add new data sources to expand the tool's capabilities.
+  * **Web-Based UI**: An intuitive and interactive web interface for managing scans and visualizing results.
+  * **Export Options**: Export scan results to JSON, a list of targets to a text file, or an executive summary.
+  * **API Key Management**: Securely manage API keys for various providers through the web interface.
+  * **Provider Management**: Enable or disable providers for the current session.
 
-## Installation
+-----
+
+## Technical Architecture
+
+DNScope is a web-based application built with a modern technology stack:
+
+  * **Backend**: The backend is a **Flask** application that provides a REST API for the frontend and manages the scanning process.
+  * **Scanning Engine**: The core scanning engine is a multi-threaded Python application that uses a provider-based architecture to query different data sources.
+  * **Session Management**: **Redis** is used for session management, allowing for concurrent user sessions with isolated scanner instances.
+  * **Data Storage**: The application uses an in-memory graph to store and analyze the relationships between different pieces of information. The graph is built using the **NetworkX** library.
+  * **Frontend**: The frontend is a single-page application that uses JavaScript to interact with the backend API and visualize the graph.
+
+-----
+
+## Data Sources
+
+DNScope queries the following data sources:
+
+  * **DNS**: Standard DNS lookups (A, AAAA, CNAME, MX, NS, SOA, TXT).
+  * **crt.sh**: A certificate transparency log that provides information about SSL/TLS certificates.
+  * **Shodan**: A search engine for internet-connected devices (requires an API key).
+
+-----
+
+## Installation and Setup
 
 ### Prerequisites
 
-  - Python 3.8 or higher
+  * Python 3.8 or higher
-  - A modern web browser with JavaScript enabled
+  * A modern web browser with JavaScript enabled
-  - (Recommended) A Linux host for running the application and the optional DNS cache.
+  * A Linux host for running the application
+  * Redis Server
 
-### 1\. Clone the Project
+### 1\. Install Redis
+
+It is recommended to install Redis from the official repositories.
+
+**On Debian/Ubuntu:**
 
 ```bash
-git clone https://github.com/your-repo/dnsrecon.git
+sudo apt-get update
-cd dnsrecon
+sudo apt-get install redis-server
 ```
 
-### 2\. Install Python Dependencies
+**On CentOS/RHEL:**
+
+```bash
+sudo yum install redis
+sudo systemctl start redis
+sudo systemctl enable redis
+```
+
+You can verify that Redis is running with the following command:
+
+```bash
+redis-cli ping
+```
+
+You should see `PONG` as the response.
+
+### 2\. Clone the Project
+
+```bash
+git clone https://github.com/overcuriousity/DNScope
+cd DNScope
+```
+
+### 3\. Install Python Dependencies
 
 It is highly recommended to use a virtual environment:
 
@@ -44,181 +106,90 @@ source venv/bin/activate
 pip install -r requirements.txt
 ```
 
-### 3\. (Optional but Recommended) Set up a Local DNS Caching Resolver
+The `requirements.txt` file contains the following dependencies:
 
-Running a local DNS caching resolver can significantly speed up DNS queries and reduce your network footprint. Here’s how to set up `unbound` on a Debian-based Linux distribution (like Ubuntu).
+  * Flask
+  * networkx
+  * requests
+  * python-dateutil
+  * Werkzeug
+  * urllib3
+  * dnspython
+  * gunicorn
+  * redis
+  * python-dotenv
+  * psycopg2-binary
 
-**a. Install Unbound:**
+### 4\. Configure the Application
+
+DNScope is configured using a `.env` file. You can copy the provided example file and edit it to suit your needs:
 
 ```bash
-sudo apt update
+cp .env.example .env
-sudo apt install unbound -y
 ```
 
-**b. Configure Unbound:**
+The following environment variables are available for configuration:
-Create a new configuration file for DNSRecon:
 
-```bash
+| Variable | Description | Default |
-sudo nano /etc/unbound/unbound.conf.d/dnsrecon.conf
+| :--- | :--- | :--- |
-```
+| `SHODAN_API_KEY` | Your Shodan API key. | |
+| `FLASK_SECRET_KEY`| A strong, random secret key for session security. | `your-very-secret-and-random-key-here` |
+| `FLASK_HOST` | The host address for the Flask application. | `127.0.0.1` |
+| `FLASK_PORT` | The port for the Flask application. | `5000` |
+| `FLASK_DEBUG` | Enable or disable Flask's debug mode. | `True` |
+| `FLASK_PERMANENT_SESSION_LIFETIME_HOURS`| How long a user's session in the browser lasts (in hours). | `2` |
+| `SESSION_TIMEOUT_MINUTES` | How long inactive scanner data is stored in Redis (in minutes). | `60` |
+| `DEFAULT_RECURSION_DEPTH` | The default number of levels to recurse when scanning. | `2` |
+| `DEFAULT_TIMEOUT` | Default timeout for provider API requests in seconds. | `30` |
+| `MAX_CONCURRENT_REQUESTS`| The number of concurrent provider requests to make. | `5` |
+| `LARGE_ENTITY_THRESHOLD`| The number of results from a provider that triggers the "large entity" grouping. | `100` |
+| `MAX_RETRIES_PER_TARGET`| The number of times to retry a target if a provider fails. | `8` |
+| `CACHE_EXPIRY_HOURS`| How long cached provider responses are stored (in hours). | `12` |
 
-Add the following content to the file:
+-----
 
-```
+## Running the Application
-server:
-    # Listen on localhost for all users
-    interface: 127.0.0.1
-    access-control: 0.0.0.0/0 refuse
-    access-control: 127.0.0.0/8 allow
 
-    # Enable prefetching of popular items
+For development, you can run the application using the following command:
-    prefetch: yes
-```
-
-**c. Restart Unbound and set it as the default resolver:**
-
-```bash
-sudo systemctl restart unbound
-sudo systemctl enable unbound
-```
-
-To use this resolver for your system, you may need to update your network settings to point to `127.0.0.1` as your DNS server.
-
-**d. Update DNSProvider to use the local resolver:**
-In `dnsrecon/providers/dns_provider.py`, you can explicitly set the resolver's nameservers in the `__init__` method:
-
-```python
-# dnsrecon/providers/dns_provider.py
-
-class DNSProvider(BaseProvider):
-    def __init__(self, session_config=None):
-        """Initialize DNS provider with session-specific configuration."""
-        super().__init__(...)
-
-        # Configure DNS resolver
-        self.resolver = dns.resolver.Resolver()
-        self.resolver.nameservers = ['127.0.0.1'] # Use local caching resolver
-        self.resolver.timeout = 5
-        self.resolver.lifetime = 10
-```
-
-## Usage (Development)
-
-### 1\. Start the Application
 
 ```bash
 python app.py
 ```
 
-### 2\. Open Your Browser
+For production, it is recommended to use a more robust server, such as Gunicorn:
-
-Navigate to `http://127.0.0.1:5000`.
-
-### 3\. Basic Reconnaissance Workflow
-
-1.  **Enter Target Domain**: Input a domain like `example.com`.
-2.  **Select Recursion Depth**: Depth 2 is recommended for most investigations.
-3.  **Start Reconnaissance**: Click "Start Reconnaissance" to begin.
-4.  **Monitor Progress**: Watch the real-time graph build as relationships are discovered.
-5.  **Analyze and Export**: Interact with the graph and download the results when the scan is complete.
-
-## Production Deployment
-
-To deploy DNSRecon in a production environment, follow these steps:
-
-### 1\. Use a Production WSGI Server
-
-Do not use the built-in Flask development server for production. Use a WSGI server like **Gunicorn**:
 
 ```bash
-pip install gunicorn
 gunicorn --workers 4 --bind 0.0.0.0:5000 app:app
 ```
 
-### 2\. Configure Environment Variables
+-----
 
-Set the following environment variables for a secure and configurable deployment:
+## Systemd Service
 
-```bash
+To run DNScope as a service that starts automatically on boot, you can use `systemd`.
-# Generate a strong, random secret key
-export SECRET_KEY='your-super-secret-and-random-key'
-
-# Set Flask to production mode
-export FLASK_ENV='production'
-export FLASK_DEBUG=False
-
-# API keys (optional, but recommended for full functionality)
-export SHODAN_API_KEY="your_shodan_key"
-```
-
-### 3\. Use a Reverse Proxy
-
-Set up a reverse proxy like **Nginx** to sit in front of the Gunicorn server. This provides several benefits, including:
-
-  - **TLS/SSL Termination**: Securely handle HTTPS traffic.
-  - **Load Balancing**: Distribute traffic across multiple application instances.
-  - **Serving Static Files**: Efficiently serve CSS and JavaScript files.
-
-**Example Nginx Configuration:**
-
-```nginx
-server {
-    listen 80;
-    server_name your_domain.com;
-
-    location / {
-        return 301 https://$host$request_uri;
-    }
-}
-
-server {
-    listen 443 ssl;
-    server_name your_domain.com;
-
-    # SSL cert configuration
-    ssl_certificate /etc/letsencrypt/live/your_domain.com/fullchain.pem;
-    ssl_certificate_key /etc/letsencrypt/live/your_domain.com/privkey.pem;
-
-    location / {
-        proxy_pass http://127.0.0.1:5000;
-        proxy_set_header Host $host;
-        proxy_set_header X-Real-IP $remote_addr;
-        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-    }
-
-    location /static {
-        alias /path/to/your/dnsrecon/static;
-        expires 30d;
-    }
-}
-```
-
-## Autostart with systemd
-
-To run DNSRecon as a service that starts automatically on boot, you can use `systemd`.
 
 ### 1\. Create a `.service` file
 
 Create a new service file in `/etc/systemd/system/`:
 
 ```bash
-sudo nano /etc/systemd/system/dnsrecon.service
+sudo nano /etc/systemd/system/DNScope.service
 ```
 
 ### 2\. Add the Service Configuration
 
-Paste the following configuration into the file. **Remember to replace `/path/to/your/dnsrecon` and `your_user` with your actual project path and username.**
+Paste the following configuration into the file. **Remember to replace `/path/to/your/DNScope` and `your_user` with your actual project path and username.**
 
 ```ini
 [Unit]
-Description=DNSRecon Application
+Description=DNScope Application
 After=network.target
 
 [Service]
 User=your_user
 Group=your_user
-WorkingDirectory=/path/to/your/dnsrecon
+WorkingDirectory=/path/to/your/DNScope
-ExecStart=/path/to/your/dnsrecon/venv/bin/gunicorn --workers 4 --bind 0.0.0.0:5000 app:app
+ExecStart=/path/to/your/DNScope/venv/bin/gunicorn --workers 4 --bind 0.0.0.0:5000 app:app
 Restart=always
 Environment="SECRET_KEY=your-super-secret-and-random-key"
 Environment="FLASK_ENV=production"
@@ -235,22 +206,87 @@ Reload the `systemd` daemon, enable the service to start on boot, and then start
 
 ```bash
 sudo systemctl daemon-reload
-sudo systemctl enable dnsrecon.service
+sudo systemctl enable DNScope.service
-sudo systemctl start dnsrecon.service
+sudo systemctl start DNScope.service
 ```
 
 You can check the status of the service at any time with:
 
 ```bash
-sudo systemctl status dnsrecon.service
+sudo systemctl status DNScope.service
 ```
 
-## Security Considerations
+-----
 
-  - **API Keys**: API keys are stored in memory for the duration of a user session and are not written to disk.
+## Updating the Application
-  - **Rate Limiting**: DNSRecon includes built-in rate limiting to be respectful to data sources.
+
-  - **Local Use**: The application is designed for local or trusted network use and does not have built-in authentication. **Do not expose it directly to the internet without proper security controls.**
+To update the application, you should first pull the latest changes from the git repository. Then, you will need to wipe the Redis database and the local cache to ensure that you are using the latest data.
+
+### 1\. Update the Code
+
+```bash
+git pull
+```
+
+### 2\. Wipe the Redis Database
+
+```bash
+redis-cli FLUSHALL
+```
+
+### 3\. Wipe the Local Cache
+
+```bash
+rm -rf cache/*
+```
+
+### 4\. Restart the Service
+
+```bash
+sudo systemctl restart DNScope.service
+```
+
+-----
+
+## Extensibility
+
+DNScope is designed to be extensible, and adding new providers is a straightforward process. To add a new provider, you will need to create a new Python file in the `providers` directory that inherits from the `BaseProvider` class. The new provider will need to implement the following methods:
+
+  * `get_name()`: Return the name of the provider.
+  * `get_display_name()`: Return a display-friendly name for the provider.
+  * `requires_api_key()`: Return `True` if the provider requires an API key.
+  * `get_eligibility()`: Return a dictionary indicating whether the provider can query domains and/or IPs.
+  * `is_available()`: Return `True` if the provider is available (e.g., if an API key is configured).
+  * `query_domain(domain)`: Query the provider for information about a domain.
+  * `query_ip(ip)`: Query the provider for information about an IP address.
+
+-----
+
+## Unique Capabilities and Limitations
+
+### Unique Capabilities
+
+  * **Graph-Based Analysis**: The use of a graph-based data model allows for a more intuitive and powerful analysis of the relationships between different pieces of information.
+  * **Real-Time Visualization**: The real-time visualization of the graph provides immediate feedback and allows for a more interactive and engaging analysis experience.
+  * **Session Management**: The session management feature allows multiple users to use the application concurrently without interfering with each other's work.
+
+### Limitations
+
+  * **Passive-Only by Default**: While the passive-only approach is a key feature of the tool, it also means that the information it can gather is limited to what is publicly available.
+  * **No Active Scanning**: The tool does not perform any active scanning, such as port scanning or vulnerability scanning.
+
+-----
 
 ## License
 
-This project is licensed under the terms of the license agreement found in the `LICENSE` file.
+This project is licensed under the terms of the **BSD-3-Clause** license.
+
+Copyright (c) 2025 mstoeck3.
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+1.  Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+2.  Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+3.  Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

config.py

@@ -1,114 +1,163 @@
+# DNScope-reduced/config.py
+
 """
-Configuration management for DNSRecon tool.
+Configuration management for DNScope tool.
 Handles API key storage, rate limiting, and default settings.
 """
 
 import os
 from typing import Dict, Optional
+from dotenv import load_dotenv
 
+# Load environment variables from .env file
+load_dotenv()
 
 class Config:
-    """Configuration manager for DNSRecon application."""
+    """Configuration manager for DNScope application."""
     
     def __init__(self):
         """Initialize configuration with default values."""
-        self.api_keys: Dict[str, Optional[str]] = {
+        self.api_keys: Dict[str, Optional[str]] = {}
-            'shodan': None
-        }
         
-        # Default settings
+        # --- General Settings ---
         self.default_recursion_depth = 2
-        self.default_timeout = 10
+        self.default_timeout = 60
-        self.max_concurrent_requests = 5
+        self.max_concurrent_requests = 1
         self.large_entity_threshold = 100
+        self.max_retries_per_target = 8
         
-        # Rate limiting settings (requests per minute)
+        # --- Graph Polling Performance Settings ---
+        self.graph_polling_node_threshold = 100  # Stop graph auto-polling above this many nodes
+        
+        # --- Provider Caching Settings ---
+        self.cache_timeout_hours = 6  # Provider-specific cache timeout
+        
+        # --- Rate Limiting (requests per minute) ---
         self.rate_limits = {
-            'crtsh': 60,          # Free service, be respectful
+            'crtsh': 5,
-            'shodan': 60,         # API dependent
+            'shodan': 60,
-            'dns': 100            # Local DNS queries
+            'dns': 100,
+            'correlation': 0 # Set to 0 to make sure correlations run last
         }
         
-        # Provider settings
+        # --- Provider Settings ---
         self.enabled_providers = {
-            'crtsh': True,        # Always enabled (free)
+            'crtsh': True,
-            'dns': True,          # Always enabled (free)
+            'dns': True,
-            'shodan': False       # Requires API key
+            'shodan': False,
+            'correlation': True # Enable the new provider by default
         }
         
-        # Logging configuration
+        # --- Logging ---
         self.log_level = 'INFO'
         self.log_format = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
         
-        # Flask configuration
+        # --- Flask & Session Settings ---
         self.flask_host = '127.0.0.1'
         self.flask_port = 5000
         self.flask_debug = True
+        self.flask_secret_key = 'default-secret-key-change-me'
+        self.flask_permanent_session_lifetime_hours = 2
+        self.session_timeout_minutes = 60
         
-    def set_api_key(self, provider: str, api_key: str) -> bool:
+        # Load environment variables to override defaults
+        self.load_from_env()
+        
+    def load_from_env(self):
+        """Load configuration from environment variables."""
+        self.set_api_key('shodan', os.getenv('SHODAN_API_KEY'))
+
+        # Override settings from environment
+        self.default_recursion_depth = int(os.getenv('DEFAULT_RECURSION_DEPTH', self.default_recursion_depth))
+        self.default_timeout = int(os.getenv('DEFAULT_TIMEOUT', self.default_timeout))
+        self.max_concurrent_requests = int(os.getenv('MAX_CONCURRENT_REQUESTS', self.max_concurrent_requests))
+        self.large_entity_threshold = int(os.getenv('LARGE_ENTITY_THRESHOLD', self.large_entity_threshold))
+        self.max_retries_per_target = int(os.getenv('MAX_RETRIES_PER_TARGET', self.max_retries_per_target))
+        self.cache_timeout_hours = int(os.getenv('CACHE_TIMEOUT_HOURS', self.cache_timeout_hours))
+        
+        # Override graph polling threshold from environment
+        self.graph_polling_node_threshold = int(os.getenv('GRAPH_POLLING_NODE_THRESHOLD', self.graph_polling_node_threshold))
+        
+        # Override Flask and session settings
+        self.flask_host = os.getenv('FLASK_HOST', self.flask_host)
+        self.flask_port = int(os.getenv('FLASK_PORT', self.flask_port))
+        self.flask_debug = os.getenv('FLASK_DEBUG', str(self.flask_debug)).lower() == 'true'
+        self.flask_secret_key = os.getenv('FLASK_SECRET_KEY', self.flask_secret_key)
+        self.flask_permanent_session_lifetime_hours = int(os.getenv('FLASK_PERMANENT_SESSION_LIFETIME_HOURS', self.flask_permanent_session_lifetime_hours))
+        self.session_timeout_minutes = int(os.getenv('SESSION_TIMEOUT_MINUTES', self.session_timeout_minutes))
+
+    def set_api_key(self, provider: str, api_key: Optional[str]) -> bool:
+        """Set API key for a provider."""
+        self.api_keys[provider] = api_key
+        if api_key:
+            self.enabled_providers[provider] = True
+        return True
+    
+    def set_provider_enabled(self, provider: str, enabled: bool) -> bool:
         """
-        Set API key for a provider.
+        Set provider enabled status for the session.
         
         Args:
-            provider: Provider name (shodan, etc)
+            provider: Provider name
-            api_key: API key string
+            enabled: Whether the provider should be enabled
             
         Returns:
-            bool: True if key was set successfully
+            True if the setting was applied successfully
         """
-        if provider in self.api_keys:
+        provider_key = provider.lower()
-            self.api_keys[provider] = api_key
+        self.enabled_providers[provider_key] = enabled
-            self.enabled_providers[provider] = True if api_key else False
+        return True
-            return True
+
-        return False
+    def get_provider_enabled(self, provider: str) -> bool:
-    
-    def get_api_key(self, provider: str) -> Optional[str]:
         """
-        Get API key for a provider.
+        Get provider enabled status.
         
         Args:
             provider: Provider name
             
         Returns:
-            API key or None if not set
+            True if the provider is enabled
         """
+        provider_key = provider.lower()
+        return self.enabled_providers.get(provider_key, True)  # Default to enabled
+
+    def bulk_set_provider_settings(self, provider_settings: dict) -> dict:
+        """
+        Set multiple provider settings at once.
+        
+        Args:
+            provider_settings: Dict of provider_name -> {'enabled': bool, ...}
+            
+        Returns:
+            Dict with results for each provider
+        """
+        results = {}
+        
+        for provider_name, settings in provider_settings.items():
+            provider_key = provider_name.lower()
+            
+            try:
+                if 'enabled' in settings:
+                    self.enabled_providers[provider_key] = settings['enabled']
+                    results[provider_key] = {'success': True, 'enabled': settings['enabled']}
+                else:
+                    results[provider_key] = {'success': False, 'error': 'No enabled setting provided'}
+            except Exception as e:
+                results[provider_key] = {'success': False, 'error': str(e)}
+        
+        return results
+    
+    def get_api_key(self, provider: str) -> Optional[str]:
+        """Get API key for a provider."""
         return self.api_keys.get(provider)
     
     def is_provider_enabled(self, provider: str) -> bool:
-        """
+        """Check if a provider is enabled."""
-        Check if a provider is enabled.
-        
-        Args:
-            provider: Provider name
-            
-        Returns:
-            bool: True if provider is enabled
-        """
         return self.enabled_providers.get(provider, False)
     
     def get_rate_limit(self, provider: str) -> int:
-        """
+        """Get rate limit for a provider."""
-        Get rate limit for a provider.
-        
-        Args:
-            provider: Provider name
-            
-        Returns:
-            Rate limit in requests per minute
-        """
         return self.rate_limits.get(provider, 60)
-    
-    def load_from_env(self):
-        """Load configuration from environment variables."""
-        if os.getenv('SHODAN_API_KEY'):
-            self.set_api_key('shodan', os.getenv('SHODAN_API_KEY'))
-        
-        # Override default settings from environment
-        self.default_recursion_depth = int(os.getenv('DEFAULT_RECURSION_DEPTH', '2'))
-        self.flask_debug = os.getenv('FLASK_DEBUG', 'True').lower() == 'true'
-        self.default_timeout = 30
-        self.max_concurrent_requests = 5
-
 
 # Global configuration instance
 config = Config()

core/__init__.py

@@ -1,5 +1,5 @@
 """
-Core modules for DNSRecon passive reconnaissance tool.
+Core modules for DNScope passive reconnaissance tool.
 Contains graph management, scanning orchestration, and forensic logging.
 """
 
@@ -8,7 +8,6 @@ from .scanner import Scanner, ScanStatus
 from .logger import ForensicLogger, get_forensic_logger, new_session
 from .session_manager import session_manager 
 from .session_config import SessionConfig, create_session_config 
-from .task_manager import TaskManager, TaskType, ReconTask
 
 __all__ = [
     'GraphManager',
@@ -20,10 +19,7 @@ __all__ = [
     'new_session',
     'session_manager', 
     'SessionConfig',
-    'create_session_config',
+    'create_session_config'
-    'TaskManager',
-    'TaskType',
-    'ReconTask'
 ]
 
 __version__ = "1.0.0-phase2"

core/graph_manager.py

@@ -1,6 +1,11 @@
+# DNScope-reduced/core/graph_manager.py
+
 """
-Graph data model for DNSRecon using NetworkX.
+Graph data model for DNScope using NetworkX.
-Manages in-memory graph storage with confidence scoring and forensic metadata.
+Manages in-memory graph storage with forensic metadata.
+Now fully compatible with the unified ProviderResult data model.
+UPDATED: Fixed correlation exclusion keys to match actual attribute names.
+UPDATED: Removed export_json() method - now handled by ExportManager.
 """
 import re
 from datetime import datetime, timezone
@@ -14,7 +19,8 @@ class NodeType(Enum):
     """Enumeration of supported node types."""
     DOMAIN = "domain"
     IP = "ip"
-    ASN = "asn"
+    ISP = "isp"
+    CA = "ca"
     LARGE_ENTITY = "large_entity"
     CORRELATION_OBJECT = "correlation_object"
 
@@ -24,8 +30,9 @@ class NodeType(Enum):
 
 class GraphManager:
     """
-    Thread-safe graph manager for DNSRecon infrastructure mapping.
+    Thread-safe graph manager for DNScope infrastructure mapping.
-    Uses NetworkX for in-memory graph storage with confidence scoring.
+    Uses NetworkX for in-memory graph storage.
+    Compatible with unified ProviderResult data model.
     """
 
     def __init__(self):
@@ -33,118 +40,37 @@ class GraphManager:
         self.graph = nx.DiGraph()
         self.creation_time = datetime.now(timezone.utc).isoformat()
         self.last_modified = self.creation_time
-        self.correlation_index = {}
+        
-        # Compile regex for date filtering for efficiency
+    def add_node(self, node_id: str, node_type: NodeType, attributes: Optional[List[Dict[str, Any]]] = None,
-        self.date_pattern = re.compile(r'^\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}')
-
-    def __getstate__(self):
-        """Prepare GraphManager for pickling, excluding compiled regex."""
-        state = self.__dict__.copy()
-        # Compiled regex patterns are not always picklable
-        if 'date_pattern' in state:
-            del state['date_pattern']
-        return state
-
-    def __setstate__(self, state):
-        """Restore GraphManager state and recompile regex."""
-        self.__dict__.update(state)
-        self.date_pattern = re.compile(r'^\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}')
-
-    def _update_correlation_index(self, node_id: str, data: Any, path: List[str] = None):
-        """Recursively traverse metadata and add hashable values to the index."""
-        if path is None:
-            path = []
-
-        if isinstance(data, dict):
-            for key, value in data.items():
-                self._update_correlation_index(node_id, value, path + [key])
-        elif isinstance(data, list):
-            for i, item in enumerate(data):
-                self._update_correlation_index(node_id, item, path + [f"[{i}]"])
-        else:
-            self._add_to_correlation_index(node_id, data, ".".join(path))
-
-    def _add_to_correlation_index(self, node_id: str, value: Any, path_str: str):
-        """Add a hashable value to the correlation index, filtering out noise."""
-        if not isinstance(value, (str, int, float, bool)) or value is None:
-            return
-
-        # Ignore certain paths that contain noisy, non-unique identifiers
-        if any(keyword in path_str.lower() for keyword in ['count', 'total', 'timestamp', 'date']):
-            return
-
-        # Filter out common low-entropy values and date-like strings
-        if isinstance(value, str):
-            # FIXED: Prevent correlation on date/time strings.
-            if self.date_pattern.match(value):
-                return
-            if len(value) < 4 or value.lower() in ['true', 'false', 'unknown', 'none', 'crt.sh']:
-                return
-        elif isinstance(value, int) and abs(value) < 9999:
-            return  # Ignore small integers
-        elif isinstance(value, bool):
-            return  # Ignore boolean values
-
-        # Add the valuable correlation data to the index
-        if value not in self.correlation_index:
-            self.correlation_index[value] = {}
-        if node_id not in self.correlation_index[value]:
-            self.correlation_index[value][node_id] = []
-        if path_str not in self.correlation_index[value][node_id]:
-            self.correlation_index[value][node_id].append(path_str)
-
-    def _check_for_correlations(self, new_node_id: str, data: Any, path: List[str] = None) -> List[Dict]:
-        """Recursively traverse metadata to find correlations with existing data."""
-        if path is None:
-            path = []
-
-        all_correlations = []
-        if isinstance(data, dict):
-            for key, value in data.items():
-                if key == 'source':  # Avoid correlating on the provider name
-                    continue
-                all_correlations.extend(self._check_for_correlations(new_node_id, value, path + [key]))
-        elif isinstance(data, list):
-            for i, item in enumerate(data):
-                all_correlations.extend(self._check_for_correlations(new_node_id, item, path + [f"[{i}]"]))
-        else:
-            value = data
-            if value in self.correlation_index:
-                existing_nodes_with_paths = self.correlation_index[value]
-                unique_nodes = set(existing_nodes_with_paths.keys())
-                unique_nodes.add(new_node_id)
-
-                if len(unique_nodes) < 2:
-                    return all_correlations # Correlation must involve at least two distinct nodes
-
-                new_source = {'node_id': new_node_id, 'path': ".".join(path)}
-                all_sources = [new_source]
-                for node_id, paths in existing_nodes_with_paths.items():
-                    for p_str in paths:
-                        all_sources.append({'node_id': node_id, 'path': p_str})
-
-                all_correlations.append({
-                    'value': value,
-                    'sources': all_sources,
-                    'nodes': list(unique_nodes)
-                })
-        return all_correlations
-
-    def add_node(self, node_id: str, node_type: NodeType, attributes: Optional[Dict[str, Any]] = None,
                 description: str = "", metadata: Optional[Dict[str, Any]] = None) -> bool:
-        """Add a node to the graph, update attributes, and process correlations."""
+        """
+        Add a node to the graph, update attributes, and process correlations.
+        Now compatible with unified data model - attributes are dictionaries from converted StandardAttribute objects.
+        """
         is_new_node = not self.graph.has_node(node_id)
         if is_new_node:
             self.graph.add_node(node_id, type=node_type.value,
                                 added_timestamp=datetime.now(timezone.utc).isoformat(),
-                                attributes=attributes or {},
+                                attributes=attributes or [],  # Store as a list from the start
                                 description=description,
                                 metadata=metadata or {})
         else:
-            # Safely merge new attributes into existing attributes
+            # Safely merge new attributes into the existing list of attributes
             if attributes:
-                existing_attributes = self.graph.nodes[node_id].get('attributes', {})
+                existing_attributes = self.graph.nodes[node_id].get('attributes', [])
-                existing_attributes.update(attributes)
+                
+                # Handle cases where old data might still be in dictionary format
+                if not isinstance(existing_attributes, list):
+                    existing_attributes = []
+
+                # Create a set of existing attribute names for efficient duplicate checking
+                existing_attr_names = {attr['name'] for attr in existing_attributes}
+
+                for new_attr in attributes:
+                    if new_attr['name'] not in existing_attr_names:
+                        existing_attributes.append(new_attr)
+                        existing_attr_names.add(new_attr['name'])
+                
                 self.graph.nodes[node_id]['attributes'] = existing_attributes
             if description:
                 self.graph.nodes[node_id]['description'] = description
@@ -153,198 +79,42 @@ class GraphManager:
                 existing_metadata.update(metadata)
                 self.graph.nodes[node_id]['metadata'] = existing_metadata
 
-        if attributes and node_type != NodeType.CORRELATION_OBJECT:
-            correlations = self._check_for_correlations(node_id, attributes)
-            for corr in correlations:
-                value = corr['value']
-                
-                # STEP 1: Substring check against all existing nodes
-                if self._correlation_value_matches_existing_node(value):
-                    # Skip creating correlation node - would be redundant
-                    continue
-                
-                # STEP 2: Filter out node pairs that already have direct edges
-                eligible_nodes = self._filter_nodes_without_direct_edges(set(corr['nodes']))
-                
-                if len(eligible_nodes) < 2:
-                    # Need at least 2 nodes to create a correlation
-                    continue
-                    
-                # STEP 3: Check for existing correlation node with same connection pattern
-                correlation_nodes_with_pattern = self._find_correlation_nodes_with_same_pattern(eligible_nodes)
-                
-                if correlation_nodes_with_pattern:
-                    # STEP 4: Merge with existing correlation node
-                    target_correlation_node = correlation_nodes_with_pattern[0]
-                    self._merge_correlation_values(target_correlation_node, value, corr)
-                else:
-                    # STEP 5: Create new correlation node for eligible nodes only
-                    correlation_node_id = f"corr_{abs(hash(str(sorted(eligible_nodes))))}"
-                    self.add_node(correlation_node_id, NodeType.CORRELATION_OBJECT,
-                                metadata={'values': [value], 'sources': corr['sources'],
-                                            'correlated_nodes': list(eligible_nodes)})
-                    
-                    # Create edges from eligible nodes to this correlation node
-                    for c_node_id in eligible_nodes:
-                        if self.graph.has_node(c_node_id):
-                            attribute = corr['sources'][0]['path'].split('.')[-1]
-                            relationship_type = f"c_{attribute}"
-                            self.add_edge(c_node_id, correlation_node_id, relationship_type, confidence_score=0.9)
-
-            self._update_correlation_index(node_id, attributes)
-
         self.last_modified = datetime.now(timezone.utc).isoformat()
         return is_new_node
 
-    def _filter_nodes_without_direct_edges(self, node_set: set) -> set:
-        """
-        Filter out nodes that already have direct edges between them.
-        Returns set of nodes that should be included in correlation.
-        """
-        nodes_list = list(node_set)
-        eligible_nodes = set(node_set)  # Start with all nodes
-        
-        # Check all pairs of nodes
-        for i in range(len(nodes_list)):
-            for j in range(i + 1, len(nodes_list)):
-                node_a = nodes_list[i]
-                node_b = nodes_list[j]
-                
-                # Check if direct edge exists in either direction
-                if self._has_direct_edge_bidirectional(node_a, node_b):
-                    # Remove both nodes from eligible set since they're already connected
-                    eligible_nodes.discard(node_a)
-                    eligible_nodes.discard(node_b)
-        
-        return eligible_nodes
-
-    def _has_direct_edge_bidirectional(self, node_a: str, node_b: str) -> bool:
-        """
-        Check if there's a direct edge between two nodes in either direction.
-        Returns True if node_a→node_b OR node_b→node_a exists.
-        """
-        return (self.graph.has_edge(node_a, node_b) or 
-                self.graph.has_edge(node_b, node_a))
-
-    def _correlation_value_matches_existing_node(self, correlation_value: str) -> bool:
-        """
-        Check if correlation value contains any existing node ID as substring.
-        Returns True if match found (correlation node should NOT be created).
-        """
-        correlation_str = str(correlation_value).lower()
-        
-        # Check against all existing nodes
-        for existing_node_id in self.graph.nodes():
-            if existing_node_id.lower() in correlation_str:
-                return True
-        
-        return False
-
-    def _find_correlation_nodes_with_same_pattern(self, node_set: set) -> List[str]:
-        """
-        Find existing correlation nodes that have the exact same pattern of connected nodes.
-        Returns list of correlation node IDs with matching patterns.
-        """
-        correlation_nodes = self.get_nodes_by_type(NodeType.CORRELATION_OBJECT)
-        matching_nodes = []
-        
-        for corr_node_id in correlation_nodes:
-            # Get all nodes connected to this correlation node
-            connected_nodes = set()
-            
-            # Add all predecessors (nodes pointing TO the correlation node)
-            connected_nodes.update(self.graph.predecessors(corr_node_id))
-            
-            # Add all successors (nodes pointed TO by the correlation node)  
-            connected_nodes.update(self.graph.successors(corr_node_id))
-            
-            # Check if the pattern matches exactly
-            if connected_nodes == node_set:
-                matching_nodes.append(corr_node_id)
-        
-        return matching_nodes
-
-    def _merge_correlation_values(self, target_node_id: str, new_value: Any, corr_data: Dict) -> None:
-        """
-        Merge a new correlation value into an existing correlation node.
-        Uses same logic as large entity merging.
-        """
-        if not self.graph.has_node(target_node_id):
-            return
-            
-        target_metadata = self.graph.nodes[target_node_id]['metadata']
-        
-        # Get existing values (ensure it's a list)
-        existing_values = target_metadata.get('values', [])
-        if not isinstance(existing_values, list):
-            existing_values = [existing_values]
-        
-        # Add new value if not already present
-        if new_value not in existing_values:
-            existing_values.append(new_value)
-            
-        # Merge sources
-        existing_sources = target_metadata.get('sources', [])
-        new_sources = corr_data.get('sources', [])
-        
-        # Create set of unique sources based on (node_id, path) tuples
-        source_set = set()
-        for source in existing_sources + new_sources:
-            source_tuple = (source['node_id'], source['path'])
-            source_set.add(source_tuple)
-        
-        # Convert back to list of dictionaries
-        merged_sources = [{'node_id': nid, 'path': path} for nid, path in source_set]
-        
-        # Update metadata
-        target_metadata.update({
-            'values': existing_values,
-            'sources': merged_sources,
-            'correlated_nodes': list(set(target_metadata.get('correlated_nodes', []) + corr_data.get('nodes', []))),
-            'merge_count': len(existing_values),
-            'last_merge_timestamp': datetime.now(timezone.utc).isoformat()
-        })
-        
-        # Update description to reflect merged nature
-        value_count = len(existing_values)
-        node_count = len(target_metadata['correlated_nodes'])
-        self.graph.nodes[target_node_id]['description'] = (
-            f"Correlation container with {value_count} merged values "
-            f"across {node_count} nodes"
-        )
-        
     def add_edge(self, source_id: str, target_id: str, relationship_type: str,
-                 confidence_score: float = 0.5, source_provider: str = "unknown",
+                source_provider: str = "unknown",
-                 raw_data: Optional[Dict[str, Any]] = None) -> bool:
+                raw_data: Optional[Dict[str, Any]] = None) -> bool:
-        """Add or update an edge between two nodes, ensuring nodes exist."""
+        """
+        UPDATED: Add or update an edge between two nodes with raw relationship labels.
+        """
         if not self.graph.has_node(source_id) or not self.graph.has_node(target_id):
             return False
 
-        new_confidence = confidence_score
         
-        if relationship_type.startswith("c_"):
+        # UPDATED: Use raw relationship type - no formatting
-            edge_label = relationship_type
+        edge_label = relationship_type
-        else:
-            edge_label = f"{source_provider}_{relationship_type}"
-        
-        if self.graph.has_edge(source_id, target_id):
-            # If edge exists, update confidence if the new score is higher.
-            if new_confidence > self.graph.edges[source_id, target_id].get('confidence_score', 0):
-                self.graph.edges[source_id, target_id]['confidence_score'] = new_confidence
-                self.graph.edges[source_id, target_id]['updated_timestamp'] = datetime.now(timezone.utc).isoformat()
-                self.graph.edges[source_id, target_id]['updated_by'] = source_provider
-            return False
 
-        # Add a new edge with all attributes.
+        # Add a new edge with raw attributes
         self.graph.add_edge(source_id, target_id,
                             relationship_type=edge_label,
-                            confidence_score=new_confidence,
                             source_provider=source_provider,
                             discovery_timestamp=datetime.now(timezone.utc).isoformat(),
                             raw_data=raw_data or {})
         self.last_modified = datetime.now(timezone.utc).isoformat()
         return True
 
+    def remove_node(self, node_id: str) -> bool:
+        """Remove a node and its connected edges from the graph."""
+        if not self.graph.has_node(node_id):
+            return False
+
+        # Remove node from the graph (NetworkX handles removing connected edges)
+        self.graph.remove_node(node_id)
+        
+        self.last_modified = datetime.now(timezone.utc).isoformat()
+        return True
+
     def get_node_count(self) -> int:
         """Get total number of nodes in the graph."""
         return self.graph.number_of_nodes()
@@ -357,97 +127,92 @@ class GraphManager:
         """Get all nodes of a specific type."""
         return [n for n, d in self.graph.nodes(data=True) if d.get('type') == node_type.value]
 
-    def get_neighbors(self, node_id: str) -> List[str]:
-        """Get all unique neighbors (predecessors and successors) for a node."""
-        if not self.graph.has_node(node_id):
-            return []
-        return list(set(self.graph.predecessors(node_id)) | set(self.graph.successors(node_id)))
-
-    def get_high_confidence_edges(self, min_confidence: float = 0.8) -> List[Tuple[str, str, Dict]]:
-        """Get edges with confidence score above a given threshold."""
-        return [(u, v, d) for u, v, d in self.graph.edges(data=True)
-                if d.get('confidence_score', 0) >= min_confidence]
-
     def get_graph_data(self) -> Dict[str, Any]:
-        """Export graph data formatted for frontend visualization."""
+        """
+        Export graph data formatted for frontend visualization.
+        SIMPLIFIED: No certificate styling - frontend handles all visual styling.
+        """
         nodes = []
         for node_id, attrs in self.graph.nodes(data=True):
-            node_data = {'id': node_id, 'label': node_id, 'type': attrs.get('type', 'unknown'),
+            node_data = {
-                         'attributes': attrs.get('attributes', {}),
+                'id': node_id, 
-                         'description': attrs.get('description', ''),
+                'label': node_id, 
-                         'metadata': attrs.get('metadata', {}),
+                'type': attrs.get('type', 'unknown'),
-                         'added_timestamp': attrs.get('added_timestamp')}
+                'attributes': attrs.get('attributes', []), # Raw attributes list
-            # Customize node appearance based on type and attributes
+                'description': attrs.get('description', ''),
-            node_type = node_data['type']
+                'metadata': attrs.get('metadata', {}),
-            attributes = node_data['attributes']
+                'added_timestamp': attrs.get('added_timestamp'),
-            if node_type == 'domain' and attributes.get('certificates', {}).get('has_valid_cert') is False:
+                'max_depth_reached': attrs.get('metadata', {}).get('max_depth_reached', False)
-                node_data['color'] = {'background': '#c7c7c7', 'border': '#999'} # Gray for invalid cert
+            }
             
             # Add incoming and outgoing edges to node data
             if self.graph.has_node(node_id):
-                node_data['incoming_edges'] = [{'from': u, 'data': d} for u, _, d in self.graph.in_edges(node_id, data=True)]
+                node_data['incoming_edges'] = [
-                node_data['outgoing_edges'] = [{'to': v, 'data': d} for _, v, d in self.graph.out_edges(node_id, data=True)]
+                    {'from': u, 'data': d} for u, _, d in self.graph.in_edges(node_id, data=True)
+                ]
+                node_data['outgoing_edges'] = [
+                    {'to': v, 'data': d} for _, v, d in self.graph.out_edges(node_id, data=True)
+                ]
             
             nodes.append(node_data)
 
         edges = []
         for source, target, attrs in self.graph.edges(data=True):
-            edges.append({'from': source, 'to': target,
+            edges.append({
-                          'label': attrs.get('relationship_type', ''),
+                'from': source, 
-                          'confidence_score': attrs.get('confidence_score', 0),
+                'to': target,
-                          'source_provider': attrs.get('source_provider', ''),
+                'label': attrs.get('relationship_type', ''),
-                          'discovery_timestamp': attrs.get('discovery_timestamp')})
+                'source_provider': attrs.get('source_provider', ''),
+                'discovery_timestamp': attrs.get('discovery_timestamp'),
+                'raw_data': attrs.get('raw_data', {})
+            })
+        
         return {
-            'nodes': nodes, 'edges': edges,
+            'nodes': nodes, 
+            'edges': edges,
             'statistics': self.get_statistics()['basic_metrics']
         }
 
-    def export_json(self) -> Dict[str, Any]:
-        """Export complete graph data as a JSON-serializable dictionary."""
-        graph_data = nx.node_link_data(self.graph) # Use NetworkX's built-in robust serializer
-        return {
-            'export_metadata': {
-                'export_timestamp': datetime.now(timezone.utc).isoformat(),
-                'graph_creation_time': self.creation_time,
-                'last_modified': self.last_modified,
-                'total_nodes': self.get_node_count(),
-                'total_edges': self.get_edge_count(),
-                'graph_format': 'dnsrecon_v1_nodeling'
-            },
-            'graph': graph_data,
-            'statistics': self.get_statistics()
-        }
-
-    def _get_confidence_distribution(self) -> Dict[str, int]:
-        """Get distribution of edge confidence scores."""
-        distribution = {'high': 0, 'medium': 0, 'low': 0}
-        for _, _, confidence in self.graph.edges(data='confidence_score', default=0):
-            if confidence >= 0.8: distribution['high'] += 1
-            elif confidence >= 0.6: distribution['medium'] += 1
-            else: distribution['low'] += 1
-        return distribution
-
     def get_statistics(self) -> Dict[str, Any]:
-        """Get comprehensive statistics about the graph."""
+        """Get comprehensive statistics about the graph with proper empty graph handling."""
-        stats = {'basic_metrics': {'total_nodes': self.get_node_count(),
+        
-                                   'total_edges': self.get_edge_count(),
+        # FIXED: Handle empty graph case properly
-                                   'creation_time': self.creation_time,
+        node_count = self.get_node_count()
-                                   'last_modified': self.last_modified},
+        edge_count = self.get_edge_count()
-                 'node_type_distribution': {}, 'relationship_type_distribution': {},
+        
-                 'confidence_distribution': self._get_confidence_distribution(),
+        stats = {
-                 'provider_distribution': {}}
+            'basic_metrics': {
-        # Calculate distributions
+                'total_nodes': node_count,
-        for node_type in NodeType:
+                'total_edges': edge_count,
-            stats['node_type_distribution'][node_type.value] = self.get_nodes_by_type(node_type).__len__()
+                'creation_time': self.creation_time,
-        for _, _, rel_type in self.graph.edges(data='relationship_type', default='unknown'):
+                'last_modified': self.last_modified
-            stats['relationship_type_distribution'][rel_type] = stats['relationship_type_distribution'].get(rel_type, 0) + 1
+            },
-        for _, _, provider in self.graph.edges(data='source_provider', default='unknown'):
+            'node_type_distribution': {}, 
-            stats['provider_distribution'][provider] = stats['provider_distribution'].get(provider, 0) + 1
+            'relationship_type_distribution': {},
+            'provider_distribution': {}
+        }
+        
+        # FIXED: Only calculate distributions if we have data
+        if node_count > 0:
+            # Calculate node type distributions
+            for node_type in NodeType:
+                count = len(self.get_nodes_by_type(node_type))
+                if count > 0:  # Only include types that exist
+                    stats['node_type_distribution'][node_type.value] = count
+        
+        if edge_count > 0:
+            # Calculate edge distributions
+            for _, _, data in self.graph.edges(data=True):
+                rel_type = data.get('relationship_type', 'unknown')
+                stats['relationship_type_distribution'][rel_type] = stats['relationship_type_distribution'].get(rel_type, 0) + 1
+                
+                provider = data.get('source_provider', 'unknown')
+                stats['provider_distribution'][provider] = stats['provider_distribution'].get(provider, 0) + 1
+        
         return stats
 
     def clear(self) -> None:
-        """Clear all nodes, edges, and indices from the graph."""
+        """Clear all nodes and edges from the graph."""
         self.graph.clear()
-        self.correlation_index.clear()
         self.creation_time = datetime.now(timezone.utc).isoformat()
         self.last_modified = self.creation_time

core/logger.py

@@ -1,4 +1,4 @@
-# dnsrecon/core/logger.py
+# DNScope/core/logger.py
 
 import logging
 import threading
@@ -30,7 +30,6 @@ class RelationshipDiscovery:
     source_node: str
     target_node: str
     relationship_type: str
-    confidence_score: float
     provider: str
     raw_data: Dict[str, Any]
     discovery_method: str
@@ -38,11 +37,11 @@ class RelationshipDiscovery:
 
 class ForensicLogger:
     """
-    Thread-safe forensic logging system for DNSRecon.
+    Thread-safe forensic logging system for DNScope.
     Maintains detailed audit trail of all reconnaissance activities.
     """
     
-    def __init__(self, session_id: str = None):
+    def __init__(self, session_id: str = ""):
         """
         Initialize forensic logger.
         
@@ -50,7 +49,7 @@ class ForensicLogger:
             session_id: Unique identifier for this reconnaissance session
         """
         self.session_id = session_id or self._generate_session_id()
-        #self.lock = threading.Lock()
+        self.lock = threading.Lock()
         
         # Initialize audit trail storage
         self.api_requests: List[APIRequest] = []
@@ -66,7 +65,7 @@ class ForensicLogger:
         }
         
         # Configure standard logger
-        self.logger = logging.getLogger(f'dnsrecon.{self.session_id}')
+        self.logger = logging.getLogger(f'DNScope.{self.session_id}')
         self.logger.setLevel(logging.INFO)
         
         # Create formatter for structured logging
@@ -86,13 +85,15 @@ class ForensicLogger:
         # Remove the unpickleable 'logger' attribute
         if 'logger' in state:
             del state['logger']
+        if 'lock' in state:
+            del state['lock']
         return state
 
     def __setstate__(self, state):
         """Restore ForensicLogger after unpickling by reconstructing logger."""
         self.__dict__.update(state)
         # Re-initialize the 'logger' attribute
-        self.logger = logging.getLogger(f'dnsrecon.{self.session_id}')
+        self.logger = logging.getLogger(f'DNScope.{self.session_id}')
         self.logger.setLevel(logging.INFO)
         formatter = logging.Formatter(
             '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
@@ -101,10 +102,11 @@ class ForensicLogger:
             console_handler = logging.StreamHandler()
             console_handler.setFormatter(formatter)
             self.logger.addHandler(console_handler)
+        self.lock = threading.Lock()
 
     def _generate_session_id(self) -> str:
         """Generate unique session identifier."""
-        return f"dnsrecon_{datetime.now(timezone.utc).strftime('%Y%m%d_%H%M%S')}"
+        return f"DNScope_{datetime.now(timezone.utc).strftime('%Y%m%d_%H%M%S')}"
     
     def log_api_request(self, provider: str, url: str, method: str = "GET", 
                        status_code: Optional[int] = None, 
@@ -149,12 +151,12 @@ class ForensicLogger:
         
         # Log to standard logger
         if error:
-            self.logger.error(f"API Request Failed - {provider}: {url} - {error}")
+            self.logger.error(f"API Request Failed.")
         else:
             self.logger.info(f"API Request - {provider}: {url} - Status: {status_code}")
     
     def log_relationship_discovery(self, source_node: str, target_node: str,
-                                 relationship_type: str, confidence_score: float,
+                                 relationship_type: str,
                                  provider: str, raw_data: Dict[str, Any],
                                  discovery_method: str) -> None:
         """
@@ -164,7 +166,6 @@ class ForensicLogger:
             source_node: Source node identifier
             target_node: Target node identifier
             relationship_type: Type of relationship (e.g., 'SAN', 'A_Record')
-            confidence_score: Confidence score (0.0 to 1.0)
             provider: Provider that discovered this relationship
             raw_data: Raw data from provider response
             discovery_method: Method used to discover relationship
@@ -174,7 +175,6 @@ class ForensicLogger:
             source_node=source_node,
             target_node=target_node,
             relationship_type=relationship_type,
-            confidence_score=confidence_score,
             provider=provider,
             raw_data=raw_data,
             discovery_method=discovery_method
@@ -185,7 +185,7 @@ class ForensicLogger:
         
         self.logger.info(
             f"Relationship Discovered - {source_node} -> {target_node} "
-            f"({relationship_type}) - Confidence: {confidence_score:.2f} - Provider: {provider}"
+            f"({relationship_type}) - Provider: {provider}"
         )
     
     def log_scan_start(self, target_domain: str, recursion_depth: int, 
@@ -199,12 +199,8 @@ class ForensicLogger:
     def log_scan_complete(self) -> None:
         """Log the completion of a reconnaissance scan."""
         self.session_metadata['end_time'] = datetime.now(timezone.utc).isoformat()
-        self.session_metadata['providers_used'] = list(self.session_metadata['providers_used'])
-        self.session_metadata['target_domains'] = list(self.session_metadata['target_domains'])
         
         self.logger.info(f"Scan Complete - Session: {self.session_id}")
-        self.logger.info(f"Total API Requests: {self.session_metadata['total_requests']}")
-        self.logger.info(f"Total Relationships: {self.session_metadata['total_relationships']}")
     
     def export_audit_trail(self) -> Dict[str, Any]:
         """
@@ -213,8 +209,12 @@ class ForensicLogger:
         Returns:
             Dictionary containing complete session audit trail
         """
+        session_metadata_export = self.session_metadata.copy()
+        session_metadata_export['providers_used'] = list(session_metadata_export['providers_used'])
+        session_metadata_export['target_domains'] = list(session_metadata_export['target_domains'])
+        
         return {
-            'session_metadata': self.session_metadata.copy(),
+            'session_metadata': session_metadata_export,
             'api_requests': [asdict(req) for req in self.api_requests],
             'relationships': [asdict(rel) for rel in self.relationships],
             'export_timestamp': datetime.now(timezone.utc).isoformat()
@@ -237,7 +237,6 @@ class ForensicLogger:
                 'successful_requests': len([req for req in provider_requests if req.error is None]),
                 'failed_requests': len([req for req in provider_requests if req.error is not None]),
                 'relationships_discovered': len(provider_relationships),
-                'avg_confidence': sum(rel.confidence_score for rel in provider_relationships) / len(provider_relationships) if provider_relationships else 0
             }
         
         return {

core/provider_result.py

@@ -0,0 +1,89 @@
+# DNScope-reduced/core/provider_result.py
+
+"""
+Unified data model for DNScope passive reconnaissance.
+Standardizes the data structure across all providers to ensure consistent processing.
+"""
+
+from typing import Any, Optional, List, Dict
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+
+
+@dataclass
+class StandardAttribute:
+    """A unified data structure for a single piece of information about a node."""
+    target_node: str
+    name: str
+    value: Any
+    type: str
+    provider: str
+    timestamp: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
+    metadata: Optional[Dict[str, Any]] = field(default_factory=dict)
+
+@dataclass
+class Relationship:
+    """A unified data structure for a directional link between two nodes."""
+    source_node: str
+    target_node: str
+    relationship_type: str
+    provider: str
+    timestamp: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
+    raw_data: Optional[Dict[str, Any]] = field(default_factory=dict)
+
+@dataclass
+class ProviderResult:
+    """A container for all data returned by a provider from a single query."""
+    attributes: List[StandardAttribute] = field(default_factory=list)
+    relationships: List[Relationship] = field(default_factory=list)
+
+    def add_attribute(self, target_node: str, name: str, value: Any, attr_type: str, 
+                     provider: str, metadata: Optional[Dict[str, Any]] = None) -> None:
+        """Helper method to add an attribute to the result."""
+        self.attributes.append(StandardAttribute(
+            target_node=target_node,
+            name=name,
+            value=value,
+            type=attr_type,
+            provider=provider,
+            metadata=metadata or {}
+        ))
+
+    def add_relationship(self, source_node: str, target_node: str, relationship_type: str,
+                        provider: str, raw_data: Optional[Dict[str, Any]] = None) -> None:
+        """Helper method to add a relationship to the result."""
+        self.relationships.append(Relationship(
+            source_node=source_node,
+            target_node=target_node,
+            relationship_type=relationship_type,
+            provider=provider,
+            raw_data=raw_data or {}
+        ))
+
+    def get_discovered_nodes(self) -> set:
+        """Get all unique node identifiers discovered in this result."""
+        nodes = set()
+        
+        # Add nodes from relationships
+        for rel in self.relationships:
+            nodes.add(rel.source_node)
+            nodes.add(rel.target_node)
+            
+        # Add nodes from attributes
+        for attr in self.attributes:
+            nodes.add(attr.target_node)
+            
+        return nodes
+
+    def get_relationship_count(self) -> int:
+        """Get the total number of relationships in this result."""
+        return len(self.relationships)
+
+    def get_attribute_count(self) -> int:
+        """Get the total number of attributes in this result."""
+        return len(self.attributes)
+
+    ##TODO
+    #def is_large_entity(self, threshold: int) -> bool:
+    #    """Check if this result qualifies as a large entity based on relationship count."""
+    #    return self.get_relationship_count() > threshold

core/rate_limiter.py

@@ -0,0 +1,145 @@
+# DNScope-reduced/core/rate_limiter.py
+
+import time
+import logging
+
+class GlobalRateLimiter:
+    """
+    FIXED: Improved rate limiter with better cleanup and error handling.
+    Prevents accumulation of stale entries that cause infinite retry loops.
+    """
+    
+    def __init__(self, redis_client):
+        self.redis = redis_client
+        self.logger = logging.getLogger('DNScope.rate_limiter')
+        # Track last cleanup times to avoid excessive Redis operations
+        self._last_cleanup = {}
+
+    def is_rate_limited(self, key, limit, period):
+        """
+        FIXED: Check if a key is rate-limited with improved cleanup and error handling.
+        
+        Args:
+            key: Rate limit key (e.g., provider name)
+            limit: Maximum requests allowed
+            period: Time period in seconds (60 for per-minute)
+            
+        Returns:
+            bool: True if rate limited, False otherwise
+        """
+        if limit <= 0:
+            # Rate limit of 0 or negative means no limiting
+            return False
+            
+        now = time.time()
+        rate_key = f"rate_limit:{key}"
+        
+        try:
+            # FIXED: More aggressive cleanup to prevent accumulation
+            # Only clean up if we haven't cleaned recently (every 10 seconds max)
+            should_cleanup = (
+                rate_key not in self._last_cleanup or 
+                now - self._last_cleanup.get(rate_key, 0) > 10
+            )
+            
+            if should_cleanup:
+                # Remove entries older than the period
+                removed_count = self.redis.zremrangebyscore(rate_key, 0, now - period)
+                self._last_cleanup[rate_key] = now
+                
+                if removed_count > 0:
+                    self.logger.debug(f"Rate limiter cleaned up {removed_count} old entries for {key}")
+            
+            # Get current count
+            current_count = self.redis.zcard(rate_key)
+            
+            if current_count >= limit:
+                self.logger.debug(f"Rate limited: {key} has {current_count}/{limit} requests in period")
+                return True
+            
+            # Add new timestamp with error handling
+            try:
+                # Use pipeline for atomic operations
+                pipe = self.redis.pipeline()
+                pipe.zadd(rate_key, {str(now): now})
+                pipe.expire(rate_key, int(period * 2))  # Set TTL to 2x period for safety
+                pipe.execute()
+            except Exception as e:
+                self.logger.warning(f"Failed to record rate limit entry for {key}: {e}")
+                # Don't block the request if we can't record it
+                return False
+            
+            return False
+            
+        except Exception as e:
+            self.logger.error(f"Rate limiter error for {key}: {e}")
+            # FIXED: On Redis errors, don't block requests to avoid infinite loops
+            return False
+    
+    def get_rate_limit_status(self, key, limit, period):
+        """
+        Get detailed rate limit status for debugging.
+        
+        Returns:
+            dict: Status information including current count, limit, and time to reset
+        """
+        now = time.time()
+        rate_key = f"rate_limit:{key}"
+        
+        try:
+            current_count = self.redis.zcard(rate_key)
+            
+            # Get oldest entry to calculate reset time
+            oldest_entries = self.redis.zrange(rate_key, 0, 0, withscores=True)
+            time_to_reset = 0
+            if oldest_entries:
+                oldest_time = oldest_entries[0][1]
+                time_to_reset = max(0, period - (now - oldest_time))
+            
+            return {
+                'key': key,
+                'current_count': current_count,
+                'limit': limit,
+                'period': period,
+                'is_limited': current_count >= limit,
+                'time_to_reset': time_to_reset
+            }
+        except Exception as e:
+            self.logger.error(f"Failed to get rate limit status for {key}: {e}")
+            return {
+                'key': key,
+                'current_count': 0,
+                'limit': limit,
+                'period': period,
+                'is_limited': False,
+                'time_to_reset': 0,
+                'error': str(e)
+            }
+    
+    def reset_rate_limit(self, key):
+        """
+        ADDED: Reset rate limit for a specific key (useful for debugging).
+        """
+        rate_key = f"rate_limit:{key}"
+        try:
+            deleted = self.redis.delete(rate_key)
+            self.logger.info(f"Reset rate limit for {key} (deleted: {deleted})")
+            return True
+        except Exception as e:
+            self.logger.error(f"Failed to reset rate limit for {key}: {e}")
+            return False
+    
+    def cleanup_all_rate_limits(self):
+        """
+        ADDED: Clean up all rate limit entries (useful for maintenance).
+        """
+        try:
+            keys = self.redis.keys("rate_limit:*")
+            if keys:
+                deleted = self.redis.delete(*keys)
+                self.logger.info(f"Cleaned up {deleted} rate limit keys")
+                return deleted
+            return 0
+        except Exception as e:
+            self.logger.error(f"Failed to cleanup rate limits: {e}")
+            return 0

core/session_config.py

@@ -1,372 +1,20 @@
 """
-Enhanced per-session configuration management for DNSRecon.
+Per-session configuration management for DNScope.
-Provides isolated configuration instances for each user session while supporting global caching.
+Provides isolated configuration instances for each user session.
 """
 
-import os
+from config import Config
-from typing import Dict, Optional
 
-
+class SessionConfig(Config):
-class SessionConfig:
     """
-    Enhanced session-specific configuration that inherits from global config
+    Session-specific configuration that inherits from global config
-    but maintains isolated API keys and provider settings while supporting global caching.
+    but maintains isolated API keys and provider settings.
     """
     
     def __init__(self):
-        """Initialize enhanced session config with global cache support."""
+        """Initialize session config with global defaults."""
-        # Copy all attributes from global config
+        super().__init__()
-        self.api_keys: Dict[str, Optional[str]] = {
-            'shodan': None
-        }
-        
-        # Default settings (copied from global config)
-        self.default_recursion_depth = 2
-        self.default_timeout = 30
-        self.max_concurrent_requests = 5
-        self.large_entity_threshold = 100
-        
-        # Enhanced rate limiting settings (per session)
-        self.rate_limits = {
-            'crtsh': 60,
-            'shodan': 60,
-            'dns': 100
-        }
-        
-        # Enhanced provider settings (per session)
-        self.enabled_providers = {
-            'crtsh': True,
-            'dns': True,
-            'shodan': False
-        }
-        
-        # Task-based execution settings
-        self.task_retry_settings = {
-            'max_retries': 3,
-            'base_backoff_seconds': 1.0,
-            'max_backoff_seconds': 60.0,
-            'retry_on_rate_limit': True,
-            'retry_on_connection_error': True,
-            'retry_on_timeout': True
-        }
-        
-        # Cache settings (global across all sessions)
-        self.cache_settings = {
-            'enabled': True,
-            'expiry_hours': 12,
-            'cache_base_dir': '.cache',
-            'per_provider_directories': True,
-            'thread_safe_operations': True
-        }
-        
-        # Logging configuration
-        self.log_level = 'INFO'
-        self.log_format = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
-        
-        # Flask configuration (shared)
-        self.flask_host = '127.0.0.1'
-        self.flask_port = 5000
-        self.flask_debug = True
-        
-        # Session isolation settings
-        self.session_isolation = {
-            'enforce_single_session_per_user': True,
-            'consolidate_session_data_on_replacement': True,
-            'user_fingerprinting_enabled': True,
-            'session_timeout_minutes': 60
-        }
-        
-        # Circuit breaker settings for provider reliability
-        self.circuit_breaker = {
-            'enabled': True,
-            'failure_threshold': 5,  # Failures before opening circuit
-            'recovery_timeout_seconds': 300,  # 5 minutes before trying again
-            'half_open_max_calls': 3  # Test calls when recovering
-        }
-    
-    def set_api_key(self, provider: str, api_key: str) -> bool:
-        """
-        Set API key for a provider in this session.
-        
-        Args:
-            provider: Provider name (shodan, etc)
-            api_key: API key string (empty string to clear)
-            
-        Returns:
-            bool: True if key was set successfully
-        """
-        if provider in self.api_keys:
-            # Handle clearing of API keys
-            if api_key and api_key.strip():
-                self.api_keys[provider] = api_key.strip()
-                self.enabled_providers[provider] = True
-            else:
-                self.api_keys[provider] = None
-                self.enabled_providers[provider] = False
-            return True
-        return False
-    
-    def get_api_key(self, provider: str) -> Optional[str]:
-        """
-        Get API key for a provider in this session.
-        
-        Args:
-            provider: Provider name
-            
-        Returns:
-            API key or None if not set
-        """
-        return self.api_keys.get(provider)
-    
-    def is_provider_enabled(self, provider: str) -> bool:
-        """
-        Check if a provider is enabled in this session.
-        
-        Args:
-            provider: Provider name
-            
-        Returns:
-            bool: True if provider is enabled
-        """
-        return self.enabled_providers.get(provider, False)
-    
-    def get_rate_limit(self, provider: str) -> int:
-        """
-        Get rate limit for a provider in this session.
-        
-        Args:
-            provider: Provider name
-            
-        Returns:
-            Rate limit in requests per minute
-        """
-        return self.rate_limits.get(provider, 60)
-    
-    def get_task_retry_config(self) -> Dict[str, any]:
-        """
-        Get task retry configuration for this session.
-        
-        Returns:
-            Dictionary with retry settings
-        """
-        return self.task_retry_settings.copy()
-    
-    def get_cache_config(self) -> Dict[str, any]:
-        """
-        Get cache configuration (global settings).
-        
-        Returns:
-            Dictionary with cache settings
-        """
-        return self.cache_settings.copy()
-    
-    def is_circuit_breaker_enabled(self) -> bool:
-        """Check if circuit breaker is enabled for provider reliability."""
-        return self.circuit_breaker.get('enabled', True)
-    
-    def get_circuit_breaker_config(self) -> Dict[str, any]:
-        """Get circuit breaker configuration."""
-        return self.circuit_breaker.copy()
-    
-    def update_provider_settings(self, provider_updates: Dict[str, Dict[str, any]]) -> bool:
-        """
-        Update provider-specific settings in bulk.
-        
-        Args:
-            provider_updates: Dictionary of provider -> settings updates
-            
-        Returns:
-            bool: True if updates were applied successfully
-        """
-        try:
-            for provider_name, updates in provider_updates.items():
-                # Update rate limits
-                if 'rate_limit' in updates:
-                    self.rate_limits[provider_name] = updates['rate_limit']
-                
-                # Update enabled status
-                if 'enabled' in updates:
-                    self.enabled_providers[provider_name] = updates['enabled']
-                
-                # Update API key
-                if 'api_key' in updates:
-                    self.set_api_key(provider_name, updates['api_key'])
-            
-            return True
-        except Exception as e:
-            print(f"Error updating provider settings: {e}")
-            return False
-    
-    def validate_configuration(self) -> Dict[str, any]:
-        """
-        Validate the current configuration and return validation results.
-        
-        Returns:
-            Dictionary with validation results and any issues found
-        """
-        validation_result = {
-            'valid': True,
-            'warnings': [],
-            'errors': [],
-            'provider_status': {}
-        }
-        
-        # Validate provider configurations
-        for provider_name, enabled in self.enabled_providers.items():
-            provider_status = {
-                'enabled': enabled,
-                'has_api_key': bool(self.api_keys.get(provider_name)),
-                'rate_limit': self.rate_limits.get(provider_name, 60)
-            }
-            
-            # Check for potential issues
-            if enabled and provider_name in ['shodan'] and not provider_status['has_api_key']:
-                validation_result['warnings'].append(
-                    f"Provider '{provider_name}' is enabled but missing API key"
-                )
-            
-            validation_result['provider_status'][provider_name] = provider_status
-        
-        # Validate task settings
-        if self.task_retry_settings['max_retries'] > 10:
-            validation_result['warnings'].append(
-                f"High retry count ({self.task_retry_settings['max_retries']}) may cause long delays"
-            )
-        
-        # Validate concurrent settings
-        if self.max_concurrent_requests > 10:
-            validation_result['warnings'].append(
-                f"High concurrency ({self.max_concurrent_requests}) may overwhelm providers"
-            )
-        
-        # Validate cache settings
-        if not os.path.exists(self.cache_settings['cache_base_dir']):
-            try:
-                os.makedirs(self.cache_settings['cache_base_dir'], exist_ok=True)
-            except Exception as e:
-                validation_result['errors'].append(f"Cannot create cache directory: {e}")
-                validation_result['valid'] = False
-        
-        return validation_result
-    
-    def load_from_env(self):
-        """Load configuration from environment variables with enhanced validation."""
-        # Load API keys from environment
-        if os.getenv('SHODAN_API_KEY') and not self.api_keys['shodan']:
-            self.set_api_key('shodan', os.getenv('SHODAN_API_KEY'))
-            print("Loaded Shodan API key from environment")
-        
-        # Override default settings from environment
-        self.default_recursion_depth = int(os.getenv('DEFAULT_RECURSION_DEPTH', '2'))
-        self.default_timeout = int(os.getenv('DEFAULT_TIMEOUT', '30'))
-        self.max_concurrent_requests = int(os.getenv('MAX_CONCURRENT_REQUESTS', '5'))
-        
-        # Load task retry settings from environment
-        if os.getenv('TASK_MAX_RETRIES'):
-            self.task_retry_settings['max_retries'] = int(os.getenv('TASK_MAX_RETRIES'))
-        
-        if os.getenv('TASK_BASE_BACKOFF'):
-            self.task_retry_settings['base_backoff_seconds'] = float(os.getenv('TASK_BASE_BACKOFF'))
-        
-        # Load cache settings from environment
-        if os.getenv('CACHE_EXPIRY_HOURS'):
-            self.cache_settings['expiry_hours'] = int(os.getenv('CACHE_EXPIRY_HOURS'))
-        
-        if os.getenv('CACHE_DISABLED'):
-            self.cache_settings['enabled'] = os.getenv('CACHE_DISABLED').lower() != 'true'
-        
-        # Load circuit breaker settings
-        if os.getenv('CIRCUIT_BREAKER_DISABLED'):
-            self.circuit_breaker['enabled'] = os.getenv('CIRCUIT_BREAKER_DISABLED').lower() != 'true'
-        
-        # Flask settings
-        self.flask_debug = os.getenv('FLASK_DEBUG', 'True').lower() == 'true'
-        
-        print("Enhanced configuration loaded from environment")
-    
-    def export_config_summary(self) -> Dict[str, any]:
-        """
-        Export a summary of the current configuration for debugging/logging.
-        
-        Returns:
-            Dictionary with configuration summary (API keys redacted)
-        """
-        return {
-            'providers': {
-                provider: {
-                    'enabled': self.enabled_providers.get(provider, False),
-                    'has_api_key': bool(self.api_keys.get(provider)),
-                    'rate_limit': self.rate_limits.get(provider, 60)
-                }
-                for provider in self.enabled_providers.keys()
-            },
-            'task_settings': {
-                'max_retries': self.task_retry_settings['max_retries'],
-                'max_concurrent_requests': self.max_concurrent_requests,
-                'large_entity_threshold': self.large_entity_threshold
-            },
-            'cache_settings': {
-                'enabled': self.cache_settings['enabled'],
-                'expiry_hours': self.cache_settings['expiry_hours'],
-                'base_directory': self.cache_settings['cache_base_dir']
-            },
-            'session_settings': {
-                'isolation_enabled': self.session_isolation['enforce_single_session_per_user'],
-                'consolidation_enabled': self.session_isolation['consolidate_session_data_on_replacement'],
-                'timeout_minutes': self.session_isolation['session_timeout_minutes']
-            },
-            'circuit_breaker': {
-                'enabled': self.circuit_breaker['enabled'],
-                'failure_threshold': self.circuit_breaker['failure_threshold'],
-                'recovery_timeout': self.circuit_breaker['recovery_timeout_seconds']
-            }
-        }
 
-
+def create_session_config() -> 'SessionConfig':
-def create_session_config() -> SessionConfig:
+    """Create a new session configuration instance."""
-    """
+    return SessionConfig()
-    Create a new enhanced session configuration instance.
-    
-    Returns:
-        Configured SessionConfig instance
-    """
-    session_config = SessionConfig()
-    session_config.load_from_env()
-    
-    # Validate configuration and log any issues
-    validation = session_config.validate_configuration()
-    if validation['warnings']:
-        print("Configuration warnings:")
-        for warning in validation['warnings']:
-            print(f"  WARNING: {warning}")
-    
-    if validation['errors']:
-        print("Configuration errors:")
-        for error in validation['errors']:
-            print(f"  ERROR: {error}")
-    
-    if not validation['valid']:
-        raise ValueError("Configuration validation failed - see errors above")
-    
-    print(f"Enhanced session configuration created successfully")
-    return session_config
-
-
-def create_test_config() -> SessionConfig:
-    """
-    Create a test configuration with safe defaults for testing.
-    
-    Returns:
-        Test-safe SessionConfig instance
-    """
-    test_config = SessionConfig()
-    
-    # Override settings for testing
-    test_config.max_concurrent_requests = 2
-    test_config.task_retry_settings['max_retries'] = 1
-    test_config.task_retry_settings['base_backoff_seconds'] = 0.1
-    test_config.cache_settings['expiry_hours'] = 1
-    test_config.session_isolation['session_timeout_minutes'] = 10
-    
-    print("Test configuration created")
-    return test_config

core/session_manager.py

@@ -1,158 +1,46 @@
-# dnsrecon/core/session_manager.py
+# DNScope/core/session_manager.py
 
 import threading
 import time
 import uuid
 import redis
 import pickle
-import hashlib
+from typing import Dict, Optional, Any
-from typing import Dict, Optional, Any, List, Tuple
 
 from core.scanner import Scanner
-
+from config import config
-
-class UserIdentifier:
-    """Handles user identification for session management."""
-    
-    @staticmethod
-    def generate_user_fingerprint(client_ip: str, user_agent: str) -> str:
-        """
-        Generate a unique fingerprint for a user based on IP and User-Agent.
-        
-        Args:
-            client_ip: Client IP address
-            user_agent: User-Agent header value
-            
-        Returns:
-            Unique user fingerprint hash
-        """
-        # Create deterministic user identifier
-        user_data = f"{client_ip}:{user_agent[:100]}"  # Limit UA to 100 chars
-        fingerprint = hashlib.sha256(user_data.encode()).hexdigest()[:16]  # 16 char fingerprint
-        return f"user_{fingerprint}"
-    
-    @staticmethod
-    def extract_request_info(request) -> Tuple[str, str]:
-        """
-        Extract client IP and User-Agent from Flask request.
-        
-        Args:
-            request: Flask request object
-            
-        Returns:
-            Tuple of (client_ip, user_agent)
-        """
-        # Handle proxy headers for real IP
-        client_ip = request.headers.get('X-Forwarded-For', '').split(',')[0].strip()
-        if not client_ip:
-            client_ip = request.headers.get('X-Real-IP', '')
-        if not client_ip:
-            client_ip = request.remote_addr or 'unknown'
-        
-        user_agent = request.headers.get('User-Agent', 'unknown')
-        
-        return client_ip, user_agent
-
-
-class SessionConsolidator:
-    """Handles consolidation of session data when replacing sessions."""
-    
-    @staticmethod
-    def consolidate_scanner_data(old_scanner: 'Scanner', new_scanner: 'Scanner') -> 'Scanner':
-        """
-        Consolidate useful data from old scanner into new scanner.
-        
-        Args:
-            old_scanner: Scanner from terminated session
-            new_scanner: New scanner instance
-            
-        Returns:
-            Enhanced new scanner with consolidated data
-        """
-        try:
-            # Consolidate graph data if old scanner has valuable data
-            if old_scanner and hasattr(old_scanner, 'graph') and old_scanner.graph:
-                old_stats = old_scanner.graph.get_statistics()
-                if old_stats['basic_metrics']['total_nodes'] > 0:
-                    print(f"Consolidating graph data: {old_stats['basic_metrics']['total_nodes']} nodes, {old_stats['basic_metrics']['total_edges']} edges")
-                    
-                    # Transfer nodes and edges to new scanner's graph
-                    for node_id, node_data in old_scanner.graph.graph.nodes(data=True):
-                        # Add node to new graph with all attributes
-                        new_scanner.graph.graph.add_node(node_id, **node_data)
-                    
-                    for source, target, edge_data in old_scanner.graph.graph.edges(data=True):
-                        # Add edge to new graph with all attributes  
-                        new_scanner.graph.graph.add_edge(source, target, **edge_data)
-                    
-                    # Update correlation index
-                    if hasattr(old_scanner.graph, 'correlation_index'):
-                        new_scanner.graph.correlation_index = old_scanner.graph.correlation_index.copy()
-                    
-                    # Update timestamps
-                    new_scanner.graph.creation_time = old_scanner.graph.creation_time
-                    new_scanner.graph.last_modified = old_scanner.graph.last_modified
-            
-            # Consolidate provider statistics
-            if old_scanner and hasattr(old_scanner, 'providers') and old_scanner.providers:
-                for old_provider in old_scanner.providers:
-                    # Find matching provider in new scanner
-                    matching_new_provider = None
-                    for new_provider in new_scanner.providers:
-                        if new_provider.get_name() == old_provider.get_name():
-                            matching_new_provider = new_provider
-                            break
-                    
-                    if matching_new_provider:
-                        # Transfer cumulative statistics
-                        matching_new_provider.total_requests += old_provider.total_requests
-                        matching_new_provider.successful_requests += old_provider.successful_requests
-                        matching_new_provider.failed_requests += old_provider.failed_requests
-                        matching_new_provider.total_relationships_found += old_provider.total_relationships_found
-                        
-                        # Transfer cache statistics if available
-                        if hasattr(old_provider, 'cache_hits'):
-                            matching_new_provider.cache_hits += getattr(old_provider, 'cache_hits', 0)
-                            matching_new_provider.cache_misses += getattr(old_provider, 'cache_misses', 0)
-                            
-                        print(f"Consolidated {old_provider.get_name()} provider stats: {old_provider.total_requests} requests")
-            
-            return new_scanner
-            
-        except Exception as e:
-            print(f"Warning: Error during session consolidation: {e}")
-            return new_scanner
-
 
 class SessionManager:
     """
-    Manages single scanner session per user using Redis with user identification.
+    FIXED: Manages multiple scanner instances for concurrent user sessions using Redis.
-    Enforces one active session per user for consistent state management.
+    Now more conservative about session creation to preserve API keys and configuration.
     """
     
-    def __init__(self, session_timeout_minutes: int = 60):
+    def __init__(self, session_timeout_minutes: int = 0):
         """
-        Initialize session manager with Redis backend and user tracking.
+        Initialize session manager with a Redis backend.
         """
+        if session_timeout_minutes is None:
+            session_timeout_minutes = config.session_timeout_minutes
+
         self.redis_client = redis.StrictRedis(db=0, decode_responses=False)
         self.session_timeout = session_timeout_minutes * 60  # Convert to seconds
         self.lock = threading.Lock()
         
-        # User identification helper
+        # FIXED: Add a creation lock to prevent race conditions
-        self.user_identifier = UserIdentifier()
+        self.creation_lock = threading.Lock()
-        self.consolidator = SessionConsolidator()
         
         # Start cleanup thread
         self.cleanup_thread = threading.Thread(target=self._cleanup_loop, daemon=True)
         self.cleanup_thread.start()
         
-        print(f"SessionManager initialized with Redis backend, user tracking, and {session_timeout_minutes}min timeout")
+        print(f"SessionManager initialized with Redis backend and {session_timeout_minutes}min timeout")
     
     def __getstate__(self):
         """Prepare SessionManager for pickling."""
         state = self.__dict__.copy()
-        # Exclude unpickleable attributes
+        # Exclude unpickleable attributes - Redis client and threading objects
-        unpicklable_attrs = ['lock', 'cleanup_thread', 'redis_client']
+        unpicklable_attrs = ['lock', 'cleanup_thread', 'redis_client', 'creation_lock']
         for attr in unpicklable_attrs:
             if attr in state:
                 del state[attr]
@@ -162,115 +50,77 @@ class SessionManager:
         """Restore SessionManager after unpickling."""
         self.__dict__.update(state)
         # Re-initialize unpickleable attributes
-        import redis
         self.redis_client = redis.StrictRedis(db=0, decode_responses=False)
         self.lock = threading.Lock()
+        self.creation_lock = threading.Lock()
         self.cleanup_thread = threading.Thread(target=self._cleanup_loop, daemon=True)
         self.cleanup_thread.start()
 
     def _get_session_key(self, session_id: str) -> str:
-        """Generate Redis key for a session."""
+        """Generates the Redis key for a session."""
-        return f"dnsrecon:session:{session_id}"
+        return f"DNScope:session:{session_id}"
-
-    def _get_user_session_key(self, user_fingerprint: str) -> str:
-        """Generate Redis key for user -> session mapping."""
-        return f"dnsrecon:user:{user_fingerprint}"
 
     def _get_stop_signal_key(self, session_id: str) -> str:
-        """Generate Redis key for session stop signal."""
+        """Generates the Redis key for a session's stop signal."""
-        return f"dnsrecon:stop:{session_id}"
+        return f"DNScope:stop:{session_id}"
 
-    def create_or_replace_user_session(self, client_ip: str, user_agent: str) -> str:
+    def create_session(self) -> str:
         """
-        Create new session for user, replacing any existing session.
+        FIXED: Create a new user session with thread-safe creation to prevent duplicates.
-        Consolidates data from previous session if it exists.
-        
-        Args:
-            client_ip: Client IP address
-            user_agent: User-Agent header
-            
-        Returns:
-            New session ID
         """
-        user_fingerprint = self.user_identifier.generate_user_fingerprint(client_ip, user_agent)
+        # FIXED: Use creation lock to prevent race conditions
-        new_session_id = str(uuid.uuid4())
+        with self.creation_lock:
-        
+            session_id = str(uuid.uuid4())
-        print(f"=== CREATING/REPLACING SESSION FOR USER {user_fingerprint} ===")
+            print(f"=== CREATING SESSION {session_id} IN REDIS ===")
-        
-        try:
-            # Check for existing user session
-            existing_session_id = self._get_user_current_session(user_fingerprint)
-            old_scanner = None
             
-            if existing_session_id:
+            try:
-                print(f"Found existing session {existing_session_id} for user {user_fingerprint}")
+                from core.session_config import create_session_config
-                # Get old scanner data for consolidation
+                session_config = create_session_config()
-                old_scanner = self.get_session(existing_session_id)
+                scanner_instance = Scanner(session_config=session_config)
-                # Terminate old session
+                
-                self._terminate_session_internal(existing_session_id, cleanup_user_mapping=False)
+                # Set the session ID on the scanner for cross-process stop signal management
-                print(f"Terminated old session {existing_session_id}")
+                scanner_instance.session_id = session_id
-            
+                
-            # Create new session config and scanner
+                session_data = {
-            from core.session_config import create_session_config
+                    'scanner': scanner_instance,
-            session_config = create_session_config()
+                    'config': session_config,
-            new_scanner = Scanner(session_config=session_config)
+                    'created_at': time.time(),
-            
+                    'last_activity': time.time(),
-            # Set session ID on scanner for cross-process operations
+                    'status': 'active'
-            new_scanner.session_id = new_session_id
+                }
-            
+                
-            # Consolidate data from old session if available
+                # Serialize the entire session data dictionary using pickle
-            if old_scanner:
+                serialized_data = pickle.dumps(session_data)
-                new_scanner = self.consolidator.consolidate_scanner_data(old_scanner, new_scanner)
+                
-                print(f"Consolidated data from previous session")
+                # Store in Redis
-            
+                session_key = self._get_session_key(session_id)
-            # Create session data
+                self.redis_client.setex(session_key, self.session_timeout, serialized_data)
-            session_data = {
+                
-                'scanner': new_scanner,
+                # Initialize stop signal as False
-                'config': session_config,
+                stop_key = self._get_stop_signal_key(session_id)
-                'created_at': time.time(),
+                self.redis_client.setex(stop_key, self.session_timeout, b'0')
-                'last_activity': time.time(),
+                
-                'status': 'active',
+                print(f"Session {session_id} stored in Redis with stop signal initialized")
-                'user_fingerprint': user_fingerprint,
+                print(f"Session has {len(scanner_instance.providers)} providers: {[p.get_name() for p in scanner_instance.providers]}")
-                'client_ip': client_ip,
+                return session_id
-                'user_agent': user_agent[:200]  # Truncate for storage
+                
-            }
+            except Exception as e:
-            
+                print(f"ERROR: Failed to create session {session_id}: {e}")
-            # Store session in Redis
+                raise
-            session_key = self._get_session_key(new_session_id)
-            serialized_data = pickle.dumps(session_data)
-            self.redis_client.setex(session_key, self.session_timeout, serialized_data)
-            
-            # Update user -> session mapping
-            user_session_key = self._get_user_session_key(user_fingerprint)
-            self.redis_client.setex(user_session_key, self.session_timeout, new_session_id.encode('utf-8'))
-            
-            # Initialize stop signal
-            stop_key = self._get_stop_signal_key(new_session_id)
-            self.redis_client.setex(stop_key, self.session_timeout, b'0')
-            
-            print(f"Created new session {new_session_id} for user {user_fingerprint}")
-            return new_session_id
-            
-        except Exception as e:
-            print(f"ERROR: Failed to create session for user {user_fingerprint}: {e}")
-            raise
-
-    def _get_user_current_session(self, user_fingerprint: str) -> Optional[str]:
-        """Get current session ID for a user."""
-        try:
-            user_session_key = self._get_user_session_key(user_fingerprint)
-            session_id_bytes = self.redis_client.get(user_session_key)
-            if session_id_bytes:
-                return session_id_bytes.decode('utf-8')
-            return None
-        except Exception as e:
-            print(f"Error getting user session: {e}")
-            return None
 
     def set_stop_signal(self, session_id: str) -> bool:
-        """Set stop signal for session (cross-process safe)."""
+        """
+        Set the stop signal for a session (cross-process safe).
+        
+        Args:
+            session_id: Session identifier
+            
+        Returns:
+            bool: True if signal was set successfully
+        """
         try:
             stop_key = self._get_stop_signal_key(session_id)
+            # Set stop signal to '1' with the same TTL as the session
             self.redis_client.setex(stop_key, self.session_timeout, b'1')
             print(f"Stop signal set for session {session_id}")
             return True
@@ -279,7 +129,15 @@ class SessionManager:
             return False
 
     def is_stop_requested(self, session_id: str) -> bool:
-        """Check if stop is requested for session (cross-process safe)."""
+        """
+        Check if stop is requested for a session (cross-process safe).
+        
+        Args:
+            session_id: Session identifier
+            
+        Returns:
+            bool: True if stop is requested
+        """
         try:
             stop_key = self._get_stop_signal_key(session_id)
             value = self.redis_client.get(stop_key)
@@ -289,7 +147,15 @@ class SessionManager:
             return False
 
     def clear_stop_signal(self, session_id: str) -> bool:
-        """Clear stop signal for session."""
+        """
+        Clear the stop signal for a session.
+        
+        Args:
+            session_id: Session identifier
+            
+        Returns:
+            bool: True if signal was cleared successfully
+        """
         try:
             stop_key = self._get_stop_signal_key(session_id)
             self.redis_client.setex(stop_key, self.session_timeout, b'0')
@@ -300,13 +166,13 @@ class SessionManager:
             return False
 
     def _get_session_data(self, session_id: str) -> Optional[Dict[str, Any]]:
-        """Retrieve and deserialize session data from Redis."""
+        """Retrieves and deserializes session data from Redis."""
         try:
             session_key = self._get_session_key(session_id)
             serialized_data = self.redis_client.get(session_key)
             if serialized_data:
                 session_data = pickle.loads(serialized_data)
-                # Ensure scanner has correct session ID
+                # Ensure the scanner has the correct session ID for stop signal checking
                 if 'scanner' in session_data and session_data['scanner']:
                     session_data['scanner'].session_id = session_id
                 return session_data
@@ -316,35 +182,47 @@ class SessionManager:
             return None
 
     def _save_session_data(self, session_id: str, session_data: Dict[str, Any]) -> bool:
-        """Serialize and save session data to Redis with updated TTL."""
+        """
+        Serializes and saves session data back to Redis with updated TTL.
+        
+        Returns:
+            bool: True if save was successful
+        """
         try:
             session_key = self._get_session_key(session_id)
             serialized_data = pickle.dumps(session_data)
             result = self.redis_client.setex(session_key, self.session_timeout, serialized_data)
-            
-            # Also refresh user mapping TTL if available
-            if 'user_fingerprint' in session_data:
-                user_session_key = self._get_user_session_key(session_data['user_fingerprint'])
-                self.redis_client.setex(user_session_key, self.session_timeout, session_id.encode('utf-8'))
-                
             return result
         except Exception as e:
             print(f"ERROR: Failed to save session data for {session_id}: {e}")
             return False
 
     def update_session_scanner(self, session_id: str, scanner: 'Scanner') -> bool:
-        """Update scanner object in session with immediate persistence."""
+        """
+        Updates just the scanner object in a session with immediate persistence.
+        
+        Returns:
+            bool: True if update was successful
+        """
         try:
             session_data = self._get_session_data(session_id)
             if session_data:
-                # Ensure scanner has session ID
+                # Ensure scanner has the session ID
                 scanner.session_id = session_id
                 session_data['scanner'] = scanner
                 session_data['last_activity'] = time.time()
                 
+                # Immediately save to Redis for GUI updates
                 success = self._save_session_data(session_id, session_data)
                 if success:
-                    print(f"Scanner state updated for session {session_id} (status: {scanner.status})")
+                    # Only log occasionally to reduce noise
+                    if hasattr(self, '_last_update_log'):
+                        if time.time() - self._last_update_log > 5:  # Log every 5 seconds max
+                            #print(f"Scanner state updated for session {session_id} (status: {scanner.status})")
+                            self._last_update_log = time.time()
+                    else:
+                        #print(f"Scanner state updated for session {session_id} (status: {scanner.status})")
+                        self._last_update_log = time.time()
                 else:
                     print(f"WARNING: Failed to save scanner state for session {session_id}")
                 return success
@@ -356,7 +234,16 @@ class SessionManager:
             return False
 
     def update_scanner_status(self, session_id: str, status: str) -> bool:
-        """Quickly update scanner status for immediate GUI feedback."""
+        """
+        Quickly update just the scanner status for immediate GUI feedback.
+        
+        Args:
+            session_id: Session identifier
+            status: New scanner status
+            
+        Returns:
+            bool: True if update was successful
+        """
         try:
             session_data = self._get_session_data(session_id)
             if session_data and 'scanner' in session_data:
@@ -375,7 +262,9 @@ class SessionManager:
             return False
 
     def get_session(self, session_id: str) -> Optional[Scanner]:
-        """Get scanner instance for session with session ID management."""
+        """
+        Get scanner instance for a session from Redis with session ID management.
+        """
         if not session_id:
             return None
             
@@ -390,13 +279,21 @@ class SessionManager:
         
         scanner = session_data.get('scanner')
         if scanner:
-            # Ensure scanner can check Redis-based stop signal
+            # Ensure the scanner can check the Redis-based stop signal
             scanner.session_id = session_id
         
         return scanner
 
     def get_session_status_only(self, session_id: str) -> Optional[str]:
-        """Get scanner status without full session retrieval (for performance)."""
+        """
+        Get just the scanner status without full session retrieval (for performance).
+        
+        Args:
+            session_id: Session identifier
+            
+        Returns:
+            Scanner status string or None if not found
+        """
         try:
             session_data = self._get_session_data(session_id)
             if session_data and 'scanner' in session_data:
@@ -407,18 +304,16 @@ class SessionManager:
             return None
 
     def terminate_session(self, session_id: str) -> bool:
-        """Terminate specific session with reliable stop signal and immediate status update."""
+        """
-        return self._terminate_session_internal(session_id, cleanup_user_mapping=True)
+        Terminate a specific session in Redis with reliable stop signal and immediate status update.
-
+        """
-    def _terminate_session_internal(self, session_id: str, cleanup_user_mapping: bool = True) -> bool:
-        """Internal session termination with configurable user mapping cleanup."""
         print(f"=== TERMINATING SESSION {session_id} ===")
         
         try:
-            # Set stop signal first
+            # First, set the stop signal
             self.set_stop_signal(session_id)
             
-            # Update scanner status immediately for GUI feedback
+            # Update scanner status to stopped immediately for GUI feedback
             self.update_scanner_status(session_id, 'stopped')
             
             session_data = self._get_session_data(session_id)
@@ -429,19 +324,16 @@ class SessionManager:
             scanner = session_data.get('scanner')
             if scanner and scanner.status == 'running':
                 print(f"Stopping scan for session: {session_id}")
+                # The scanner will check the Redis stop signal
                 scanner.stop_scan()
+                
+                # Update the scanner state immediately
                 self.update_session_scanner(session_id, scanner)
             
-            # Wait for graceful shutdown
+            # Wait a moment for graceful shutdown
             time.sleep(0.5)
             
-            # Clean up user mapping if requested
+            # Delete session data and stop signal from Redis
-            if cleanup_user_mapping and 'user_fingerprint' in session_data:
-                user_session_key = self._get_user_session_key(session_data['user_fingerprint'])
-                self.redis_client.delete(user_session_key)
-                print(f"Cleaned up user mapping for {session_data['user_fingerprint']}")
-            
-            # Delete session data and stop signal
             session_key = self._get_session_key(session_id)
             stop_key = self._get_stop_signal_key(session_id)
             self.redis_client.delete(session_key)
@@ -455,72 +347,35 @@ class SessionManager:
             return False
 
     def _cleanup_loop(self) -> None:
-        """Background thread to cleanup inactive sessions and orphaned signals."""
+        """
+        Background thread to cleanup inactive sessions and orphaned stop signals.
+        """
         while True:
             try:
                 # Clean up orphaned stop signals
-                stop_keys = self.redis_client.keys("dnsrecon:stop:*")
+                stop_keys = self.redis_client.keys("DNScope:stop:*")
                 for stop_key in stop_keys:
+                    # Extract session ID from stop key
                     session_id = stop_key.decode('utf-8').split(':')[-1]
                     session_key = self._get_session_key(session_id)
                     
+                    # If session doesn't exist but stop signal does, clean it up
                     if not self.redis_client.exists(session_key):
                         self.redis_client.delete(stop_key)
                         print(f"Cleaned up orphaned stop signal for session {session_id}")
-                
-                # Clean up orphaned user mappings
-                user_keys = self.redis_client.keys("dnsrecon:user:*")
-                for user_key in user_keys:
-                    session_id_bytes = self.redis_client.get(user_key)
-                    if session_id_bytes:
-                        session_id = session_id_bytes.decode('utf-8')
-                        session_key = self._get_session_key(session_id)
-                        
-                        if not self.redis_client.exists(session_key):
-                            self.redis_client.delete(user_key)
-                            print(f"Cleaned up orphaned user mapping for session {session_id}")
                         
             except Exception as e:
                 print(f"Error in cleanup loop: {e}")
             
             time.sleep(300)  # Sleep for 5 minutes
 
-    def list_active_sessions(self) -> List[Dict[str, Any]]:
-        """List all active sessions for admin purposes."""
-        try:
-            session_keys = self.redis_client.keys("dnsrecon:session:*")
-            sessions = []
-            
-            for session_key in session_keys:
-                session_id = session_key.decode('utf-8').split(':')[-1]
-                session_data = self._get_session_data(session_id)
-                
-                if session_data:
-                    scanner = session_data.get('scanner')
-                    sessions.append({
-                        'session_id': session_id,
-                        'user_fingerprint': session_data.get('user_fingerprint', 'unknown'),
-                        'client_ip': session_data.get('client_ip', 'unknown'),
-                        'created_at': session_data.get('created_at'),
-                        'last_activity': session_data.get('last_activity'),
-                        'scanner_status': scanner.status if scanner else 'unknown',
-                        'current_target': scanner.current_target if scanner else None
-                    })
-            
-            return sessions
-        except Exception as e:
-            print(f"ERROR: Failed to list active sessions: {e}")
-            return []
-
     def get_statistics(self) -> Dict[str, Any]:
         """Get session manager statistics."""
         try:
-            session_keys = self.redis_client.keys("dnsrecon:session:*")
+            session_keys = self.redis_client.keys("DNScope:session:*")
-            user_keys = self.redis_client.keys("dnsrecon:user:*")
+            stop_keys = self.redis_client.keys("DNScope:stop:*")
-            stop_keys = self.redis_client.keys("dnsrecon:stop:*")
             
             active_sessions = len(session_keys)
-            unique_users = len(user_keys)
             running_scans = 0
             
             for session_key in session_keys:
@@ -531,46 +386,16 @@ class SessionManager:
             
             return {
                 'total_active_sessions': active_sessions,
-                'unique_users': unique_users,
                 'running_scans': running_scans,
-                'total_stop_signals': len(stop_keys),
+                'total_stop_signals': len(stop_keys)
-                'average_sessions_per_user': round(active_sessions / unique_users, 2) if unique_users > 0 else 0
             }
         except Exception as e:
             print(f"ERROR: Failed to get statistics: {e}")
             return {
                 'total_active_sessions': 0,
-                'unique_users': 0,
                 'running_scans': 0,
-                'total_stop_signals': 0,
+                'total_stop_signals': 0
-                'average_sessions_per_user': 0
             }
 
-    def get_session_info(self, session_id: str) -> Dict[str, Any]:
-        """Get detailed information about a specific session."""
-        try:
-            session_data = self._get_session_data(session_id)
-            if not session_data:
-                return {'error': 'Session not found'}
-            
-            scanner = session_data.get('scanner')
-            
-            return {
-                'session_id': session_id,
-                'user_fingerprint': session_data.get('user_fingerprint', 'unknown'),
-                'client_ip': session_data.get('client_ip', 'unknown'),
-                'user_agent': session_data.get('user_agent', 'unknown'),
-                'created_at': session_data.get('created_at'),
-                'last_activity': session_data.get('last_activity'),
-                'status': session_data.get('status'),
-                'scanner_status': scanner.status if scanner else 'unknown',
-                'current_target': scanner.current_target if scanner else None,
-                'session_age_minutes': round((time.time() - session_data.get('created_at', time.time())) / 60, 1)
-            }
-        except Exception as e:
-            print(f"ERROR: Failed to get session info for {session_id}: {e}")
-            return {'error': f'Failed to get session info: {str(e)}'}
-
-
 # Global session manager instance
 session_manager = SessionManager(session_timeout_minutes=60)

core/task_manager.py

@@ -1,564 +0,0 @@
-# dnsrecon/core/task_manager.py
-
-import threading
-import time
-import uuid
-from enum import Enum
-from dataclasses import dataclass, field
-from typing import Dict, List, Optional, Any, Set
-from datetime import datetime, timezone, timedelta
-from collections import deque
-
-from utils.helpers import _is_valid_ip, _is_valid_domain
-
-
-class TaskStatus(Enum):
-    """Enumeration of task execution statuses."""
-    PENDING = "pending"
-    RUNNING = "running"
-    SUCCEEDED = "succeeded"
-    FAILED_RETRYING = "failed_retrying"
-    FAILED_PERMANENT = "failed_permanent"
-    CANCELLED = "cancelled"
-
-
-class TaskType(Enum):
-    """Enumeration of task types for provider queries."""
-    DOMAIN_QUERY = "domain_query"
-    IP_QUERY = "ip_query"
-    GRAPH_UPDATE = "graph_update"
-
-
-@dataclass
-class TaskResult:
-    """Result of a task execution."""
-    success: bool
-    data: Optional[Any] = None
-    error: Optional[str] = None
-    metadata: Dict[str, Any] = field(default_factory=dict)
-
-
-@dataclass
-class ReconTask:
-    """Represents a single reconnaissance task with retry logic."""
-    task_id: str
-    task_type: TaskType
-    target: str
-    provider_name: str
-    depth: int
-    status: TaskStatus = TaskStatus.PENDING
-    created_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
-    
-    # Retry configuration
-    max_retries: int = 3
-    current_retry: int = 0
-    base_backoff_seconds: float = 1.0
-    max_backoff_seconds: float = 60.0
-    
-    # Execution tracking
-    last_attempt_at: Optional[datetime] = None
-    next_retry_at: Optional[datetime] = None
-    execution_history: List[Dict[str, Any]] = field(default_factory=list)
-    
-    # Results
-    result: Optional[TaskResult] = None
-    
-    def __post_init__(self):
-        """Initialize additional fields after creation."""
-        if not self.task_id:
-            self.task_id = str(uuid.uuid4())[:8]
-    
-    def calculate_next_retry_time(self) -> datetime:
-        """Calculate next retry time with exponential backoff and jitter."""
-        if self.current_retry >= self.max_retries:
-            return None
-        
-        # Exponential backoff with jitter
-        backoff_time = min(
-            self.max_backoff_seconds,
-            self.base_backoff_seconds * (2 ** self.current_retry)
-        )
-        
-        # Add jitter (±25%)
-        jitter = backoff_time * 0.25 * (0.5 - hash(self.task_id) % 1000 / 1000.0)
-        final_backoff = max(self.base_backoff_seconds, backoff_time + jitter)
-        
-        return datetime.now(timezone.utc) + timedelta(seconds=final_backoff)
-    
-    def should_retry(self) -> bool:
-        """Determine if task should be retried based on status and retry count."""
-        if self.status != TaskStatus.FAILED_RETRYING:
-            return False
-        if self.current_retry >= self.max_retries:
-            return False
-        if self.next_retry_at and datetime.now(timezone.utc) < self.next_retry_at:
-            return False
-        return True
-    
-    def mark_failed(self, error: str, metadata: Dict[str, Any] = None):
-        """Mark task as failed and prepare for retry or permanent failure."""
-        self.current_retry += 1
-        self.last_attempt_at = datetime.now(timezone.utc)
-        
-        # Record execution history
-        execution_record = {
-            'attempt': self.current_retry,
-            'timestamp': self.last_attempt_at.isoformat(),
-            'error': error,
-            'metadata': metadata or {}
-        }
-        self.execution_history.append(execution_record)
-        
-        if self.current_retry >= self.max_retries:
-            self.status = TaskStatus.FAILED_PERMANENT
-            self.result = TaskResult(success=False, error=f"Permanent failure after {self.max_retries} attempts: {error}")
-        else:
-            self.status = TaskStatus.FAILED_RETRYING
-            self.next_retry_at = self.calculate_next_retry_time()
-    
-    def mark_succeeded(self, data: Any = None, metadata: Dict[str, Any] = None):
-        """Mark task as successfully completed."""
-        self.status = TaskStatus.SUCCEEDED
-        self.last_attempt_at = datetime.now(timezone.utc)
-        self.result = TaskResult(success=True, data=data, metadata=metadata or {})
-        
-        # Record successful execution
-        execution_record = {
-            'attempt': self.current_retry + 1,
-            'timestamp': self.last_attempt_at.isoformat(),
-            'success': True,
-            'metadata': metadata or {}
-        }
-        self.execution_history.append(execution_record)
-    
-    def get_summary(self) -> Dict[str, Any]:
-        """Get task summary for progress reporting."""
-        return {
-            'task_id': self.task_id,
-            'task_type': self.task_type.value,
-            'target': self.target,
-            'provider': self.provider_name,
-            'status': self.status.value,
-            'current_retry': self.current_retry,
-            'max_retries': self.max_retries,
-            'created_at': self.created_at.isoformat(),
-            'last_attempt_at': self.last_attempt_at.isoformat() if self.last_attempt_at else None,
-            'next_retry_at': self.next_retry_at.isoformat() if self.next_retry_at else None,
-            'total_attempts': len(self.execution_history),
-            'has_result': self.result is not None
-        }
-
-
-class TaskQueue:
-    """Thread-safe task queue with retry logic and priority handling."""
-    
-    def __init__(self, max_concurrent_tasks: int = 5):
-        """Initialize task queue."""
-        self.max_concurrent_tasks = max_concurrent_tasks
-        self.tasks: Dict[str, ReconTask] = {}
-        self.pending_queue = deque()
-        self.retry_queue = deque()
-        self.running_tasks: Set[str] = set()
-        
-        self._lock = threading.Lock()
-        self._stop_event = threading.Event()
-    
-    def __getstate__(self):
-        """Prepare TaskQueue for pickling by excluding unpicklable objects."""
-        state = self.__dict__.copy()
-        # Exclude the unpickleable '_lock' and '_stop_event' attributes
-        if '_lock' in state:
-            del state['_lock']
-        if '_stop_event' in state:
-            del state['_stop_event']
-        return state
-
-    def __setstate__(self, state):
-        """Restore TaskQueue after unpickling by reconstructing threading objects."""
-        self.__dict__.update(state)
-        # Re-initialize the '_lock' and '_stop_event' attributes
-        self._lock = threading.Lock()
-        self._stop_event = threading.Event()
-    
-    def add_task(self, task: ReconTask) -> str:
-        """Add task to queue."""
-        with self._lock:
-            self.tasks[task.task_id] = task
-            self.pending_queue.append(task.task_id)
-            print(f"Added task {task.task_id}: {task.provider_name} query for {task.target}")
-            return task.task_id
-    
-    def get_next_ready_task(self) -> Optional[ReconTask]:
-        """Get next task ready for execution."""
-        with self._lock:
-            # Check if we have room for more concurrent tasks
-            if len(self.running_tasks) >= self.max_concurrent_tasks:
-                return None
-            
-            # First priority: retry queue (tasks ready for retry)
-            while self.retry_queue:
-                task_id = self.retry_queue.popleft()
-                if task_id in self.tasks:
-                    task = self.tasks[task_id]
-                    if task.should_retry():
-                        task.status = TaskStatus.RUNNING
-                        self.running_tasks.add(task_id)
-                        print(f"Retrying task {task_id} (attempt {task.current_retry + 1})")
-                        return task
-            
-            # Second priority: pending queue (new tasks)
-            while self.pending_queue:
-                task_id = self.pending_queue.popleft()
-                if task_id in self.tasks:
-                    task = self.tasks[task_id]
-                    if task.status == TaskStatus.PENDING:
-                        task.status = TaskStatus.RUNNING
-                        self.running_tasks.add(task_id)
-                        print(f"Starting task {task_id}")
-                        return task
-            
-            return None
-    
-    def complete_task(self, task_id: str, success: bool, data: Any = None, 
-                     error: str = None, metadata: Dict[str, Any] = None):
-        """Mark task as completed (success or failure)."""
-        with self._lock:
-            if task_id not in self.tasks:
-                return
-            
-            task = self.tasks[task_id]
-            self.running_tasks.discard(task_id)
-            
-            if success:
-                task.mark_succeeded(data=data, metadata=metadata)
-                print(f"Task {task_id} succeeded")
-            else:
-                task.mark_failed(error or "Unknown error", metadata=metadata)
-                if task.status == TaskStatus.FAILED_RETRYING:
-                    self.retry_queue.append(task_id)
-                    print(f"Task {task_id} failed, scheduled for retry at {task.next_retry_at}")
-                else:
-                    print(f"Task {task_id} permanently failed after {task.current_retry} attempts")
-    
-    def cancel_all_tasks(self):
-        """Cancel all pending and running tasks."""
-        with self._lock:
-            self._stop_event.set()
-            for task in self.tasks.values():
-                if task.status in [TaskStatus.PENDING, TaskStatus.RUNNING, TaskStatus.FAILED_RETRYING]:
-                    task.status = TaskStatus.CANCELLED
-            self.pending_queue.clear()
-            self.retry_queue.clear()
-            self.running_tasks.clear()
-            print("All tasks cancelled")
-    
-    def is_complete(self) -> bool:
-        """Check if all tasks are complete (succeeded, permanently failed, or cancelled)."""
-        with self._lock:
-            for task in self.tasks.values():
-                if task.status in [TaskStatus.PENDING, TaskStatus.RUNNING, TaskStatus.FAILED_RETRYING]:
-                    return False
-            return True
-    
-    def get_statistics(self) -> Dict[str, Any]:
-        """Get queue statistics."""
-        with self._lock:
-            stats = {
-                'total_tasks': len(self.tasks),
-                'pending': len(self.pending_queue),
-                'running': len(self.running_tasks),
-                'retry_queue': len(self.retry_queue),
-                'succeeded': 0,
-                'failed_permanent': 0,
-                'cancelled': 0,
-                'failed_retrying': 0
-            }
-            
-            for task in self.tasks.values():
-                if task.status == TaskStatus.SUCCEEDED:
-                    stats['succeeded'] += 1
-                elif task.status == TaskStatus.FAILED_PERMANENT:
-                    stats['failed_permanent'] += 1
-                elif task.status == TaskStatus.CANCELLED:
-                    stats['cancelled'] += 1
-                elif task.status == TaskStatus.FAILED_RETRYING:
-                    stats['failed_retrying'] += 1
-            
-            stats['completion_rate'] = (stats['succeeded'] / stats['total_tasks'] * 100) if stats['total_tasks'] > 0 else 0
-            stats['is_complete'] = self.is_complete()
-            
-            return stats
-    
-    def get_task_summaries(self) -> List[Dict[str, Any]]:
-        """Get summaries of all tasks for detailed progress reporting."""
-        with self._lock:
-            return [task.get_summary() for task in self.tasks.values()]
-    
-    def get_failed_tasks(self) -> List[ReconTask]:
-        """Get all permanently failed tasks for analysis."""
-        with self._lock:
-            return [task for task in self.tasks.values() if task.status == TaskStatus.FAILED_PERMANENT]
-
-
-class TaskExecutor:
-    """Executes reconnaissance tasks using providers."""
-    
-    def __init__(self, providers: List, graph_manager, logger):
-        """Initialize task executor."""
-        self.providers = {provider.get_name(): provider for provider in providers}
-        self.graph = graph_manager
-        self.logger = logger
-    
-    def execute_task(self, task: ReconTask) -> TaskResult:
-        """
-        Execute a single reconnaissance task.
-        
-        Args:
-            task: Task to execute
-            
-        Returns:
-            TaskResult with success/failure information
-        """
-        try:
-            print(f"Executing task {task.task_id}: {task.provider_name} query for {task.target}")
-            
-            provider = self.providers.get(task.provider_name)
-            if not provider:
-                return TaskResult(
-                    success=False, 
-                    error=f"Provider {task.provider_name} not available"
-                )
-            
-            if not provider.is_available():
-                return TaskResult(
-                    success=False,
-                    error=f"Provider {task.provider_name} is not available (missing API key or configuration)"
-                )
-            
-            # Execute provider query based on task type
-            if task.task_type == TaskType.DOMAIN_QUERY:
-                if not _is_valid_domain(task.target):
-                    return TaskResult(success=False, error=f"Invalid domain: {task.target}")
-                    
-                relationships = provider.query_domain(task.target)
-                
-            elif task.task_type == TaskType.IP_QUERY:
-                if not _is_valid_ip(task.target):
-                    return TaskResult(success=False, error=f"Invalid IP: {task.target}")
-                    
-                relationships = provider.query_ip(task.target)
-                
-            else:
-                return TaskResult(success=False, error=f"Unsupported task type: {task.task_type}")
-            
-            # Process results and update graph
-            new_targets = set()
-            relationships_added = 0
-            
-            for source, target, rel_type, confidence, raw_data in relationships:
-                # Add nodes to graph
-                from core.graph_manager import NodeType
-                
-                if _is_valid_ip(target):
-                    self.graph.add_node(target, NodeType.IP)
-                    new_targets.add(target)
-                elif target.startswith('AS') and target[2:].isdigit():
-                    self.graph.add_node(target, NodeType.ASN)
-                elif _is_valid_domain(target):
-                    self.graph.add_node(target, NodeType.DOMAIN)
-                    new_targets.add(target)
-                
-                # Add edge to graph
-                if self.graph.add_edge(source, target, rel_type, confidence, task.provider_name, raw_data):
-                    relationships_added += 1
-            
-            # Log forensic information
-            self.logger.logger.info(
-                f"Task {task.task_id} completed: {len(relationships)} relationships found, "
-                f"{relationships_added} added to graph, {len(new_targets)} new targets"
-            )
-            
-            return TaskResult(
-                success=True,
-                data={
-                    'relationships': relationships,
-                    'new_targets': list(new_targets),
-                    'relationships_added': relationships_added
-                },
-                metadata={
-                    'provider': task.provider_name,
-                    'target': task.target,
-                    'depth': task.depth,
-                    'execution_time': datetime.now(timezone.utc).isoformat()
-                }
-            )
-            
-        except Exception as e:
-            error_msg = f"Task execution failed: {str(e)}"
-            print(f"ERROR: {error_msg} for task {task.task_id}")
-            self.logger.logger.error(error_msg)
-            
-            return TaskResult(
-                success=False,
-                error=error_msg,
-                metadata={
-                    'provider': task.provider_name,
-                    'target': task.target,
-                    'exception_type': type(e).__name__
-                }
-            )
-
-
-class TaskManager:
-    """High-level task management for reconnaissance scans."""
-    
-    def __init__(self, providers: List, graph_manager, logger, max_concurrent_tasks: int = 5):
-        """Initialize task manager."""
-        self.task_queue = TaskQueue(max_concurrent_tasks)
-        self.task_executor = TaskExecutor(providers, graph_manager, logger)
-        self.logger = logger
-        
-        # Execution control
-        self._stop_event = threading.Event()
-        self._execution_threads: List[threading.Thread] = []
-        self._is_running = False
-    
-    def create_provider_tasks(self, target: str, depth: int, providers: List) -> List[str]:
-        """
-        Create tasks for querying all eligible providers for a target.
-        
-        Args:
-            target: Domain or IP to query
-            depth: Current recursion depth
-            providers: List of available providers
-            
-        Returns:
-            List of created task IDs
-        """
-        task_ids = []
-        is_ip = _is_valid_ip(target)
-        target_key = 'ips' if is_ip else 'domains'
-        task_type = TaskType.IP_QUERY if is_ip else TaskType.DOMAIN_QUERY
-        
-        for provider in providers:
-            if provider.get_eligibility().get(target_key) and provider.is_available():
-                task = ReconTask(
-                    task_id=str(uuid.uuid4())[:8],
-                    task_type=task_type,
-                    target=target,
-                    provider_name=provider.get_name(),
-                    depth=depth,
-                    max_retries=3  # Configure retries per task type/provider
-                )
-                
-                task_id = self.task_queue.add_task(task)
-                task_ids.append(task_id)
-        
-        return task_ids
-    
-    def start_execution(self, max_workers: int = 3):
-        """Start task execution with specified number of worker threads."""
-        if self._is_running:
-            print("Task execution already running")
-            return
-        
-        self._is_running = True
-        self._stop_event.clear()
-        
-        print(f"Starting task execution with {max_workers} workers")
-        
-        for i in range(max_workers):
-            worker_thread = threading.Thread(
-                target=self._worker_loop,
-                name=f"TaskWorker-{i+1}",
-                daemon=True
-            )
-            worker_thread.start()
-            self._execution_threads.append(worker_thread)
-    
-    def stop_execution(self):
-        """Stop task execution and cancel all tasks."""
-        print("Stopping task execution")
-        self._stop_event.set()
-        self.task_queue.cancel_all_tasks()
-        self._is_running = False
-        
-        # Wait for worker threads to finish
-        for thread in self._execution_threads:
-            thread.join(timeout=5.0)
-        
-        self._execution_threads.clear()
-        print("Task execution stopped")
-    
-    def _worker_loop(self):
-        """Worker thread loop for executing tasks."""
-        thread_name = threading.current_thread().name
-        print(f"{thread_name} started")
-        
-        while not self._stop_event.is_set():
-            try:
-                # Get next task to execute
-                task = self.task_queue.get_next_ready_task()
-                
-                if task is None:
-                    # No tasks ready, check if we should exit
-                    if self.task_queue.is_complete() or self._stop_event.is_set():
-                        break
-                    time.sleep(0.1)  # Brief sleep before checking again
-                    continue
-                
-                # Execute the task
-                result = self.task_executor.execute_task(task)
-                
-                # Complete the task in queue
-                self.task_queue.complete_task(
-                    task.task_id,
-                    success=result.success,
-                    data=result.data,
-                    error=result.error,
-                    metadata=result.metadata
-                )
-                
-            except Exception as e:
-                print(f"ERROR: Worker {thread_name} encountered error: {e}")
-                # Continue running even if individual task fails
-                continue
-        
-        print(f"{thread_name} finished")
-    
-    def wait_for_completion(self, timeout_seconds: int = 300) -> bool:
-        """
-        Wait for all tasks to complete.
-        
-        Args:
-            timeout_seconds: Maximum time to wait
-            
-        Returns:
-            True if all tasks completed, False if timeout
-        """
-        start_time = time.time()
-        
-        while time.time() - start_time < timeout_seconds:
-            if self.task_queue.is_complete():
-                return True
-            
-            if self._stop_event.is_set():
-                return False
-            
-            time.sleep(1.0)  # Check every second
-        
-        print(f"Timeout waiting for task completion after {timeout_seconds} seconds")
-        return False
-    
-    def get_progress_report(self) -> Dict[str, Any]:
-        """Get detailed progress report for UI updates."""
-        stats = self.task_queue.get_statistics()
-        failed_tasks = self.task_queue.get_failed_tasks()
-        
-        return {
-            'statistics': stats,
-            'failed_tasks': [task.get_summary() for task in failed_tasks],
-            'is_running': self._is_running,
-            'worker_count': len(self._execution_threads),
-            'detailed_tasks': self.task_queue.get_task_summaries() if stats['total_tasks'] < 50 else []  # Limit detail for performance
-        }

providers/__init__.py

@@ -1,19 +1,22 @@
 """
-Data provider modules for DNSRecon.
+Data provider modules for DNScope.
 Contains implementations for various reconnaissance data sources.
 """
 
-from .base_provider import BaseProvider, RateLimiter
+from .base_provider import BaseProvider
 from .crtsh_provider import CrtShProvider
 from .dns_provider import DNSProvider
 from .shodan_provider import ShodanProvider
+from .correlation_provider import CorrelationProvider
+from core.rate_limiter import GlobalRateLimiter
 
 __all__ = [
     'BaseProvider',
-    'RateLimiter', 
+    'GlobalRateLimiter',
     'CrtShProvider',
     'DNSProvider',
-    'ShodanProvider'
+    'ShodanProvider',
+    'CorrelationProvider'
 ]
 
 __version__ = "0.0.0-rc"

providers/base_provider.py

@@ -1,177 +1,25 @@
-# dnsrecon/providers/base_provider.py
+# DNScope/providers/base_provider.py
 
 import time
 import requests
 import threading
-import os
-import json
-import hashlib
 from abc import ABC, abstractmethod
-from typing import List, Dict, Any, Optional, Tuple
+from typing import Dict, Any, Optional
-from datetime import datetime, timezone
 
-from core.logger import get_forensic_logger
+from core.logger import get_forensic_logger  # Ensure this import is present
-
+from core.rate_limiter import GlobalRateLimiter
-
+from core.provider_result import ProviderResult
-class RateLimiter:
-    """Thread-safe rate limiter for API calls."""
-
-    def __init__(self, requests_per_minute: int):
-        """
-        Initialize rate limiter.
-
-        Args:
-            requests_per_minute: Maximum requests allowed per minute
-        """
-        self.requests_per_minute = requests_per_minute
-        self.min_interval = 60.0 / requests_per_minute
-        self.last_request_time = 0
-        self._lock = threading.Lock()
-
-    def __getstate__(self):
-        """RateLimiter is fully picklable, return full state."""
-        state = self.__dict__.copy()
-        # Exclude unpickleable lock
-        if '_lock' in state:
-            del state['_lock']
-        return state
-
-    def __setstate__(self, state):
-        """Restore RateLimiter state."""
-        self.__dict__.update(state)
-        self._lock = threading.Lock()
-
-    def wait_if_needed(self) -> None:
-        """Wait if necessary to respect rate limits."""
-        with self._lock:
-            current_time = time.time()
-            time_since_last = current_time - self.last_request_time
-
-            if time_since_last < self.min_interval:
-                sleep_time = self.min_interval - time_since_last
-                time.sleep(sleep_time)
-
-            self.last_request_time = time.time()
-
-
-class ProviderCache:
-    """Thread-safe global cache for provider queries."""
-    
-    def __init__(self, provider_name: str, cache_expiry_hours: int = 12):
-        """
-        Initialize provider-specific cache.
-        
-        Args:
-            provider_name: Name of the provider for cache directory
-            cache_expiry_hours: Cache expiry time in hours
-        """
-        self.provider_name = provider_name
-        self.cache_expiry = cache_expiry_hours * 3600  # Convert to seconds
-        self.cache_dir = os.path.join('.cache', provider_name)
-        self._lock = threading.Lock()
-        
-        # Ensure cache directory exists with thread-safe creation
-        os.makedirs(self.cache_dir, exist_ok=True)
-        
-    def _generate_cache_key(self, method: str, url: str, params: Optional[Dict[str, Any]]) -> str:
-        """Generate unique cache key for request."""
-        cache_data = f"{method}:{url}:{json.dumps(params or {}, sort_keys=True)}"
-        return hashlib.md5(cache_data.encode()).hexdigest() + ".json"
-    
-    def get_cached_response(self, method: str, url: str, params: Optional[Dict[str, Any]]) -> Optional[requests.Response]:
-        """
-        Retrieve cached response if available and not expired.
-        
-        Returns:
-            Cached Response object or None if cache miss/expired
-        """
-        cache_key = self._generate_cache_key(method, url, params)
-        cache_path = os.path.join(self.cache_dir, cache_key)
-        
-        with self._lock:
-            if not os.path.exists(cache_path):
-                return None
-                
-            # Check if cache is expired
-            cache_age = time.time() - os.path.getmtime(cache_path)
-            if cache_age >= self.cache_expiry:
-                try:
-                    os.remove(cache_path)
-                except OSError:
-                    pass  # File might have been removed by another thread
-                return None
-            
-            try:
-                with open(cache_path, 'r', encoding='utf-8') as f:
-                    cached_data = json.load(f)
-                    
-                # Reconstruct Response object
-                response = requests.Response()
-                response.status_code = cached_data['status_code']
-                response._content = cached_data['content'].encode('utf-8')
-                response.headers.update(cached_data['headers'])
-                
-                return response
-                
-            except (json.JSONDecodeError, KeyError, IOError) as e:
-                # Cache file corrupted, remove it
-                try:
-                    os.remove(cache_path)
-                except OSError:
-                    pass
-                return None
-    
-    def cache_response(self, method: str, url: str, params: Optional[Dict[str, Any]], 
-                      response: requests.Response) -> bool:
-        """
-        Cache successful response to disk.
-        
-        Returns:
-            True if cached successfully, False otherwise
-        """
-        if response.status_code != 200:
-            return False
-            
-        cache_key = self._generate_cache_key(method, url, params)
-        cache_path = os.path.join(self.cache_dir, cache_key)
-        
-        with self._lock:
-            try:
-                cache_data = {
-                    'status_code': response.status_code,
-                    'content': response.text,
-                    'headers': dict(response.headers),
-                    'cached_at': datetime.now(timezone.utc).isoformat()
-                }
-                
-                # Write to temporary file first, then rename for atomic operation
-                temp_path = cache_path + '.tmp'
-                with open(temp_path, 'w', encoding='utf-8') as f:
-                    json.dump(cache_data, f)
-                
-                # Atomic rename to prevent partial cache files
-                os.rename(temp_path, cache_path)
-                return True
-                
-            except (IOError, OSError) as e:
-                # Clean up temp file if it exists
-                try:
-                    if os.path.exists(temp_path):
-                        os.remove(temp_path)
-                except OSError:
-                    pass
-                return False
 
 
 class BaseProvider(ABC):
     """
-    Abstract base class for all DNSRecon data providers.
+    Abstract base class for all DNScope data providers.
-    Now supports global provider-specific caching and session-specific configuration.
+    Now supports session-specific configuration and returns standardized ProviderResult objects.
     """
 
     def __init__(self, name: str, rate_limit: int = 60, timeout: int = 30, session_config=None):
         """
-        Initialize base provider with global caching and session-specific configuration.
+        Initialize base provider with session-specific configuration.
 
         Args:
             name: Provider name for logging
@@ -188,35 +36,27 @@ class BaseProvider(ABC):
             # Fallback to global config for backwards compatibility
             from config import config as global_config
             self.config = global_config
-            actual_rate_limit = rate_limit
             actual_timeout = timeout
 
         self.name = name
-        self.rate_limiter = RateLimiter(actual_rate_limit)
         self.timeout = actual_timeout
         self._local = threading.local()
-        self.logger = get_forensic_logger()
         self._stop_event = None
 
-        # GLOBAL provider-specific caching (not session-based)
-        self.cache = ProviderCache(name, cache_expiry_hours=12)
-
         # Statistics (per provider instance)
         self.total_requests = 0
         self.successful_requests = 0
         self.failed_requests = 0
         self.total_relationships_found = 0
-        self.cache_hits = 0
-        self.cache_misses = 0
-
-        print(f"Initialized {name} provider with global cache and session config (rate: {actual_rate_limit}/min)")
 
     def __getstate__(self):
         """Prepare BaseProvider for pickling by excluding unpicklable objects."""
         state = self.__dict__.copy()
         # Exclude the unpickleable '_local' attribute and stop event
-        state['_local'] = None
+        unpicklable_attrs = ['_local', '_stop_event']
-        state['_stop_event'] = None
+        for attr in unpicklable_attrs:
+            if attr in state:
+                del state[attr]
         return state
 
     def __setstate__(self, state):
@@ -231,10 +71,15 @@ class BaseProvider(ABC):
         if not hasattr(self._local, 'session'):
             self._local.session = requests.Session()
             self._local.session.headers.update({
-                'User-Agent': 'DNSRecon/2.0 (Passive Reconnaissance Tool)'
+                'User-Agent': 'DNScope/1.0 (Passive Reconnaissance Tool)'
             })
         return self._local.session
 
+    @property
+    def logger(self):
+        """Get the current forensic logger instance."""
+        return get_forensic_logger()
+
     @abstractmethod
     def get_name(self) -> str:
         """Return the provider name."""
@@ -261,7 +106,7 @@ class BaseProvider(ABC):
         pass
 
     @abstractmethod
-    def query_domain(self, domain: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]:
+    def query_domain(self, domain: str) -> ProviderResult:
         """
         Query the provider for information about a domain.
 
@@ -269,12 +114,12 @@ class BaseProvider(ABC):
             domain: Domain to investigate
 
         Returns:
-            List of tuples: (source_node, target_node, relationship_type, confidence, raw_data)
+            ProviderResult containing standardized attributes and relationships
         """
         pass
 
     @abstractmethod
-    def query_ip(self, ip: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]:
+    def query_ip(self, ip: str) -> ProviderResult:
         """
         Query the provider for information about an IP address.
 
@@ -282,160 +127,92 @@ class BaseProvider(ABC):
             ip: IP address to investigate
 
         Returns:
-            List of tuples: (source_node, target_node, relationship_type, confidence, raw_data)
+            ProviderResult containing standardized attributes and relationships
         """
         pass
 
     def make_request(self, url: str, method: str = "GET",
                     params: Optional[Dict[str, Any]] = None,
                     headers: Optional[Dict[str, str]] = None,
-                    target_indicator: str = "",
+                    target_indicator: str = "") -> Optional[requests.Response]:
-                    max_retries: int = 3) -> Optional[requests.Response]:
         """
-        Make a rate-limited HTTP request with global caching and aggressive stop signal handling.
+        Make a rate-limited HTTP request.
+        FIXED: Returns response without automatically raising HTTPError exceptions.
+        Individual providers should handle status codes appropriately.
         """
-        # Check for cancellation before starting
         if self._is_stop_requested():
             print(f"Request cancelled before start: {url}")
             return None
 
-        # Check global cache first
+        start_time = time.time()
-        cached_response = self.cache.get_cached_response(method, url, params)
+        response = None
-        if cached_response is not None:
+        error = None
-            print(f"Cache hit for {self.name}: {url}")
-            self.cache_hits += 1
-            return cached_response
-        
-        self.cache_misses += 1
 
-        # Determine effective max_retries based on stop signal
+        try:
-        effective_max_retries = 0 if self._is_stop_requested() else max_retries
+            self.total_requests += 1
-        last_exception = None
 
-        for attempt in range(effective_max_retries + 1):
+            request_headers = dict(self.session.headers).copy()
-            # Check for cancellation before each attempt
+            if headers:
-            if self._is_stop_requested():
+                request_headers.update(headers)
-                print(f"Request cancelled during attempt {attempt + 1}: {url}")
-                return None
 
-            # Apply rate limiting with cancellation awareness
+            print(f"Making {method} request to: {url}")
-            if not self._wait_with_cancellation_check():
-                print(f"Request cancelled during rate limiting: {url}")
-                return None
 
-            # Final check before making HTTP request
+            if method.upper() == "GET":
-            if self._is_stop_requested():
+                response = self.session.get(
-                print(f"Request cancelled before HTTP call: {url}")
+                    url,
-                return None
+                    params=params,
-
+                    headers=request_headers,
-            start_time = time.time()
+                    timeout=self.timeout
-            response = None
-            error = None
-
-            try:
-                self.total_requests += 1
-
-                # Prepare request
-                request_headers = self.session.headers.copy()
-                if headers:
-                    request_headers.update(headers)
-
-                print(f"Making {method} request to: {url} (attempt {attempt + 1})")
-
-                # Use shorter timeout if termination is requested
-                request_timeout = 2 if self._is_stop_requested() else self.timeout
-
-                # Make request
-                if method.upper() == "GET":
-                    response = self.session.get(
-                        url,
-                        params=params,
-                        headers=request_headers,
-                        timeout=request_timeout
-                    )
-                elif method.upper() == "POST":
-                    response = self.session.post(
-                        url,
-                        json=params,
-                        headers=request_headers,
-                        timeout=request_timeout
-                    )
-                else:
-                    raise ValueError(f"Unsupported HTTP method: {method}")
-
-                print(f"Response status: {response.status_code}")
-                response.raise_for_status()
-                self.successful_requests += 1
-                
-                # Success - log, cache, and return
-                duration_ms = (time.time() - start_time) * 1000
-                self.logger.log_api_request(
-                    provider=self.name,
-                    url=url,
-                    method=method.upper(),
-                    status_code=response.status_code,
-                    response_size=len(response.content),
-                    duration_ms=duration_ms,
-                    error=None,
-                    target_indicator=target_indicator
                 )
-                
+            elif method.upper() == "POST":
-                # Cache the successful response globally
+                response = self.session.post(
-                self.cache.cache_response(method, url, params, response)
+                    url,
-                return response
+                    json=params,
+                    headers=request_headers,
+                    timeout=self.timeout
+                )
+            else:
+                raise ValueError(f"Unsupported HTTP method: {method}")
 
-            except requests.exceptions.RequestException as e:
+            print(f"Response status: {response.status_code}")
-                error = str(e)
+            
+            # FIXED: Don't automatically raise for HTTP error status codes
+            # Let individual providers handle status codes appropriately
+            # Only count 2xx responses as successful
+            if 200 <= response.status_code < 300:
+                self.successful_requests += 1
+            else:
                 self.failed_requests += 1
-                print(f"Request failed (attempt {attempt + 1}): {error}")
+            
-                last_exception = e
+            duration_ms = (time.time() - start_time) * 1000
-                
+            self.logger.log_api_request(
-                # Immediately abort retries if stop requested
+                provider=self.name,
-                if self._is_stop_requested():
+                url=url,
-                    print(f"Stop requested - aborting retries for: {url}")
+                method=method.upper(),
-                    break
+                status_code=response.status_code,
-                
+                response_size=len(response.content),
-                # Check if we should retry
+                duration_ms=duration_ms,
-                if attempt < effective_max_retries and self._should_retry(e):
+                error=None,
-                    # Exponential backoff with jitter for 429 errors
+                target_indicator=target_indicator
-                    if isinstance(e, requests.exceptions.HTTPError) and e.response and e.response.status_code == 429:
+            )
-                        backoff_time = min(60, 10 * (2 ** attempt))
+            
-                        print(f"Rate limit hit. Retrying in {backoff_time} seconds...")
+            return response
-                    else:
-                        backoff_time = min(2.0, (2 ** attempt) * 0.5)
-                        print(f"Retrying in {backoff_time} seconds...")
-                    
-                    if not self._sleep_with_cancellation_check(backoff_time):
-                        print(f"Stop requested during backoff - aborting: {url}")
-                        return None
-                    continue
-                else:
-                    break
 
-            except Exception as e:
+        except requests.exceptions.RequestException as e:
-                error = f"Unexpected error: {str(e)}"
+            error = str(e)
-                self.failed_requests += 1
+            self.failed_requests += 1
-                print(f"Unexpected error: {error}")
+            duration_ms = (time.time() - start_time) * 1000
-                last_exception = e
+            self.logger.log_api_request(
-                break
+                provider=self.name,
-
+                url=url,
-        # All attempts failed - log and return None
+                method=method.upper(),
-        duration_ms = (time.time() - start_time) * 1000
+                status_code=response.status_code if response else None,
-        self.logger.log_api_request(
+                response_size=len(response.content) if response else None,
-            provider=self.name,
+                duration_ms=duration_ms,
-            url=url,
+                error=error,
-            method=method.upper(),
+                target_indicator=target_indicator
-            status_code=response.status_code if response else None,
+            )
-            response_size=len(response.content) if response else None,
+            raise e
-            duration_ms=duration_ms,
-            error=error,
-            target_indicator=target_indicator
-        )
-        
-        if error and last_exception:
-            raise last_exception
-        
-        return None
 
     def _is_stop_requested(self) -> bool:
         """
@@ -445,43 +222,6 @@ class BaseProvider(ABC):
             return True
         return False
 
-    def _wait_with_cancellation_check(self) -> bool:
-        """
-        Wait for rate limiting while aggressively checking for cancellation.
-        Returns False if cancelled during wait.
-        """
-        current_time = time.time()
-        time_since_last = current_time - self.rate_limiter.last_request_time
-
-        if time_since_last < self.rate_limiter.min_interval:
-            sleep_time = self.rate_limiter.min_interval - time_since_last
-            if not self._sleep_with_cancellation_check(sleep_time):
-                return False
-
-        self.rate_limiter.last_request_time = time.time()
-        return True
-
-    def _sleep_with_cancellation_check(self, sleep_time: float) -> bool:
-        """
-        Sleep for the specified time while aggressively checking for cancellation.
-        
-        Args:
-            sleep_time: Time to sleep in seconds
-            
-        Returns:
-            bool: True if sleep completed, False if cancelled
-        """
-        sleep_start = time.time()
-        check_interval = 0.05  # Check every 50ms for aggressive responsiveness
-        
-        while time.time() - sleep_start < sleep_time:
-            if self._is_stop_requested():
-                return False
-            remaining_time = sleep_time - (time.time() - sleep_start)
-            time.sleep(min(check_interval, remaining_time))
-        
-        return True
-
     def set_stop_event(self, stop_event: threading.Event) -> None:
         """
         Set the stop event for this provider to enable cancellation.
@@ -491,31 +231,8 @@ class BaseProvider(ABC):
         """
         self._stop_event = stop_event
 
-    def _should_retry(self, exception: requests.exceptions.RequestException) -> bool:
-        """
-        Determine if a request should be retried based on the exception.
-        
-        Args:
-            exception: The request exception that occurred
-            
-        Returns:
-            True if the request should be retried
-        """
-        # Retry on connection errors and timeouts
-        if isinstance(exception, (requests.exceptions.ConnectionError, 
-                                requests.exceptions.Timeout)):
-            return True
-        
-        if isinstance(exception, requests.exceptions.HTTPError):
-            if hasattr(exception, 'response') and exception.response:
-                # Retry on server errors (5xx) AND on rate-limiting errors (429)
-                return exception.response.status_code >= 500 or exception.response.status_code == 429
-        
-        return False
-
     def log_relationship_discovery(self, source_node: str, target_node: str,
                                  relationship_type: str,
-                                 confidence_score: float,
                                  raw_data: Dict[str, Any],
                                  discovery_method: str) -> None:
         """
@@ -525,7 +242,6 @@ class BaseProvider(ABC):
             source_node: Source node identifier
             target_node: Target node identifier
             relationship_type: Type of relationship
-            confidence_score: Confidence score
             raw_data: Raw data from provider
             discovery_method: Method used for discovery
         """
@@ -535,7 +251,6 @@ class BaseProvider(ABC):
             source_node=source_node,
             target_node=target_node,
             relationship_type=relationship_type,
-            confidence_score=confidence_score,
             provider=self.name,
             raw_data=raw_data,
             discovery_method=discovery_method
@@ -543,7 +258,7 @@ class BaseProvider(ABC):
 
     def get_statistics(self) -> Dict[str, Any]:
         """
-        Get provider statistics including cache performance.
+        Get provider statistics.
 
         Returns:
             Dictionary containing provider performance metrics
@@ -555,8 +270,5 @@ class BaseProvider(ABC):
             'failed_requests': self.failed_requests,
             'success_rate': (self.successful_requests / self.total_requests * 100) if self.total_requests > 0 else 0,
             'relationships_found': self.total_relationships_found,
-            'rate_limit': self.rate_limiter.requests_per_minute,
+            'rate_limit': self.config.get_rate_limit(self.name)
-            'cache_hits': self.cache_hits,
-            'cache_misses': self.cache_misses,
-            'cache_hit_rate': (self.cache_hits / (self.cache_hits + self.cache_misses) * 100) if (self.cache_hits + self.cache_misses) > 0 else 0
         }

providers/correlation_provider.py

@@ -0,0 +1,283 @@
+# dnsrecon-reduced/providers/correlation_provider.py
+
+import re
+from typing import Dict, Any, List
+from datetime import datetime, timezone
+
+from .base_provider import BaseProvider
+from core.provider_result import ProviderResult
+from core.graph_manager import NodeType, GraphManager
+
+class CorrelationProvider(BaseProvider):
+    """
+    A provider that finds correlations between nodes in the graph.
+    UPDATED: Enhanced with discovery timestamps for time-based edge coloring.
+    """
+
+    def __init__(self, name: str = "correlation", session_config=None):
+        """
+        Initialize the correlation provider.
+        """
+        super().__init__(name, session_config=session_config)
+        self.graph: GraphManager | None = None
+        self.correlation_index = {}
+        self.date_pattern = re.compile(r'^\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}')
+        self.EXCLUDED_KEYS = [
+            'cert_source',
+            'a_records',
+            'mx_records',
+            'ns_records',
+            'ptr_records',
+            'cert_issuer_ca_id',
+            'cert_common_name',
+            'cert_validity_period_days',
+            'cert_issuer_name',
+            'cert_entry_timestamp',
+            'cert_serial_number', # useless
+            'cert_not_before',
+            'cert_not_after',
+            'dns_ttl',
+            'timestamp',
+            'last_update',
+            'updated_timestamp',
+            'discovery_timestamp',
+            'query_timestamp',
+            'shodan_ip_str',
+            'shodan_a_record',
+        ]
+
+    def get_name(self) -> str:
+        """Return the provider name."""
+        return "correlation"
+
+    def get_display_name(self) -> str:
+        """Return the provider display name for the UI."""
+        return "Correlation Engine"
+
+    def requires_api_key(self) -> bool:
+        """Return True if the provider requires an API key."""
+        return False
+
+    def get_eligibility(self) -> Dict[str, bool]:
+        """Return a dictionary indicating if the provider can query domains and/or IPs."""
+        return {'domains': True, 'ips': True}
+
+    def is_available(self) -> bool:
+        """Check if the provider is available and properly configured."""
+        return True
+
+    def query_domain(self, domain: str) -> ProviderResult:
+        """
+        Query the provider for information about a domain.
+        UPDATED: Enhanced with discovery timestamps for time-based edge coloring.
+        """
+        return self._find_correlations(domain)
+
+    def query_ip(self, ip: str) -> ProviderResult:
+        """
+        Query the provider for information about an IP address.
+        UPDATED: Enhanced with discovery timestamps for time-based edge coloring.
+        """
+        return self._find_correlations(ip)
+
+    def set_graph_manager(self, graph_manager: GraphManager):
+        """
+        Set the graph manager for the provider to use.
+        """
+        self.graph = graph_manager
+
+    def _find_correlations(self, node_id: str) -> ProviderResult:
+        """
+        Find correlations for a given node with enhanced filtering and error handling.
+        UPDATED: Enhanced with discovery timestamps for time-based edge coloring and list value processing.
+        """
+        result = ProviderResult()
+        discovery_time = datetime.now(timezone.utc)
+        
+        # Enhanced safety checks
+        if not self.graph or not self.graph.graph.has_node(node_id):
+            return result
+
+        try:
+            node_attributes = self.graph.graph.nodes[node_id].get('attributes', [])
+            
+            # Ensure attributes is a list (handle legacy data)
+            if not isinstance(node_attributes, list):
+                return result
+                
+            correlations_found = 0
+            
+            for attr in node_attributes:
+                if not isinstance(attr, dict):
+                    continue
+                    
+                attr_name = attr.get('name', '')
+                attr_value = attr.get('value')
+                attr_provider = attr.get('provider', 'unknown')
+
+                # Prepare a list of values to iterate over
+                values_to_process = []
+                if isinstance(attr_value, list):
+                    values_to_process.extend(attr_value)
+                else:
+                    values_to_process.append(attr_value)
+
+                for value_item in values_to_process:
+                    # Enhanced filtering logic
+                    should_exclude = self._should_exclude_attribute(attr_name, value_item)
+                    
+                    if should_exclude:
+                        continue
+
+                    # Build correlation index
+                    if value_item not in self.correlation_index:
+                        self.correlation_index[value_item] = {
+                            'nodes': set(),
+                            'sources': []
+                        }
+
+                    self.correlation_index[value_item]['nodes'].add(node_id)
+
+                    source_info = {
+                        'node_id': node_id,
+                        'provider': attr_provider,
+                        'attribute': attr_name,
+                        'path': f"{attr_provider}_{attr_name}"
+                    }
+
+                    # Avoid duplicate sources
+                    existing_sources = [s for s in self.correlation_index[value_item]['sources']
+                                    if s['node_id'] == node_id and s['path'] == source_info['path']]
+                    if not existing_sources:
+                        self.correlation_index[value_item]['sources'].append(source_info)
+
+                    # Create correlation if we have multiple nodes with this value
+                    if len(self.correlation_index[value_item]['nodes']) > 1:
+                        self._create_correlation_relationships(value_item, self.correlation_index[value_item], result, discovery_time)
+                        correlations_found += 1
+                    
+            # Log correlation results
+            if correlations_found > 0:
+                self.logger.logger.info(f"Found {correlations_found} correlations for node {node_id}")
+                
+        except Exception as e:
+            self.logger.logger.error(f"Error finding correlations for {node_id}: {e}")
+            
+        return result
+    
+    def _should_exclude_attribute(self, attr_name: str, attr_value: Any) -> bool:
+        """
+        Enhanced logic to determine if an attribute should be excluded from correlation.
+        """
+        # Check against excluded keys (exact match or substring)
+        if any(excluded_key in attr_name or attr_name == excluded_key for excluded_key in self.EXCLUDED_KEYS):
+            return True
+        
+        # Value type filtering
+        if not isinstance(attr_value, (str, int, float, bool)) or attr_value is None:
+            return True
+        
+        # Boolean values are not useful for correlation
+        if isinstance(attr_value, bool):
+            return True
+            
+        # String value filtering
+        if isinstance(attr_value, str):                
+            # Date/timestamp strings
+            if self.date_pattern.match(attr_value):
+                return True
+                
+            # Common non-useful values
+            if attr_value.lower() in ['unknown', 'none', 'null', 'n/a', 'true', 'false', '0', '1']:
+                return True
+                
+            # Very long strings that are likely unique (> 100 chars)
+            if len(attr_value) > 100:
+                return True
+        
+        # Numeric value filtering  
+        if isinstance(attr_value, (int, float)):
+            # Very common values
+            if attr_value in [0, 1]:
+                return True
+                
+            # Very large numbers (likely timestamps or unique IDs)
+            if abs(attr_value) > 1000000:
+                return True
+        
+        return False
+    
+    def _create_correlation_relationships(self, value: Any, correlation_data: Dict[str, Any], 
+                                        result: ProviderResult, discovery_time: datetime):
+        """
+        Create correlation relationships with enhanced deduplication and validation.
+        UPDATED: Enhanced with discovery timestamps for time-based edge coloring.
+        """
+        correlation_node_id = f"corr_{hash(str(value)) & 0x7FFFFFFF}"
+        nodes = correlation_data['nodes']
+        sources = correlation_data['sources']
+        
+        # Only create correlations if we have meaningful nodes (more than 1)
+        if len(nodes) < 2:
+            return
+            
+        # Limit correlation size to prevent overly large correlation objects
+        MAX_CORRELATION_SIZE = 50
+        if len(nodes) > MAX_CORRELATION_SIZE:
+            # Sample the nodes to keep correlation manageable
+            import random
+            sampled_nodes = random.sample(list(nodes), MAX_CORRELATION_SIZE)
+            nodes = set(sampled_nodes)
+            # Filter sources to match sampled nodes
+            sources = [s for s in sources if s['node_id'] in nodes]
+
+        # Add the correlation node as an attribute to the result
+        result.add_attribute(
+            target_node=correlation_node_id,
+            name="correlation_value",
+            value=value,
+            attr_type=str(type(value).__name__),
+            provider=self.name,
+            metadata={
+                'correlated_nodes': list(nodes),
+                'sources': sources,
+                'correlation_size': len(nodes),
+                'value_type': type(value).__name__
+            }
+        )
+
+        # Create relationships with source validation and enhanced timestamps
+        created_relationships = set()
+        
+        for source in sources:
+            node_id = source['node_id']
+            provider = source['provider']
+            attribute = source['attribute']
+            
+            # Skip if we've already created this relationship
+            relationship_key = (node_id, correlation_node_id)
+            if relationship_key in created_relationships:
+                continue
+                
+            relationship_label = f"corr_{provider}_{attribute}"
+
+            # Enhanced raw_data with discovery timestamp for time-based edge coloring
+            raw_data = {
+                'correlation_value': value,
+                'original_attribute': attribute,
+                'correlation_type': 'attribute_matching',
+                'correlation_size': len(nodes),
+                'discovery_timestamp': discovery_time.isoformat(),
+                'relevance_timestamp': discovery_time.isoformat()  # Correlation data is "fresh" when discovered
+            }
+
+            # Add the relationship to the result
+            result.add_relationship(
+                source_node=node_id,
+                target_node=correlation_node_id,
+                relationship_type=relationship_label,
+                provider=self.name,
+                raw_data=raw_data
+            )
+            
+            created_relationships.add(relationship_key)

providers/dns_provider.py

@@ -1,19 +1,21 @@
-# dnsrecon/providers/dns_provider.py
+# DNScope/providers/dns_provider.py
 
-import dns.resolver
+from dns import resolver, reversename
-import dns.reversename
+from typing import Dict
-from typing import List, Dict, Any, Tuple
+from datetime import datetime, timezone
 from .base_provider import BaseProvider
-from utils.helpers import _is_valid_ip, _is_valid_domain
+from core.provider_result import ProviderResult
+from utils.helpers import _is_valid_ip, _is_valid_domain, get_ip_version
 
 
 class DNSProvider(BaseProvider):
     """
     Provider for standard DNS resolution and reverse DNS lookups.
-    Now uses session-specific configuration.
+    Now returns standardized ProviderResult objects with IPv4 and IPv6 support.
+    UPDATED: Enhanced with discovery timestamps for time-based edge coloring.
     """
 
-    def __init__(self, session_config=None):
+    def __init__(self, name=None, session_config=None):
         """Initialize DNS provider with session-specific configuration."""
         super().__init__(
             name="dns",
@@ -23,10 +25,9 @@ class DNSProvider(BaseProvider):
         )
 
         # Configure DNS resolver
-        self.resolver = dns.resolver.Resolver()
+        self.resolver = resolver.Resolver()
         self.resolver.timeout = 5
         self.resolver.lifetime = 10
-        #self.resolver.nameservers = ['127.0.0.1']
 
     def get_name(self) -> str:
         """Return the provider name."""
@@ -48,97 +49,150 @@ class DNSProvider(BaseProvider):
         """DNS is always available - no API key required."""
         return True
 
-    def query_domain(self, domain: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]:
+    def query_domain(self, domain: str) -> ProviderResult:
         """
-        Query DNS records for the domain to discover relationships.
+        Query DNS records for the domain to discover relationships and attributes.
-
+        FIXED: Now creates separate attributes for each DNS record type.
+        UPDATED: Enhanced with discovery timestamps for time-based edge coloring.
+        
         Args:
             domain: Domain to investigate
-
+            
         Returns:
-            List of relationships discovered from DNS analysis
+            ProviderResult containing discovered relationships and attributes
         """
         if not _is_valid_domain(domain):
-            return []
+            return ProviderResult()
 
-        relationships = []
+        result = ProviderResult()
+        discovery_time = datetime.now(timezone.utc)
 
-        # Query all record types
+        # Query all record types - each gets its own attribute
         for record_type in ['A', 'AAAA', 'CNAME', 'MX', 'NS', 'SOA', 'TXT', 'SRV', 'CAA']:
-            relationships.extend(self._query_record(domain, record_type))
+            try:
+                self._query_record(domain, record_type, result, discovery_time)
+            #except resolver.NoAnswer:
+                # This is not an error, just a confirmation that the record doesn't exist.
+                #self.logger.logger.debug(f"No {record_type} record found for {domain}")
+            except Exception as e:
+                self.failed_requests += 1
+                self.logger.logger.debug(f"{record_type} record query failed for {domain}: {e}")
 
-        return relationships
+        return result
 
-    def query_ip(self, ip: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]:
+    def query_ip(self, ip: str) -> ProviderResult:
         """
-        Query reverse DNS for the IP address.
+        Query reverse DNS for the IP address (supports both IPv4 and IPv6).
+        UPDATED: Enhanced with discovery timestamps for time-based edge coloring.
 
         Args:
-            ip: IP address to investigate
+            ip: IP address to investigate (IPv4 or IPv6)
 
         Returns:
-            List of relationships discovered from reverse DNS
+            ProviderResult containing discovered relationships and attributes
         """
         if not _is_valid_ip(ip):
-            return []
+            return ProviderResult()
 
-        relationships = []
+        result = ProviderResult()
+        ip_version = get_ip_version(ip)
+        discovery_time = datetime.now(timezone.utc)
 
         try:
-            # Perform reverse DNS lookup
+            # Perform reverse DNS lookup (works for both IPv4 and IPv6)
             self.total_requests += 1
-            reverse_name = dns.reversename.from_address(ip)
+            reverse_name = reversename.from_address(ip)
             response = self.resolver.resolve(reverse_name, 'PTR')
             self.successful_requests += 1
 
+            ptr_records = []
             for ptr_record in response:
                 hostname = str(ptr_record).rstrip('.')
 
                 if _is_valid_domain(hostname):
+                    # Determine appropriate forward relationship type based on IP version
+                    if ip_version == 6:
+                        relationship_type = 'dns_aaaa_record'
+                        record_prefix = 'AAAA'
+                    else:
+                        relationship_type = 'dns_a_record'
+                        record_prefix = 'A'
+                    
+                    # Enhanced raw_data with discovery timestamp for time-based edge coloring
                     raw_data = {
                         'query_type': 'PTR',
                         'ip_address': ip,
+                        'ip_version': ip_version,
                         'hostname': hostname,
-                        'ttl': response.ttl
+                        'ttl': response.ttl,
+                        'discovery_timestamp': discovery_time.isoformat(),
+                        'relevance_timestamp': discovery_time.isoformat()  # DNS data is "fresh" when discovered
                     }
+                    
+                    # Add the relationship
+                    result.add_relationship(
+                        source_node=ip,
+                        target_node=hostname,
+                        relationship_type='dns_ptr_record',
+                        provider=self.name,
+                        raw_data=raw_data
+                    )
 
-                    relationships.append((
+                    # Add to PTR records list
-                        ip,
+                    ptr_records.append(f"PTR: {hostname}")
-                        hostname,
-                        'ptr_record',
-                        0.8,
-                        raw_data
-                    ))
 
+                    # Log the relationship discovery
                     self.log_relationship_discovery(
                         source_node=ip,
                         target_node=hostname,
-                        relationship_type='ptr_record',
+                        relationship_type='dns_ptr_record',
-                        confidence_score=0.8,
                         raw_data=raw_data,
-                        discovery_method="reverse_dns_lookup"
+                        discovery_method=f"reverse_dns_lookup_ipv{ip_version}"
                     )
 
+            # Add PTR records as separate attribute
+            if ptr_records:
+                result.add_attribute(
+                    target_node=ip,
+                    name='ptr_records',  # Specific name for PTR records
+                    value=ptr_records,
+                    attr_type='dns_record',
+                    provider=self.name,
+                    metadata={'ttl': response.ttl, 'ip_version': ip_version}
+                )
+
+        except resolver.NXDOMAIN:
+            self.failed_requests += 1
+            self.logger.logger.debug(f"Reverse DNS lookup failed for {ip}: NXDOMAIN")
         except Exception as e:
             self.failed_requests += 1
             self.logger.logger.debug(f"Reverse DNS lookup failed for {ip}: {e}")
+            # Re-raise the exception so the scanner can handle the failure
+            raise e
 
-        return relationships
+        return result
 
-    def _query_record(self, domain: str, record_type: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]:
+    def _query_record(self, domain: str, record_type: str, result: ProviderResult, discovery_time: datetime) -> None:
         """
-        Query a specific type of DNS record for the domain.
+        FIXED: Query DNS records with unique attribute names for each record type.
+        Enhanced to better handle IPv6 AAAA records.
+        UPDATED: Enhanced with discovery timestamps for time-based edge coloring.
         """
-        relationships = []
         try:
             self.total_requests += 1
             response = self.resolver.resolve(domain, record_type)
             self.successful_requests += 1
 
+            dns_records = []
+            
             for record in response:
                 target = ""
                 if record_type in ['A', 'AAAA']:
                     target = str(record)
+                    # Validate that the IP address is properly formed
+                    if not _is_valid_ip(target):
+                        self.logger.logger.debug(f"Invalid IP address in {record_type} record: {target}")
+                        continue
                 elif record_type in ['CNAME', 'NS', 'PTR']:
                     target = str(record.target).rstrip('.')
                 elif record_type == 'MX':
@@ -146,44 +200,89 @@ class DNSProvider(BaseProvider):
                 elif record_type == 'SOA':
                     target = str(record.mname).rstrip('.')
                 elif record_type in ['TXT']:
-                    # TXT records are treated as metadata, not relationships.
+                    # Keep raw TXT record value
+                    txt_value = str(record).strip('"')
+                    dns_records.append(txt_value)  # Just the value for TXT
                     continue
                 elif record_type == 'SRV':
                     target = str(record.target).rstrip('.')
                 elif record_type == 'CAA':
-                    target = f"{record.flags} {record.tag.decode('utf-8')} \"{record.value.decode('utf-8')}\""
+                    # Keep raw CAA record format
+                    caa_value = f"{record.flags} {record.tag.decode('utf-8')} \"{record.value.decode('utf-8')}\""
+                    dns_records.append(caa_value)  # Just the value for CAA
+                    continue
                 else:
                     target = str(record)
 
                 if target:
+                    # Determine IP version for metadata if this is an IP record
+                    ip_version = None
+                    if record_type in ['A', 'AAAA'] and _is_valid_ip(target):
+                        ip_version = get_ip_version(target)
+                    
+                    # Enhanced raw_data with discovery timestamp for time-based edge coloring
                     raw_data = {
                         'query_type': record_type,
                         'domain': domain,
                         'value': target,
-                        'ttl': response.ttl
+                        'ttl': response.ttl,
+                        'discovery_timestamp': discovery_time.isoformat(),
+                        'relevance_timestamp': discovery_time.isoformat()  # DNS data is "fresh" when discovered
                     }
-                    relationship_type = f"{record_type.lower()}_record"
+                    
-                    confidence = 0.8  # Default confidence for DNS records
+                    if ip_version:
+                        raw_data['ip_version'] = ip_version
+                    
+                    relationship_type = f"dns_{record_type.lower()}_record"
 
-                    relationships.append((
+                    # Add relationship
-                        domain,
+                    result.add_relationship(
-                        target,
+                        source_node=domain,
-                        relationship_type,
+                        target_node=target,
-                        confidence,
+                        relationship_type=relationship_type,
-                        raw_data
+                        provider=self.name,
-                    ))
+                        raw_data=raw_data
+                    )
 
+                    # Add target to records list
+                    dns_records.append(target)
+
+                    # Log relationship discovery with IP version info
+                    discovery_method = f"dns_{record_type.lower()}_record"
+                    if ip_version:
+                        discovery_method += f"_ipv{ip_version}"
+                    
                     self.log_relationship_discovery(
                         source_node=domain,
                         target_node=target,
                         relationship_type=relationship_type,
-                        confidence_score=confidence,
                         raw_data=raw_data,
-                        discovery_method=f"dns_{record_type.lower()}_record"
+                        discovery_method=discovery_method
                     )
 
+            # FIXED: Create attribute with specific name for each record type
+            if dns_records:
+                # Use record type specific attribute name (e.g., 'a_records', 'mx_records', etc.)
+                attribute_name = f"{record_type.lower()}_records"
+                
+                metadata = {'record_type': record_type, 'ttl': response.ttl}
+                
+                # Add IP version info for A/AAAA records
+                if record_type in ['A', 'AAAA'] and dns_records:
+                    first_ip_version = get_ip_version(dns_records[0])
+                    if first_ip_version:
+                        metadata['ip_version'] = first_ip_version
+                
+                result.add_attribute(
+                    target_node=domain,
+                    name=attribute_name,  # UNIQUE name for each record type!
+                    value=dns_records,
+                    attr_type='dns_record_list',
+                    provider=self.name,
+                    metadata=metadata
+                )
+
         except Exception as e:
             self.failed_requests += 1
             self.logger.logger.debug(f"{record_type} record query failed for {domain}: {e}")
-
+            raise e
-        return relationships

providers/shodan_provider.py

@@ -1,21 +1,24 @@
-"""
+# DNScope/providers/shodan_provider.py
-Shodan provider for DNSRecon.
-Discovers IP relationships and infrastructure context through Shodan API.
-"""
 
 import json
-from typing import List, Dict, Any, Tuple
+from pathlib import Path
+from typing import Dict, Any
+from datetime import datetime, timezone
+import requests
+
 from .base_provider import BaseProvider
-from utils.helpers import _is_valid_ip, _is_valid_domain
+from core.provider_result import ProviderResult
+from utils.helpers import _is_valid_ip, _is_valid_domain, get_ip_version, normalize_ip
 
 
 class ShodanProvider(BaseProvider):
     """
-    Provider for querying Shodan API for IP address and hostname information.
+    Provider for querying Shodan API for IP address information.
-    Now uses session-specific API keys.
+    Now returns standardized ProviderResult objects with caching support for IPv4 and IPv6.
+    UPDATED: Enhanced with last_seen timestamp for time-based edge coloring.
     """
 
-    def __init__(self, session_config=None):
+    def __init__(self, name=None, session_config=None):
         """Initialize Shodan provider with session-specific configuration."""
         super().__init__(
             name="shodan",
@@ -25,10 +28,53 @@ class ShodanProvider(BaseProvider):
         )
         self.base_url = "https://api.shodan.io"
         self.api_key = self.config.get_api_key('shodan')
+        
+        # FIXED: Don't fail initialization on connection issues - defer to actual usage
+        self._connection_tested = False
+        self._connection_works = False
+
+        # Initialize cache directory
+        self.cache_dir = Path('cache') / 'shodan'
+        self.cache_dir.mkdir(parents=True, exist_ok=True)
+
+    def _check_api_connection(self) -> bool:
+        """
+        FIXED: Lazy connection checking - only test when actually needed.
+        Don't block provider initialization on network issues.
+        """
+        if self._connection_tested:
+            return self._connection_works
+            
+        if not self.api_key:
+            self._connection_tested = True
+            self._connection_works = False
+            return False
+            
+        try:
+            print(f"Testing Shodan API connection with key: {self.api_key[:8]}...")
+            response = self.session.get(f"{self.base_url}/api-info?key={self.api_key}", timeout=5)
+            self._connection_works = response.status_code == 200
+            print(f"Shodan API test result: {response.status_code} - {'Success' if self._connection_works else 'Failed'}")
+        except requests.exceptions.RequestException as e:
+            print(f"Shodan API connection test failed: {e}")
+            self._connection_works = False
+        finally:
+            self._connection_tested = True
+            
+        return self._connection_works
 
     def is_available(self) -> bool:
-        """Check if Shodan provider is available (has valid API key in this session)."""
+        """
-        return self.api_key is not None and len(self.api_key.strip()) > 0
+        FIXED: Check if Shodan provider is available based on API key presence.
+        Don't require successful connection test during initialization.
+        """
+        has_api_key = self.api_key is not None and len(self.api_key.strip()) > 0
+        
+        if not has_api_key:
+            return False
+            
+        # FIXED: Only test connection on first actual usage, not during initialization
+        return True
 
     def get_name(self) -> str:
         """Return the provider name."""
@@ -36,7 +82,7 @@ class ShodanProvider(BaseProvider):
 
     def get_display_name(self) -> str:
         """Return the provider display name for the UI."""
-        return "shodan"
+        return "Shodan"
 
     def requires_api_key(self) -> bool:
         """Return True if the provider requires an API key."""
@@ -44,267 +90,383 @@ class ShodanProvider(BaseProvider):
 
     def get_eligibility(self) -> Dict[str, bool]:
         """Return a dictionary indicating if the provider can query domains and/or IPs."""
-        return {'domains': True, 'ips': True}
+        return {'domains': False, 'ips': True}
 
-    def query_domain(self, domain: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]:
+    def _get_cache_file_path(self, ip: str) -> Path:
         """
-        Query Shodan for information about a domain.
+        Generate cache file path for an IP address (IPv4 or IPv6).
-        Uses Shodan's hostname search to find associated IPs.
+        IPv6 addresses contain colons which are replaced with underscores for filesystem safety.
-
-        Args:
-            domain: Domain to investigate
-
-        Returns:
-            List of relationships discovered from Shodan data
         """
-        if not _is_valid_domain(domain) or not self.is_available():
+        # Normalize the IP address first to ensure consistent caching
-            return []
+        normalized_ip = normalize_ip(ip)
-
+        if not normalized_ip:
-        relationships = []
+            # Fallback for invalid IPs
+            safe_ip = ip.replace('.', '_').replace(':', '_')
+        else:
+            # Replace problematic characters for both IPv4 and IPv6
+            safe_ip = normalized_ip.replace('.', '_').replace(':', '_')
+        
+        return self.cache_dir / f"{safe_ip}.json"
 
+    def _get_cache_status(self, cache_file_path: Path) -> str:
+        """
+        Check cache status for an IP.
+        Returns: 'not_found', 'fresh', or 'stale'
+        """
+        if not cache_file_path.exists():
+            return "not_found"
+        
         try:
-            # Search for hostname in Shodan
+            with open(cache_file_path, 'r') as f:
-            search_query = f"hostname:{domain}"
+                cache_data = json.load(f)
-            url = f"{self.base_url}/shodan/host/search"
+            
-            params = {
+            last_query_str = cache_data.get("last_upstream_query")
-                'key': self.api_key,
+            if not last_query_str:
-                'query': search_query,
+                return "stale"
-                'minify': True  # Get minimal data to reduce bandwidth
+            
-            }
+            last_query = datetime.fromisoformat(last_query_str.replace('Z', '+00:00'))
+            hours_since_query = (datetime.now(timezone.utc) - last_query).total_seconds() / 3600
+            
+            cache_timeout = self.config.cache_timeout_hours
+            if hours_since_query < cache_timeout:
+                return "fresh"
+            else:
+                return "stale"
+                
+        except (json.JSONDecodeError, ValueError, KeyError):
+            return "stale"
 
-            response = self.make_request(url, method="GET", params=params, target_indicator=domain)
+    def query_domain(self, domain: str) -> ProviderResult:
-
-            if not response or response.status_code != 200:
-                return []
-
-            data = response.json()
-
-            if 'matches' not in data:
-                return []
-
-            # Process search results
-            for match in data['matches']:
-                ip_address = match.get('ip_str')
-                hostnames = match.get('hostnames', [])
-
-                if ip_address and domain in hostnames:
-                    raw_data = {
-                        'ip_address': ip_address,
-                        'hostnames': hostnames,
-                        'country': match.get('location', {}).get('country_name', ''),
-                        'city': match.get('location', {}).get('city', ''),
-                        'isp': match.get('isp', ''),
-                        'org': match.get('org', ''),
-                        'ports': match.get('ports', []),
-                        'last_update': match.get('last_update', '')
-                    }
-
-                    relationships.append((
-                        domain,
-                        ip_address,
-                        'a_record',  # Domain resolves to IP
-                        0.8,
-                        raw_data
-                    ))
-
-                    self.log_relationship_discovery(
-                        source_node=domain,
-                        target_node=ip_address,
-                        relationship_type='a_record',
-                        confidence_score=0.8,
-                        raw_data=raw_data,
-                        discovery_method="shodan_hostname_search"
-                    )
-
-                    # Also create relationships to other hostnames on the same IP
-                    for hostname in hostnames:
-                        if hostname != domain and _is_valid_domain(hostname):
-                            hostname_raw_data = {
-                                'shared_ip': ip_address,
-                                'all_hostnames': hostnames,
-                                'discovery_context': 'shared_hosting'
-                            }
-
-                            relationships.append((
-                                domain,
-                                hostname,
-                                'passive_dns',  # Shared hosting relationship
-                                0.6,  # Lower confidence for shared hosting
-                                hostname_raw_data
-                            ))
-
-                            self.log_relationship_discovery(
-                                source_node=domain,
-                                target_node=hostname,
-                                relationship_type='passive_dns',
-                                confidence_score=0.6,
-                                raw_data=hostname_raw_data,
-                                discovery_method="shodan_shared_hosting"
-                            )
-
-        except json.JSONDecodeError as e:
-            self.logger.logger.error(f"Failed to parse JSON response from Shodan: {e}")
-
-        return relationships
-
-    def query_ip(self, ip: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]:
         """
-        Query Shodan for information about an IP address.
+        Shodan does not support domain queries. This method returns an empty result.
+        """
+        return ProviderResult()
 
+    def query_ip(self, ip: str) -> ProviderResult:
+        """
+        Query Shodan for information about an IP address (IPv4 or IPv6), with caching of processed data.
+        FIXED: Proper 404 handling to prevent unnecessary retries.
+        UPDATED: Enhanced with last_seen timestamp extraction for time-based edge coloring.
+        
         Args:
-            ip: IP address to investigate
+            ip: IP address to investigate (IPv4 or IPv6)
-
+            
         Returns:
-            List of relationships discovered from Shodan IP data
+            ProviderResult containing discovered relationships and attributes
+            
+        Raises:
+            Exception: For temporary failures that should be retried (timeouts, 502/503 errors, connection issues)
         """
-        if not _is_valid_ip(ip) or not self.is_available():
+        if not _is_valid_ip(ip):
-            return []
+            return ProviderResult()
-
+            
-        relationships = []
+        # Test connection only when actually making requests
-
+        if not self._check_api_connection():
+            print(f"Shodan API not available for {ip} - API key: {'present' if self.api_key else 'missing'}")
+            return ProviderResult()
+        
+        # Normalize IP address for consistent processing
+        normalized_ip = normalize_ip(ip)
+        if not normalized_ip:
+            return ProviderResult()
+        
+        cache_file = self._get_cache_file_path(normalized_ip)
+        cache_status = self._get_cache_status(cache_file)
+        
+        if cache_status == "fresh":
+            self.logger.logger.debug(f"Using fresh cache for Shodan query: {normalized_ip}")
+            return self._load_from_cache(cache_file)
+        
+        # Need to query API
+        self.logger.logger.debug(f"Querying Shodan API for: {normalized_ip}")
+        url = f"{self.base_url}/shodan/host/{normalized_ip}"
+        params = {'key': self.api_key}
+        
         try:
-            # Query Shodan host information
+            response = self.make_request(url, method="GET", params=params, target_indicator=normalized_ip)
-            url = f"{self.base_url}/shodan/host/{ip}"
+            
-            params = {'key': self.api_key}
+            if not response:
-
+                self.logger.logger.warning(f"Shodan API unreachable for {normalized_ip} - network failure")
-            response = self.make_request(url, method="GET", params=params, target_indicator=ip)
+                if cache_status == "stale":
-
+                    self.logger.logger.info(f"Using stale cache for {normalized_ip} due to network failure")
-            if not response or response.status_code != 200:
+                    return self._load_from_cache(cache_file)
-                return []
-
-            data = response.json()
-
-            # Extract hostname relationships
-            hostnames = data.get('hostnames', [])
-            for hostname in hostnames:
-                if _is_valid_domain(hostname):
-                    raw_data = {
-                        'ip_address': ip,
-                        'hostname': hostname,
-                        'country': data.get('country_name', ''),
-                        'city': data.get('city', ''),
-                        'isp': data.get('isp', ''),
-                        'org': data.get('org', ''),
-                        'asn': data.get('asn', ''),
-                        'ports': data.get('ports', []),
-                        'last_update': data.get('last_update', ''),
-                        'os': data.get('os', '')
-                    }
-
-                    relationships.append((
-                        ip,
-                        hostname,
-                        'a_record',  # IP resolves to hostname
-                        0.8,
-                        raw_data
-                    ))
-
-                    self.log_relationship_discovery(
-                        source_node=ip,
-                        target_node=hostname,
-                        relationship_type='a_record',
-                        confidence_score=0.8,
-                        raw_data=raw_data,
-                        discovery_method="shodan_host_lookup"
-                    )
-
-            # Extract ASN relationship if available
-            asn = data.get('asn')
-            if asn:
-                # Ensure the ASN starts with "AS"
-                if isinstance(asn, str) and asn.startswith('AS'):
-                    asn_name = asn
-                    asn_number = asn[2:]
                 else:
-                    asn_name = f"AS{asn}"
+                    # FIXED: Treat network failures as "no information" rather than retryable errors
-                    asn_number = str(asn)
+                    self.logger.logger.info(f"No Shodan data available for {normalized_ip} due to network failure")
+                    result = ProviderResult()  # Empty result
+                    network_failure_data = {'shodan_status': 'network_unreachable', 'error': 'API unreachable'}
+                    self._save_to_cache(cache_file, result, network_failure_data)
+                    return result
+            
+            # FIXED: Handle different status codes more precisely
+            if response.status_code == 200:
+                self.logger.logger.debug(f"Shodan returned data for {normalized_ip}")
+                try:
+                    data = response.json()
+                    result = self._process_shodan_data(normalized_ip, data)
+                    self._save_to_cache(cache_file, result, data)
+                    return result
+                except json.JSONDecodeError as e:
+                    self.logger.logger.error(f"Invalid JSON response from Shodan for {normalized_ip}: {e}")
+                    if cache_status == "stale":
+                        return self._load_from_cache(cache_file)
+                    else:
+                        raise requests.exceptions.RequestException("Invalid JSON response from Shodan - should retry")
+            
+            elif response.status_code == 404:
+                # FIXED: 404 = "no information available" - successful but empty result, don't retry
+                self.logger.logger.debug(f"Shodan has no information for {normalized_ip} (404)")
+                result = ProviderResult()  # Empty but successful result
+                # Cache the empty result to avoid repeated queries
+                empty_data = {'shodan_status': 'no_information', 'status_code': 404}
+                self._save_to_cache(cache_file, result, empty_data)
+                return result
+            
+            elif response.status_code in [401, 403]:
+                # Authentication/authorization errors - permanent failures, don't retry
+                self.logger.logger.error(f"Shodan API authentication failed for {normalized_ip} (HTTP {response.status_code})")
+                return ProviderResult()  # Empty result, don't retry
+            
+            elif response.status_code == 429:
+                # Rate limiting - should be handled by rate limiter, but if we get here, retry
+                self.logger.logger.warning(f"Shodan API rate limited for {normalized_ip} (HTTP {response.status_code})")
+                if cache_status == "stale":
+                    self.logger.logger.info(f"Using stale cache for {normalized_ip} due to rate limiting")
+                    return self._load_from_cache(cache_file)
+                else:
+                    raise requests.exceptions.RequestException(f"Shodan API rate limited (HTTP {response.status_code}) - should retry")
+            
+            elif response.status_code in [500, 502, 503, 504]:
+                # Server errors - temporary failures that should be retried
+                self.logger.logger.warning(f"Shodan API server error for {normalized_ip} (HTTP {response.status_code})")
+                if cache_status == "stale":
+                    self.logger.logger.info(f"Using stale cache for {normalized_ip} due to server error")
+                    return self._load_from_cache(cache_file)
+                else:
+                    raise requests.exceptions.RequestException(f"Shodan API server error (HTTP {response.status_code}) - should retry")
+            
+            else:
+                # FIXED: Other HTTP status codes - treat as no information available, don't retry
+                self.logger.logger.info(f"Shodan returned status {response.status_code} for {normalized_ip} - treating as no information")
+                result = ProviderResult()  # Empty result
+                no_info_data = {'shodan_status': 'no_information', 'status_code': response.status_code}
+                self._save_to_cache(cache_file, result, no_info_data)
+                return result
+                
+        except requests.exceptions.Timeout:
+            # Timeout errors - should be retried
+            self.logger.logger.warning(f"Shodan API timeout for {normalized_ip}")
+            if cache_status == "stale":
+                self.logger.logger.info(f"Using stale cache for {normalized_ip} due to timeout")
+                return self._load_from_cache(cache_file)
+            else:
+                raise  # Re-raise timeout for retry
+        
+        except requests.exceptions.ConnectionError:
+            # Connection errors - should be retried
+            self.logger.logger.warning(f"Shodan API connection error for {normalized_ip}")
+            if cache_status == "stale":
+                self.logger.logger.info(f"Using stale cache for {normalized_ip} due to connection error")
+                return self._load_from_cache(cache_file)
+            else:
+                raise  # Re-raise connection error for retry
+        
+        except json.JSONDecodeError:
+            # JSON parsing error - treat as temporary failure
+            self.logger.logger.error(f"Invalid JSON response from Shodan for {normalized_ip}")
+            if cache_status == "stale":
+                self.logger.logger.info(f"Using stale cache for {normalized_ip} due to JSON parsing error")
+                return self._load_from_cache(cache_file)
+            else:
+                raise requests.exceptions.RequestException("Invalid JSON response from Shodan - should retry")
+        
+        # FIXED: Remove the generic RequestException handler that was causing 404s to retry
+        # Now only specific exceptions that should be retried are re-raised
+        
+        except Exception as e:
+            # FIXED: Unexpected exceptions - log but treat as no information available, don't retry
+            self.logger.logger.warning(f"Unexpected exception in Shodan query for {normalized_ip}: {e}")
+            result = ProviderResult()  # Empty result
+            error_data = {'shodan_status': 'error', 'error': str(e)}
+            self._save_to_cache(cache_file, result, error_data)
+            return result
 
-                asn_raw_data = {
+    def _load_from_cache(self, cache_file_path: Path) -> ProviderResult:
-                    'ip_address': ip,
+        """Load processed Shodan data from a cache file."""
-                    'asn': asn_number,
+        try:
-                    'isp': data.get('isp', ''),
+            with open(cache_file_path, 'r') as f:
-                    'org': data.get('org', '')
+                cache_content = json.load(f)
-                }
+            
+            result = ProviderResult()
+            
+            # Reconstruct relationships
+            for rel_data in cache_content.get("relationships", []):
+                result.add_relationship(
+                    source_node=rel_data["source_node"],
+                    target_node=rel_data["target_node"],
+                    relationship_type=rel_data["relationship_type"],
+                    provider=rel_data["provider"],
+                    raw_data=rel_data.get("raw_data", {})
+                )
+            
+            # Reconstruct attributes
+            for attr_data in cache_content.get("attributes", []):
+                result.add_attribute(
+                    target_node=attr_data["target_node"],
+                    name=attr_data["name"],
+                    value=attr_data["value"],
+                    attr_type=attr_data["type"],
+                    provider=attr_data["provider"],
+                    metadata=attr_data.get("metadata", {})
+                )
+            
+            return result
+            
+        except (json.JSONDecodeError, FileNotFoundError, KeyError):
+            return ProviderResult()
 
-                relationships.append((
+    def _save_to_cache(self, cache_file_path: Path, result: ProviderResult, raw_data: Dict[str, Any]) -> None:
-                    ip,
+        """Save processed Shodan data to a cache file."""
-                    asn_name,
+        try:
-                    'asn_membership',
+            cache_data = {
-                    0.7,
+                "last_upstream_query": datetime.now(timezone.utc).isoformat(),
-                    asn_raw_data
+                "raw_data": raw_data,  # Preserve original for forensic purposes
-                ))
+                "relationships": [
+                    {
+                        "source_node": rel.source_node,
+                        "target_node": rel.target_node,
+                        "relationship_type": rel.relationship_type,
+                        "provider": rel.provider,
+                        "raw_data": rel.raw_data
+                    } for rel in result.relationships
+                ],
+                "attributes": [
+                    {
+                        "target_node": attr.target_node,
+                        "name": attr.name,
+                        "value": attr.value,
+                        "type": attr.type,
+                        "provider": attr.provider,
+                        "metadata": attr.metadata
+                    } for attr in result.attributes
+                ]
+            }
+            with open(cache_file_path, 'w') as f:
+                json.dump(cache_data, f, separators=(',', ':'), default=str)
+        except Exception as e:
+            self.logger.logger.warning(f"Failed to save Shodan cache for {cache_file_path.name}: {e}")
 
-                self.log_relationship_discovery(
+    def _process_shodan_data(self, ip: str, data: Dict[str, Any]) -> ProviderResult:
-                    source_node=ip,
+        """
-                    target_node=asn_name,
+        VERIFIED: Process Shodan data creating ISP nodes with ASN attributes and proper relationships.
-                    relationship_type='asn_membership',
+        Enhanced to include IP version information for IPv6 addresses.
-                    confidence_score=0.7,
+        UPDATED: Enhanced with last_seen timestamp for time-based edge coloring.
-                    raw_data=asn_raw_data,
+        """
-                    discovery_method="shodan_asn_lookup"
+        result = ProviderResult()
+        
+        # Determine IP version for metadata
+        ip_version = get_ip_version(ip)
+
+        # Extract last_seen timestamp for time-based edge coloring
+        last_seen = data.get('last_seen')
+        
+        # VERIFIED: Extract ISP information and create proper ISP node with ASN
+        isp_name = data.get('org')
+        asn_value = data.get('asn')
+
+        if isp_name and asn_value:
+            # Enhanced raw_data with last_seen timestamp
+            raw_data = {
+                'asn': asn_value, 
+                'shodan_org': isp_name, 
+                'ip_version': ip_version
+            }
+            
+            # Add last_seen timestamp if available
+            if last_seen:
+                raw_data['last_seen'] = last_seen
+                raw_data['relevance_timestamp'] = last_seen  # Standardized field for time-based coloring
+            
+            # Create relationship from IP to ISP
+            result.add_relationship(
+                source_node=ip,
+                target_node=isp_name,
+                relationship_type='shodan_isp',
+                provider=self.name,
+                raw_data=raw_data
+            )
+            
+            # Add ASN as attribute to the ISP node
+            result.add_attribute(
+                target_node=isp_name,
+                name='asn',
+                value=asn_value,
+                attr_type='isp_info',
+                provider=self.name,
+                metadata={'description': 'Autonomous System Number from Shodan', 'ip_version': ip_version}
+            )
+            
+            # Also add organization name as attribute to ISP node for completeness
+            result.add_attribute(
+                target_node=isp_name,
+                name='organization_name',
+                value=isp_name,
+                attr_type='isp_info',
+                provider=self.name,
+                metadata={'description': 'Organization name from Shodan', 'ip_version': ip_version}
+            )
+
+        # Process hostnames (reverse DNS)
+        for key, value in data.items():
+            if key == 'hostnames':
+                for hostname in value:
+                    if _is_valid_domain(hostname):
+                        # Use appropriate relationship type based on IP version
+                        if ip_version == 6:
+                            relationship_type = 'shodan_aaaa_record'
+                        else:
+                            relationship_type = 'shodan_a_record'
+                        
+                        # Enhanced raw_data with last_seen timestamp
+                        hostname_raw_data = {**data, 'ip_version': ip_version}
+                        if last_seen:
+                            hostname_raw_data['last_seen'] = last_seen
+                            hostname_raw_data['relevance_timestamp'] = last_seen
+                        
+                        result.add_relationship(
+                            source_node=ip,
+                            target_node=hostname,
+                            relationship_type=relationship_type,
+                            provider=self.name,
+                            raw_data=hostname_raw_data
+                        )
+                        self.log_relationship_discovery(
+                            source_node=ip,
+                            target_node=hostname,
+                            relationship_type=relationship_type,
+                            raw_data=hostname_raw_data,
+                            discovery_method=f"shodan_host_lookup_ipv{ip_version}"
+                        )
+            elif key == 'ports':
+                # Add open ports as attributes to the IP
+                for port in value:
+                    result.add_attribute(
+                        target_node=ip,
+                        name='shodan_open_port',
+                        value=port,
+                        attr_type='shodan_network_info',
+                        provider=self.name,
+                        metadata={'ip_version': ip_version}
+                    )
+            elif isinstance(value, (str, int, float, bool)) and value is not None:
+                # Add other Shodan fields as IP attributes (keep raw field names)
+                result.add_attribute(
+                    target_node=ip,
+                    name=key,  # Raw field name from Shodan API
+                    value=value,
+                    attr_type='shodan_info',
+                    provider=self.name,
+                    metadata={'ip_version': ip_version}
                 )
 
-        except json.JSONDecodeError as e:
+        return result
-            self.logger.logger.error(f"Failed to parse JSON response from Shodan: {e}")
-
-        return relationships
-
-    def search_by_organization(self, org_name: str) -> List[Dict[str, Any]]:
-        """
-        Search Shodan for hosts belonging to a specific organization.
-
-        Args:
-            org_name: Organization name to search for
-
-        Returns:
-            List of host information dictionaries
-        """
-        if not self.is_available():
-            return []
-
-        try:
-            search_query = f"org:\"{org_name}\""
-            url = f"{self.base_url}/shodan/host/search"
-            params = {
-                'key': self.api_key,
-                'query': search_query,
-                'minify': True
-            }
-
-            response = self.make_request(url, method="GET", params=params, target_indicator=org_name)
-
-            if response and response.status_code == 200:
-                data = response.json()
-                return data.get('matches', [])
-
-        except Exception as e:
-            self.logger.logger.error(f"Error searching Shodan by organization {org_name}: {e}")
-
-        return []
-
-    def get_host_services(self, ip: str) -> List[Dict[str, Any]]:
-        """
-        Get service information for a specific IP address.
-
-        Args:
-            ip: IP address to query
-
-        Returns:
-            List of service information dictionaries
-        """
-        if not _is_valid_ip(ip) or not self.is_available():
-            return []
-
-        try:
-            url = f"{self.base_url}/shodan/host/{ip}"
-            params = {'key': self.api_key}
-
-            response = self.make_request(url, method="GET", params=params, target_indicator=ip)
-
-            if response and response.status_code == 200:
-                data = response.json()
-                return data.get('data', [])  # Service banners
-
-        except Exception as e:
-            self.logger.logger.error(f"Error getting Shodan services for IP {ip}: {e}")
-
-        return []

requirements.txt

@@ -1,9 +1,11 @@
-Flask>=2.3.3
+Flask
-networkx>=3.1
+networkx
-requests>=2.31.0
+requests
-python-dateutil>=2.8.2
+python-dateutil
-Werkzeug>=2.3.7
+Werkzeug
-urllib3>=2.0.0
+urllib3
-dnspython>=2.4.2
+dnspython
 gunicorn
-redis
+redis
+python-dotenv
+psycopg2-binary

templates/index.html

@@ -1,21 +1,25 @@
 <!DOCTYPE html>
 <html lang="en">
+
 <head>
     <meta charset="UTF-8">
     <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>DNSRecon - Infrastructure Reconnaissance</title>
+    <title>DNScope - Infrastructure Reconnaissance</title>
     <link rel="stylesheet" href="{{ url_for('static', filename='css/main.css') }}">
     <script src="https://cdnjs.cloudflare.com/ajax/libs/vis/4.21.0/vis.min.js"></script>
     <link href="https://cdnjs.cloudflare.com/ajax/libs/vis/4.21.0/vis.min.css" rel="stylesheet" type="text/css">
-    <link href="https://fonts.googleapis.com/css2?family=Roboto+Mono:wght@300;400;500;700&family=Special+Elite&display=swap" rel="stylesheet">
+    <link
+        href="https://fonts.googleapis.com/css2?family=Roboto+Mono:wght@300;400;500;700&family=Special+Elite&display=swap"
+        rel="stylesheet">
 </head>
+
 <body>
     <div class="container">
         <header class="header">
             <div class="header-content">
                 <div class="logo">
-                    <span class="logo-icon">[DNS]</span>
+                    <span class="logo-icon">[DN]</span>
-                    <span class="logo-text">RECON</span>
+                    <span class="logo-text">Scope</span>
                 </div>
                 <div class="status-indicator">
                     <span id="connection-status" class="status-dot"></span>
@@ -29,24 +33,13 @@
                 <div class="panel-header">
                     <h2>Target Configuration</h2>
                 </div>
-                
+
                 <div class="form-container">
                     <div class="input-group">
-                        <label for="target-domain">Target Domain</label>
+                        <label for="target-input">Target Domain or IP</label>
-                        <input type="text" id="target-domain" placeholder="example.com" autocomplete="off">
+                        <input type="text" id="target-input" placeholder="example.com or 8.8.8.8" autocomplete="off">
                     </div>
-                    
+
-                    <div class="input-group">
-                        <label for="max-depth">Recursion Depth</label>
-                        <select id="max-depth">
-                            <option value="1">Depth 1 - Direct relationships</option>
-                            <option value="2" selected>Depth 2 - Recommended</option>
-                            <option value="3">Depth 3 - Extended analysis</option>
-                            <option value="4">Depth 4 - Deep reconnaissance</option>
-                            <option value="5">Depth 5 - Maximum depth</option>
-                        </select>
-                    </div>
-                    
                     <div class="button-group">
                         <button id="start-scan" class="btn btn-primary">
                             <span class="btn-icon">[RUN]</span>
@@ -60,13 +53,13 @@
                             <span class="btn-icon">[STOP]</span>
                             <span>Terminate Scan</span>
                         </button>
-                        <button id="export-results" class="btn btn-secondary">
+                        <button id="export-options" class="btn btn-secondary">
                             <span class="btn-icon">[EXPORT]</span>
-                            <span>Download Results</span>
+                            <span>Export Options</span>
                         </button>
-                        <button id="configure-api-keys" class="btn btn-secondary">
+                        <button id="configure-settings" class="btn btn-secondary">
                             <span class="btn-icon">[API]</span>
-                            <span>Configure API Keys</span>
+                            <span>Settings</span>
                         </button>
                     </div>
                 </div>
@@ -76,7 +69,7 @@
                 <div class="panel-header">
                     <h2>Reconnaissance Status</h2>
                 </div>
-                
+
                 <div class="status-content">
                     <div class="status-row">
                         <span class="status-label">Current Status:</span>
@@ -90,86 +83,80 @@
                         <span class="status-label">Depth:</span>
                         <span id="depth-display" class="status-value">0/0</span>
                     </div>
-                    <div class="status-row">
-                        <span class="status-label">Progress:</span>
-                        <span id="progress-display" class="status-value">0%</span>
-                    </div>
-                    <div class="status-row">
-                        <span class="status-label">Indicators:</span>
-                        <span id="indicators-display" class="status-value">0</span>
-                    </div>
                     <div class="status-row">
                         <span class="status-label">Relationships:</span>
                         <span id="relationships-display" class="status-value">0</span>
                     </div>
                 </div>
-                
+
-                <div class="progress-bar">
+                <div class="progress-container">
-                    <div id="progress-fill" class="progress-fill"></div>
+                    <div class="progress-info">
+                        <span id="progress-label">Progress:</span>
+                        <span id="progress-compact">0/0</span>
+                    </div>
+                    <div class="progress-bar">
+                        <div id="progress-fill" class="progress-fill"></div>
+                    </div>
+                    <div class="progress-placeholder">
+                        <span class="status-label">
+                            ⚠️ <strong>Important:</strong> Scanning large public services (e.g., Google, Cloudflare,
+                            AWS) is
+                            <strong>discouraged</strong> due to rate limits (e.g., crt.sh).
+                            <br><br>
+                            Our task scheduler operates on a <strong>priority-based queue</strong>:
+                            Short, targeted tasks like DNS are processed first, while resource-intensive requests (e.g.,
+                            crt.sh)
+                            are <strong>automatically deprioritized</strong> and may be processed later.
+                        </span>
+                    </div>
                 </div>
             </section>
 
             <section class="visualization-panel">
                 <div class="panel-header">
                     <h2>Infrastructure Map</h2>
-                    <div class="view-controls">
-                        <div class="filter-group">
-                            <label for="node-type-filter">Node Type:</label>
-                            <select id="node-type-filter">
-                                <option value="all">All</option>
-                                <option value="domain">Domain</option>
-                                <option value="ip">IP</option>
-                                <option value="asn">ASN</option>
-                                <option value="correlation_object">Correlation Object</option>
-                                <option value="large_entity">Large Entity</option>
-                            </select>
-                        </div>
-                        <div class="filter-group">
-                            <label for="confidence-filter">Min Confidence:</label>
-                            <input type="range" id="confidence-filter" min="0" max="1" step="0.1" value="0">
-                            <span id="confidence-value">0</span>
-                        </div>
-                    </div>
                 </div>
-                
+
                 <div id="network-graph" class="graph-container">
                     <div class="graph-placeholder">
                         <div class="placeholder-content">
-                            <div class="placeholder-icon">[○]</div>
+                            <div class="placeholder-icon">[◯]</div>
                             <div class="placeholder-text">Infrastructure map will appear here</div>
-                            <div class="placeholder-subtext">Start a reconnaissance scan to visualize relationships</div>
+                            <div class="placeholder-subtext">Start a reconnaissance scan to visualize relationships
+                            </div>
                         </div>
                     </div>
                 </div>
-                
+
                 <div class="legend">
                     <div class="legend-item">
                         <div class="legend-color" style="background-color: #00ff41;"></div>
                         <span>Domains</span>
                     </div>
+                    <div class="legend-item">
+                        <div class="legend-color" style="background-color: #c92f2f;"></div>
+                        <span>Domain (no valid cert)</span>
+                    </div>
+                    <div class="legend-item">
+                        <div class="legend-color" style="background-color: #c7c7c7;"></div>
+                        <span>Domain (never had cert)</span>
+                    </div>
                     <div class="legend-item">
                         <div class="legend-color" style="background-color: #ff9900;"></div>
                         <span>IP Addresses</span>
                     </div>
                     <div class="legend-item">
-                        <div class="legend-color" style="background-color: #c7c7c7;"></div>
+                        <div class="legend-color" style="background-color: #00aaff;"></div>
-                        <span>Domain (invalid cert)</span>
+                        <span>ISPs</span>
-                    </div>
-                    <div class="legend-item">
-                        <div class="legend-color" style="background-color: #9d4edd;"></div>
-                        <span>Correlation Objects</span>
-                    </div>
-                    <div class="legend-item">
-                        <div class="legend-edge high-confidence"></div>
-                        <span>High Confidence</span>
-                    </div>
-                    <div class="legend-item">
-                        <div class="legend-edge medium-confidence"></div>
-                        <span>Medium Confidence</span>
                     </div>
                     <div class="legend-item">
                         <div class="legend-color" style="background-color: #ff6b6b;"></div>
-                        <span>Large Entity</span>
+                        <span>Certificate Authorities</span>
+                    </div>
+
+                    <div class="legend-item">
+                        <div class="legend-color" style="background-color: #9d4edd;"></div>
+                        <span>Correlation Objects</span>
                     </div>
                 </div>
             </section>
@@ -178,9 +165,9 @@
                 <div class="panel-header">
                     <h2>Data Providers</h2>
                 </div>
-                
+
                 <div id="provider-list" class="provider-list">
-                    </div>
+                </div>
             </section>
         </main>
 
@@ -202,49 +189,133 @@
                 </div>
                 <div class="modal-body">
                     <div id="modal-details">
-                        </div>
+                    </div>
                 </div>
             </div>
         </div>
 
-        <div id="api-key-modal" class="modal">
+        <div id="settings-modal" class="modal">
             <div class="modal-content">
                 <div class="modal-header">
-                    <h3>Configure API Keys</h3>
+                    <h3>Scanner Configuration</h3>
-                    <button id="api-key-modal-close" class="modal-close">[×]</button>
+                    <button id="settings-modal-close" class="modal-close">[×]</button>
                 </div>
                 <div class="modal-body">
-                    <p class="modal-description">
+                    <div class="modal-details">
-                        Enter your API keys for enhanced data providers. Keys are stored in memory for the current session only and are never saved to disk.
+                        <section class="modal-section">
-                    </p>
+                            <details open>
-                    <div id="api-key-inputs">
+                                <summary>
+                                    <span>⚙️ Scan Settings</span>
+                                </summary>
+                                <div class="modal-section-content">
+                                    <div class="input-group">
+                                        <label for="max-depth">Recursion Depth</label>
+                                        <select id="max-depth">
+                                            <option value="1">Depth 1 - Direct relationships</option>
+                                            <option value="2" selected>Depth 2 - Recommended</option>
+                                            <option value="3">Depth 3 - Extended analysis</option>
+                                            <option value="4">Depth 4 - Deep reconnaissance</option>
+                                            <option value="5">Depth 5 - Maximum depth</option>
+                                        </select>
+                                    </div>
+                                </div>
+                            </details>
+                        </section>
+
+                        <section class="modal-section">
+                            <details open>
+                                <summary>
+                                    <span>🔧 Provider Configuration</span>
+                                    <span class="merge-badge" id="provider-count">0</span>
+                                </summary>
+                                <div class="modal-section-content">
+                                    <div id="provider-config-list">
+                                        </div>
+                                </div>
+                            </details>
+                        </section>
+
+                        <section class="modal-section">
+                            <details>
+                                <summary>
+                                    <span>🔑 API Keys</span>
+                                    <span class="merge-badge" id="api-key-count">0</span>
+                                </summary>
+                                <div class="modal-section-content">
+                                    <p class="placeholder-subtext" style="margin-bottom: 1rem;">
+                                        ⚠️ API keys are stored in memory for the current session only.
+                                        Only provide API keys you don't use for anything else.
+                                    </p>
+                                    <div id="api-key-inputs">
+                                        </div>
+                                </div>
+                            </details>
+                        </section>
+
+                        <div class="button-group" style="margin-top: 1.5rem;">
+                            <button id="save-settings" class="btn btn-primary">
+                                <span class="btn-icon">[SAVE]</span>
+                                <span>Save Configuration</span>
+                            </button>
+                            <button id="reset-settings" class="btn btn-secondary">
+                                <span class="btn-icon">[RESET]</span>
+                                <span>Reset to Defaults</span>
+                            </button>
                         </div>
-                    <div class="button-group" style="flex-direction: row; justify-content: flex-end;">
+                    </div>
-                        <button id="reset-api-keys" class="btn btn-secondary">
+                </div>
-                            <span>Reset</span>
+            </div>
-                        </button>
+        </div>
-                        <button id="save-api-keys" class="btn btn-primary">
+
-                            <span>Save Keys</span>
+        <div id="export-modal" class="modal">
-                        </button>
+            <div class="modal-content">
+                <div class="modal-header">
+                    <h3>Export Options</h3>
+                    <button id="export-modal-close" class="modal-close">[×]</button>
+                </div>
+                <div class="modal-body">
+                    <div class="modal-details">
+                        <section class="modal-section">
+                            <details open>
+                                <summary>
+                                    <span>📊 Available Exports</span>
+                                </summary>
+                                <div class="modal-section-content">
+                                    <div class="button-group" style="margin-top: 1rem;">
+                                        <button id="export-graph-json" class="btn btn-primary">
+                                            <span class="btn-icon">[JSON]</span>
+                                            <span>Export Graph Data</span>
+                                        </button>
+                                        <div class="status-row" style="margin-top: 0.5rem;">
+                                            <span class="status-label">Complete graph data with forensic audit trail,
+                                                provider statistics, and scan metadata in JSON format for analysis and
+                                                archival.</span>
+                                        </div>
+                                        <button id="export-targets-txt" class="btn btn-primary" style="margin-top: 1rem;">
+                                            <span class="btn-icon">[TXT]</span>
+                                            <span>Export Targets</span>
+                                        </button>
+                                        <div class="status-row" style="margin-top: 0.5rem;">
+                                            <span class="status-label">A simple text file containing all discovered domains and IP addresses.</span>
+                                        </div>
+                                        <button id="export-executive-summary" class="btn btn-primary" style="margin-top: 1rem;">
+                                            <span class="btn-icon">[TXT]</span>
+                                            <span>Export Executive Summary</span>
+                                        </button>
+                                        <div class="status-row" style="margin-top: 0.5rem;">
+                                            <span class="status-label">A natural-language summary of the scan findings.</span>
+                                        </div>
+                                    </div>
+                                </div>
+                            </details>
+                        </section>
                     </div>
                 </div>
             </div>
         </div>
     </div>
-
-    <script>
-        function copyToClipboard(elementId) {
-            const element = document.getElementById(elementId);
-            const textToCopy = element.innerText;
-            navigator.clipboard.writeText(textToCopy).then(() => {
-                // Optional: Show a success message
-                console.log('Copied to clipboard');
-            }).catch(err => {
-                console.error('Failed to copy: ', err);
-            });
-        }
-    </script>
     <script src="{{ url_for('static', filename='js/graph.js') }}"></script>
     <script src="{{ url_for('static', filename='js/main.js') }}"></script>
 </body>
+
 </html>

utils/__init__.py

@@ -0,0 +1,22 @@
+# DNScope-reduced/utils/__init__.py
+
+"""
+Utility modules for DNScope.
+Contains helper functions, export management, and supporting utilities.
+"""
+
+from .helpers import is_valid_target, _is_valid_domain, _is_valid_ip, get_ip_version, normalize_ip
+from .export_manager import export_manager, ExportManager, CustomJSONEncoder
+
+__all__ = [
+    'is_valid_target',
+    '_is_valid_domain', 
+    '_is_valid_ip',
+    'get_ip_version',
+    'normalize_ip',
+    'export_manager',
+    'ExportManager',
+    'CustomJSONEncoder'
+]
+
+__version__ = "1.0.0"

utils/export_manager.py

@@ -0,0 +1,800 @@
+# DNScope-reduced/utils/export_manager.py
+
+"""
+Centralized export functionality for DNScope.
+Handles all data export operations with forensic integrity and proper formatting.
+ENHANCED: Professional forensic executive summary generation for court-ready documentation.
+"""
+
+import json
+from datetime import datetime, timezone
+from typing import Dict, Any, List, Optional, Set, Tuple
+from decimal import Decimal
+from collections import defaultdict, Counter
+import networkx as nx
+
+from utils.helpers import _is_valid_domain, _is_valid_ip
+
+
+class ExportManager:
+    """
+    Centralized manager for all DNScope export operations.
+    Maintains forensic integrity and provides consistent export formats.
+    ENHANCED: Advanced forensic analysis and professional reporting capabilities.
+    """
+    
+    def __init__(self):
+        """Initialize export manager."""
+        pass
+    
+    def export_scan_results(self, scanner) -> Dict[str, Any]:
+        """
+        Export complete scan results with forensic metadata.
+        
+        Args:
+            scanner: Scanner instance with completed scan data
+            
+        Returns:
+            Complete scan results dictionary
+        """
+        graph_data = self.export_graph_json(scanner.graph)
+        audit_trail = scanner.logger.export_audit_trail()
+        provider_stats = {}
+        
+        for provider in scanner.providers:
+            provider_stats[provider.get_name()] = provider.get_statistics()
+        
+        results = {
+            'scan_metadata': {
+                'target_domain': scanner.current_target,
+                'max_depth': scanner.max_depth,
+                'final_status': scanner.status,
+                'total_indicators_processed': scanner.indicators_processed,
+                'enabled_providers': list(provider_stats.keys()),
+                'session_id': scanner.session_id
+            },
+            'graph_data': graph_data,
+            'forensic_audit': audit_trail,
+            'provider_statistics': provider_stats,
+            'scan_summary': scanner.logger.get_forensic_summary()
+        }
+        
+        # Add export metadata
+        results['export_metadata'] = {
+            'export_timestamp': datetime.now(timezone.utc).isoformat(),
+            'export_version': '1.0.0',
+            'forensic_integrity': 'maintained'
+        }
+        
+        return results
+    
+    def export_targets_list(self, scanner) -> str:
+        """
+        Export all discovered domains and IPs as a text file.
+        
+        Args:
+            scanner: Scanner instance with graph data
+            
+        Returns:
+            Newline-separated list of targets
+        """
+        nodes = scanner.graph.get_graph_data().get('nodes', [])
+        targets = {
+            node['id'] for node in nodes 
+            if _is_valid_domain(node['id']) or _is_valid_ip(node['id'])
+        }
+        return "\n".join(sorted(list(targets)))
+    
+    def generate_executive_summary(self, scanner) -> str:
+        """
+        ENHANCED: Generate a comprehensive, court-ready forensic executive summary.
+        
+        Args:
+            scanner: Scanner instance with completed scan data
+            
+        Returns:
+            Professional forensic summary formatted for investigative use
+        """
+        report = []
+        now = datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S UTC')
+        
+        # Get comprehensive data for analysis
+        graph_data = scanner.graph.get_graph_data()
+        nodes = graph_data.get('nodes', [])
+        edges = graph_data.get('edges', [])
+        audit_trail = scanner.logger.export_audit_trail()
+        
+        # Perform advanced analysis
+        infrastructure_analysis = self._analyze_infrastructure_patterns(nodes, edges)
+        
+        # === HEADER AND METADATA ===
+        report.extend([
+            "=" * 80,
+            "DIGITAL INFRASTRUCTURE RECONNAISSANCE REPORT",
+            "=" * 80,
+            "",
+            f"Report Generated: {now}",
+            f"Investigation Target: {scanner.current_target}",
+            f"Analysis Session: {scanner.session_id}",
+            f"Scan Depth: {scanner.max_depth} levels",
+            f"Final Status: {scanner.status.upper()}",
+            ""
+        ])
+        
+        # === EXECUTIVE SUMMARY ===
+        report.extend([
+            "EXECUTIVE SUMMARY",
+            "-" * 40,
+            "",
+            f"This report presents the findings of a comprehensive passive reconnaissance analysis "
+            f"conducted against the target '{scanner.current_target}'. The investigation employed "
+            f"multiple intelligence sources and discovered {len(nodes)} distinct digital entities "
+            f"connected through {len(edges)} verified relationships.",
+            "",
+            f"The analysis reveals a digital infrastructure comprising {infrastructure_analysis['domains']} "
+            f"domain names, {infrastructure_analysis['ips']} IP addresses, and {infrastructure_analysis['isps']} "
+            f"infrastructure service providers. Certificate transparency analysis identified "
+            f"{infrastructure_analysis['cas']} certificate authorities managing the cryptographic "
+            f"infrastructure for the investigated entities.",
+            "",
+        ])
+        
+        # === METHODOLOGY ===
+        report.extend([
+            "INVESTIGATIVE METHODOLOGY",
+            "-" * 40,
+            "",
+            "This analysis employed passive reconnaissance techniques using the following verified data sources:",
+            ""
+        ])
+        
+        provider_info = {
+            'dns': 'Standard DNS resolution and reverse DNS lookups',
+            'crtsh': 'Certificate Transparency database analysis via crt.sh',
+            'shodan': 'Internet-connected device intelligence via Shodan API'
+        }
+        
+        for provider in scanner.providers:
+            provider_name = provider.get_name()
+            stats = provider.get_statistics()
+            description = provider_info.get(provider_name, f'{provider_name} data provider')
+            
+            report.extend([
+                f"• {provider.get_display_name()}: {description}",
+                f"  - Total Requests: {stats['total_requests']}",
+                f"  - Success Rate: {stats['success_rate']:.1f}%",
+                f"  - Relationships Discovered: {stats['relationships_found']}",
+                ""
+            ])
+        
+        # === INFRASTRUCTURE ANALYSIS ===
+        report.extend([
+            "INFRASTRUCTURE ANALYSIS",
+            "-" * 40,
+            ""
+        ])
+        
+        # Domain Analysis
+        if infrastructure_analysis['domains'] > 0:
+            report.extend([
+                f"Domain Name Infrastructure ({infrastructure_analysis['domains']} entities):",
+                ""
+            ])
+            
+            domain_details = self._get_detailed_domain_analysis(nodes, edges)
+            for domain_info in domain_details[:10]:  # Top 10 domains
+                report.extend([
+                    f"• {domain_info['domain']}",
+                    f"  - Type: {domain_info['classification']}",
+                    f"  - Connected IPs: {len(domain_info['ips'])}",
+                    f"  - Certificate Status: {domain_info['cert_status']}",
+                ])
+                
+                if domain_info['security_notes']:
+                    report.extend([
+                        f"  - Security Notes: {', '.join(domain_info['security_notes'])}",
+                    ])
+                report.append("")
+        
+        # IP Address Analysis
+        if infrastructure_analysis['ips'] > 0:
+            report.extend([
+                f"IP Address Infrastructure ({infrastructure_analysis['ips']} entities):",
+                ""
+            ])
+            
+            ip_details = self._get_detailed_ip_analysis(nodes, edges)
+            for ip_info in ip_details[:8]:  # Top 8 IPs
+                report.extend([
+                    f"• {ip_info['ip']} ({ip_info['version']})",
+                    f"  - Associated Domains: {len(ip_info['domains'])}",
+                    f"  - ISP: {ip_info['isp'] or 'Unknown'}",
+                    f"  - Geographic Location: {ip_info['location'] or 'Not determined'}",
+                ])
+                
+                if ip_info['open_ports']:
+                    report.extend([
+                        f"  - Exposed Services: {', '.join(map(str, ip_info['open_ports'][:5]))}"
+                        + (f" (and {len(ip_info['open_ports']) - 5} more)" if len(ip_info['open_ports']) > 5 else ""),
+                    ])
+                report.append("")
+        
+        # === RELATIONSHIP ANALYSIS ===
+        report.extend([
+            "ENTITY RELATIONSHIP ANALYSIS",
+            "-" * 40,
+            ""
+        ])
+        
+        # Network topology insights
+        topology = self._analyze_network_topology(nodes, edges)
+        report.extend([
+            f"Network Topology Assessment:",
+            f"• Central Hubs: {len(topology['hubs'])} entities serve as primary connection points",
+            f"• Isolated Clusters: {len(topology['clusters'])} distinct groupings identified",
+            f"• Relationship Density: {topology['density']:.3f} (0=sparse, 1=fully connected)",
+            f"• Average Path Length: {topology['avg_path_length']:.2f} degrees of separation",
+            ""
+        ])
+        
+        # Key relationships
+        key_relationships = self._identify_key_relationships(edges)
+        if key_relationships:
+            report.extend([
+                "Critical Infrastructure Relationships:",
+                ""
+            ])
+            
+            for rel in key_relationships[:8]:  # Top 8 relationships
+                report.extend([
+                    f"• {rel['source']} → {rel['target']}",
+                    f"  - Relationship: {self._humanize_relationship_type(rel['type'])}",
+                    f"  - Discovery Method: {rel['provider']}",
+                    ""
+                ])
+        
+        # === CERTIFICATE ANALYSIS ===
+        cert_analysis = self._analyze_certificate_infrastructure(nodes)
+        if cert_analysis['total_certs'] > 0:
+            report.extend([
+                "CERTIFICATE INFRASTRUCTURE ANALYSIS",
+                "-" * 40,
+                "",
+                f"Certificate Status Overview:",
+                f"• Total Certificates Analyzed: {cert_analysis['total_certs']}",
+                f"• Valid Certificates: {cert_analysis['valid']}",
+                f"• Expired/Invalid: {cert_analysis['expired']}",
+                f"• Certificate Authorities: {len(cert_analysis['cas'])}",
+                ""
+            ])
+            
+            if cert_analysis['cas']:
+                report.extend([
+                    "Certificate Authority Distribution:",
+                    ""
+                ])
+                for ca, count in cert_analysis['cas'].most_common(5):
+                    report.extend([
+                        f"• {ca}: {count} certificate(s)",
+                    ])
+                report.append("")
+        
+        
+        # === TECHNICAL APPENDIX ===
+        report.extend([
+            "TECHNICAL APPENDIX",
+            "-" * 40,
+            "",
+            "Data Quality Assessment:",
+            f"• Total API Requests: {audit_trail.get('session_metadata', {}).get('total_requests', 0)}",
+            f"• Data Providers Used: {len(audit_trail.get('session_metadata', {}).get('providers_used', []))}",
+        ])
+        
+        correlation_provider = next((p for p in scanner.providers if p.get_name() == 'correlation'), None)
+        correlation_count = len(correlation_provider.correlation_index) if correlation_provider else 0
+        
+        report.extend([
+            "",
+            "Correlation Analysis:",
+            f"• Entity Correlations Identified: {correlation_count}",
+            f"• Cross-Reference Validation: {self._count_cross_validated_relationships(edges)} relationships verified by multiple sources",
+            ""
+        ])
+        
+        # === CONCLUSION ===
+        report.extend([
+            "CONCLUSION",
+            "-" * 40,
+            "",
+            self._generate_conclusion(scanner.current_target, infrastructure_analysis, 
+                                    len(edges)),
+            "",
+            "This analysis was conducted using passive reconnaissance techniques and represents "
+            "the digital infrastructure observable through public data sources at the time of investigation. "
+            "All findings are supported by verifiable technical evidence and documented through "
+            "a complete audit trail maintained for forensic integrity.",
+            "",
+            f"Investigation completed: {now}",
+            f"Report authenticated by: DNScope v{self._get_version()}",
+            "",
+            "=" * 80,
+            "END OF REPORT",
+            "=" * 80
+        ])
+        
+        return "\n".join(report)
+    
+    def _analyze_infrastructure_patterns(self, nodes: List[Dict], edges: List[Dict]) -> Dict[str, Any]:
+        """Analyze infrastructure patterns and classify entities."""
+        analysis = {
+            'domains': len([n for n in nodes if n['type'] == 'domain']),
+            'ips': len([n for n in nodes if n['type'] == 'ip']),
+            'isps': len([n for n in nodes if n['type'] == 'isp']),
+            'cas': len([n for n in nodes if n['type'] == 'ca']),
+            'correlations': len([n for n in nodes if n['type'] == 'correlation_object'])
+        }
+        return analysis
+    
+    def _get_detailed_domain_analysis(self, nodes: List[Dict], edges: List[Dict]) -> List[Dict[str, Any]]:
+        """Generate detailed analysis for each domain."""
+        domain_nodes = [n for n in nodes if n['type'] == 'domain']
+        domain_analysis = []
+        
+        for domain in domain_nodes:
+            # Find connected IPs
+            connected_ips = [e['to'] for e in edges 
+                           if e['from'] == domain['id'] and _is_valid_ip(e['to'])]
+            
+            # Determine classification
+            classification = "Primary Domain"
+            if domain['id'].startswith('www.'):
+                classification = "Web Interface"
+            elif any(subdomain in domain['id'] for subdomain in ['api.', 'mail.', 'smtp.']):
+                classification = "Service Endpoint"
+            elif domain['id'].count('.') > 1:
+                classification = "Subdomain"
+            
+            # Certificate status
+            cert_status = self._determine_certificate_status(domain)
+            
+            # Security notes
+            security_notes = []
+            if cert_status == "Expired/Invalid":
+                security_notes.append("Certificate validation issues")
+            if len(connected_ips) == 0:
+                security_notes.append("No IP resolution found")
+            if len(connected_ips) > 5:
+                security_notes.append("Multiple IP endpoints")
+            
+            domain_edges = [e for e in edges if e['from'] == domain['id']]
+            
+            domain_analysis.append({
+                'domain': domain['id'],
+                'classification': classification,
+                'ips': connected_ips,
+                'cert_status': cert_status,
+                'security_notes': security_notes,
+            })
+        
+        # Sort by number of connections (most connected first)
+        return sorted(domain_analysis, key=lambda x: len(x['ips']), reverse=True)
+    
+    def _get_detailed_ip_analysis(self, nodes: List[Dict], edges: List[Dict]) -> List[Dict[str, Any]]:
+        """Generate detailed analysis for each IP address."""
+        ip_nodes = [n for n in nodes if n['type'] == 'ip']
+        ip_analysis = []
+        
+        for ip in ip_nodes:
+            # Find connected domains
+            connected_domains = [e['from'] for e in edges 
+                               if e['to'] == ip['id'] and _is_valid_domain(e['from'])]
+            
+            # Extract metadata from attributes
+            ip_version = "IPv4"
+            location = None
+            isp = None
+            open_ports = []
+            
+            for attr in ip.get('attributes', []):
+                if attr.get('name') == 'country':
+                    location = attr.get('value')
+                elif attr.get('name') == 'org':
+                    isp = attr.get('value')
+                elif attr.get('name') == 'shodan_open_port':
+                    open_ports.append(attr.get('value'))
+                elif 'ipv6' in str(attr.get('metadata', {})).lower():
+                    ip_version = "IPv6"
+            
+            # Find ISP from relationships
+            if not isp:
+                isp_edges = [e for e in edges if e['from'] == ip['id'] and e['label'].endswith('_isp')]
+                isp = isp_edges[0]['to'] if isp_edges else None
+            
+            ip_analysis.append({
+                'ip': ip['id'],
+                'version': ip_version,
+                'domains': connected_domains,
+                'isp': isp,
+                'location': location,
+                'open_ports': open_ports
+            })
+        
+        # Sort by number of connected domains
+        return sorted(ip_analysis, key=lambda x: len(x['domains']), reverse=True)
+    
+    def _analyze_network_topology(self, nodes: List[Dict], edges: List[Dict]) -> Dict[str, Any]:
+        """Analyze network topology and identify key structural patterns."""
+        if not nodes or not edges:
+            return {'hubs': [], 'clusters': [], 'density': 0, 'avg_path_length': 0}
+        
+        # Create NetworkX graph
+        G = nx.DiGraph()
+        for node in nodes:
+            G.add_node(node['id'])
+        for edge in edges:
+            G.add_edge(edge['from'], edge['to'])
+        
+        # Convert to undirected for certain analyses
+        G_undirected = G.to_undirected()
+        
+        # Identify hubs (nodes with high degree centrality)
+        centrality = nx.degree_centrality(G_undirected)
+        hub_threshold = max(centrality.values()) * 0.7 if centrality else 0
+        hubs = [node for node, cent in centrality.items() if cent >= hub_threshold]
+        
+        # Find connected components (clusters)
+        clusters = list(nx.connected_components(G_undirected))
+        
+        # Calculate density
+        density = nx.density(G_undirected)
+        
+        # Calculate average path length (for largest component)
+        if G_undirected.number_of_nodes() > 1:
+            largest_cc = max(nx.connected_components(G_undirected), key=len)
+            subgraph = G_undirected.subgraph(largest_cc)
+            try:
+                avg_path_length = nx.average_shortest_path_length(subgraph)
+            except:
+                avg_path_length = 0
+        else:
+            avg_path_length = 0
+        
+        return {
+            'hubs': hubs,
+            'clusters': clusters,
+            'density': density,
+            'avg_path_length': avg_path_length
+        }
+    
+    def _identify_key_relationships(self, edges: List[Dict]) -> List[Dict[str, Any]]:
+        """Identify the most significant relationships in the infrastructure."""
+        # Score relationships by type importance
+        relationship_importance = {
+            'dns_a_record': 0.9,
+            'dns_aaaa_record': 0.9,
+            'crtsh_cert_issuer': 0.8,
+            'shodan_isp': 0.8,
+            'crtsh_san_certificate': 0.7,
+            'dns_mx_record': 0.7,
+            'dns_ns_record': 0.7
+        }
+        
+        edges = []
+        for edge in edges:
+            type_weight = relationship_importance.get(edge.get('label', ''), 0.5)
+            
+            edges.append({
+                'source': edge['from'],
+                'target': edge['to'],
+                'type': edge.get('label', ''),
+                'provider': edge.get('source_provider', ''),
+            })
+        
+        # Return top relationships by score
+        return sorted(edges, key=lambda x: x['score'], reverse=True)
+    
+    def _analyze_certificate_infrastructure(self, nodes: List[Dict]) -> Dict[str, Any]:
+        """Analyze certificate infrastructure across all domains."""
+        domain_nodes = [n for n in nodes if n['type'] == 'domain']
+        ca_nodes = [n for n in nodes if n['type'] == 'ca']
+        
+        valid_certs = 0
+        expired_certs = 0
+        total_certs = 0
+        cas = Counter()
+        
+        for domain in domain_nodes:
+            for attr in domain.get('attributes', []):
+                if attr.get('name') == 'cert_is_currently_valid':
+                    total_certs += 1
+                    if attr.get('value') is True:
+                        valid_certs += 1
+                    else:
+                        expired_certs += 1
+                elif attr.get('name') == 'cert_issuer_name':
+                    issuer = attr.get('value')
+                    if issuer:
+                        cas[issuer] += 1
+        
+        return {
+            'total_certs': total_certs,
+            'valid': valid_certs,
+            'expired': expired_certs,
+            'cas': cas
+        }
+    
+    def _has_expired_certificates(self, domain_node: Dict) -> bool:
+        """Check if domain has expired certificates."""
+        for attr in domain_node.get('attributes', []):
+            if (attr.get('name') == 'cert_is_currently_valid' and 
+                attr.get('value') is False):
+                return True
+        return False
+    
+    def _determine_certificate_status(self, domain_node: Dict) -> str:
+        """Determine the certificate status for a domain."""
+        has_valid = False
+        has_expired = False
+        has_any = False
+        
+        for attr in domain_node.get('attributes', []):
+            if attr.get('name') == 'cert_is_currently_valid':
+                has_any = True
+                if attr.get('value') is True:
+                    has_valid = True
+                else:
+                    has_expired = True
+        
+        if not has_any:
+            return "No Certificate Data"
+        elif has_valid and not has_expired:
+            return "Valid"
+        elif has_expired and not has_valid:
+            return "Expired/Invalid"
+        else:
+            return "Mixed Status"
+    
+    def _humanize_relationship_type(self, rel_type: str) -> str:
+        """Convert technical relationship types to human-readable descriptions."""
+        type_map = {
+            'dns_a_record': 'DNS A Record Resolution',
+            'dns_aaaa_record': 'DNS AAAA Record (IPv6) Resolution',
+            'dns_mx_record': 'Email Server (MX) Configuration',
+            'dns_ns_record': 'Name Server Delegation',
+            'dns_cname_record': 'DNS Alias (CNAME) Resolution',
+            'crtsh_cert_issuer': 'SSL Certificate Issuer Relationship',
+            'crtsh_san_certificate': 'Shared SSL Certificate',
+            'shodan_isp': 'Internet Service Provider Assignment',
+            'shodan_a_record': 'IP-to-Domain Resolution (Shodan)',
+            'dns_ptr_record': 'Reverse DNS Resolution'
+        }
+        return type_map.get(rel_type, rel_type.replace('_', ' ').title())
+    
+    def _count_cross_validated_relationships(self, edges: List[Dict]) -> int:
+        """Count relationships verified by multiple providers."""
+        # Group edges by source-target pair
+        edge_pairs = defaultdict(list)
+        for edge in edges:
+            pair_key = f"{edge['from']}->{edge['to']}"
+            edge_pairs[pair_key].append(edge.get('source_provider', ''))
+        
+        # Count pairs with multiple providers
+        cross_validated = 0
+        for pair, providers in edge_pairs.items():
+            if len(set(providers)) > 1:  # Multiple unique providers
+                cross_validated += 1
+        
+        return cross_validated
+    
+    def _generate_security_recommendations(self, infrastructure_analysis: Dict) -> List[str]:
+        """Generate actionable security recommendations."""
+        recommendations = []
+        
+        # Check for complex infrastructure
+        if infrastructure_analysis['ips'] > 10:
+            recommendations.append(
+                "Document and validate the necessity of extensive IP address infrastructure"
+            )
+        
+        if infrastructure_analysis['correlations'] > 5:
+            recommendations.append(
+                "Investigate shared infrastructure components for operational security implications"
+            )
+        
+        if not recommendations:
+            recommendations.append(
+                "Continue monitoring for changes in the identified digital infrastructure"
+            )
+        
+        return recommendations
+    
+    def _generate_conclusion(self, target: str, infrastructure_analysis: Dict, total_relationships: int) -> str:
+        """Generate a professional conclusion for the report."""
+        conclusion_parts = [
+            f"The passive reconnaissance analysis of '{target}' has successfully mapped "
+            f"a digital infrastructure ecosystem consisting of {infrastructure_analysis['domains']} "
+            f"domain names, {infrastructure_analysis['ips']} IP addresses, and "
+            f"{total_relationships} verified inter-entity relationships."
+        ]
+        
+        conclusion_parts.append(
+            "All findings in this report are based on publicly available information and "
+            "passive reconnaissance techniques. The analysis maintains full forensic integrity "
+            "with complete audit trails for all data collection activities."
+        )
+        
+        return " ".join(conclusion_parts)
+    
+    def _count_bidirectional_relationships(self, graph) -> int:
+        """Count bidirectional relationships in the graph."""
+        count = 0
+        for u, v in graph.edges():
+            if graph.has_edge(v, u):
+                count += 1
+        return count // 2  # Each pair counted twice
+    
+    def _identify_hub_nodes(self, graph, nodes: List[Dict]) -> List[str]:
+        """Identify nodes that serve as major hubs in the network."""
+        if not graph.nodes():
+            return []
+        
+        degree_centrality = nx.degree_centrality(graph.to_undirected())
+        threshold = max(degree_centrality.values()) * 0.8 if degree_centrality else 0
+        
+        return [node for node, centrality in degree_centrality.items() 
+                if centrality >= threshold]
+    
+    def _get_version(self) -> str:
+        """Get DNScope version for report authentication."""
+        return "1.0.0-forensic"
+    
+    def export_graph_json(self, graph_manager) -> Dict[str, Any]:
+        """
+        Export complete graph data as a JSON-serializable dictionary.
+        Moved from GraphManager to centralize export functionality.
+        
+        Args:
+            graph_manager: GraphManager instance with graph data
+            
+        Returns:
+            Complete graph data with export metadata
+        """
+        graph_data = nx.node_link_data(graph_manager.graph, edges="edges")
+        
+        return {
+            'export_metadata': {
+                'export_timestamp': datetime.now(timezone.utc).isoformat(),
+                'graph_creation_time': graph_manager.creation_time,
+                'last_modified': graph_manager.last_modified,
+                'total_nodes': graph_manager.get_node_count(),
+                'total_edges': graph_manager.get_edge_count(),
+                'graph_format': 'DNScope_v1_unified_model'
+            },
+            'graph': graph_data,
+            'statistics': graph_manager.get_statistics()
+        }
+    
+    def serialize_to_json(self, data: Dict[str, Any], indent: int = 2) -> str:
+        """
+        Serialize data to JSON with custom handling for non-serializable objects.
+        
+        Args:
+            data: Data to serialize
+            indent: JSON indentation level
+            
+        Returns:
+            JSON string representation
+        """
+        try:
+            return json.dumps(data, indent=indent, cls=CustomJSONEncoder, ensure_ascii=False)
+        except Exception:
+            # Fallback to aggressive cleaning
+            cleaned_data = self._clean_for_json(data)
+            return json.dumps(cleaned_data, indent=indent, ensure_ascii=False)
+    
+    def _clean_for_json(self, obj, max_depth: int = 10, current_depth: int = 0) -> Any:
+        """
+        Recursively clean an object to make it JSON serializable.
+        Handles circular references and problematic object types.
+        
+        Args:
+            obj: Object to clean
+            max_depth: Maximum recursion depth
+            current_depth: Current recursion depth
+            
+        Returns:
+            JSON-serializable object
+        """
+        if current_depth > max_depth:
+            return f"<max_depth_exceeded_{type(obj).__name__}>"
+        
+        if obj is None or isinstance(obj, (bool, int, float, str)):
+            return obj
+        elif isinstance(obj, datetime):
+            return obj.isoformat()
+        elif isinstance(obj, (set, frozenset)):
+            return list(obj)
+        elif isinstance(obj, dict):
+            cleaned = {}
+            for key, value in obj.items():
+                try:
+                    # Ensure key is string
+                    clean_key = str(key) if not isinstance(key, str) else key
+                    cleaned[clean_key] = self._clean_for_json(value, max_depth, current_depth + 1)
+                except Exception:
+                    cleaned[str(key)] = f"<serialization_error_{type(value).__name__}>"
+            return cleaned
+        elif isinstance(obj, (list, tuple)):
+            cleaned = []
+            for item in obj:
+                try:
+                    cleaned.append(self._clean_for_json(item, max_depth, current_depth + 1))
+                except Exception:
+                    cleaned.append(f"<serialization_error_{type(item).__name__}>")
+            return cleaned
+        elif hasattr(obj, '__dict__'):
+            try:
+                return self._clean_for_json(obj.__dict__, max_depth, current_depth + 1)
+            except Exception:
+                return str(obj)
+        elif hasattr(obj, 'value'):
+            # For enum-like objects
+            return obj.value
+        else:
+            return str(obj)
+    
+    def generate_filename(self, target: str, export_type: str, timestamp: Optional[datetime] = None) -> str:
+        """
+        Generate standardized filename for exports.
+        
+        Args:
+            target: Target domain/IP being scanned
+            export_type: Type of export (json, txt, summary)
+            timestamp: Optional timestamp (defaults to now)
+            
+        Returns:
+            Formatted filename with forensic naming convention
+        """
+        if timestamp is None:
+            timestamp = datetime.now(timezone.utc)
+        
+        timestamp_str = timestamp.strftime('%Y%m%d_%H%M%S')
+        safe_target = "".join(c for c in target if c.isalnum() or c in ('-', '_', '.')).rstrip()
+        
+        extension_map = {
+            'json': 'json',
+            'txt': 'txt', 
+            'summary': 'txt',
+            'targets': 'txt'
+        }
+        
+        extension = extension_map.get(export_type, 'txt')
+        return f"DNScope_{export_type}_{safe_target}_{timestamp_str}.{extension}"
+
+
+class CustomJSONEncoder(json.JSONEncoder):
+    """Custom JSON encoder to handle non-serializable objects."""
+    
+    def default(self, obj):
+        if isinstance(obj, datetime):
+            return obj.isoformat()
+        elif isinstance(obj, set):
+            return list(obj)
+        elif isinstance(obj, Decimal):
+            return float(obj)
+        elif hasattr(obj, '__dict__'):
+            # For custom objects, try to serialize their dict representation
+            try:
+                return obj.__dict__
+            except:
+                return str(obj)
+        elif hasattr(obj, 'value') and hasattr(obj, 'name'):
+            # For enum objects
+            return obj.value
+        else:
+            # For any other non-serializable object, convert to string
+            return str(obj)
+
+
+# Global export manager instance
+export_manager = ExportManager()

utils/helpers.py

@@ -1,3 +1,8 @@
+# DNScope-reduced/utils/helpers.py
+
+import ipaddress
+from typing import Union
+
 def _is_valid_domain(domain: str) -> bool:
     """
     Basic domain validation.
@@ -26,25 +31,64 @@ def _is_valid_domain(domain: str) -> bool:
 
 def _is_valid_ip(ip: str) -> bool:
     """
-    Basic IP address validation.
+    IP address validation supporting both IPv4 and IPv6.
 
     Args:
         ip: IP address string to validate
 
     Returns:
-        True if IP appears valid
+        True if IP appears valid (IPv4 or IPv6)
     """
+    if not ip:
+        return False
+    
     try:
-        parts = ip.split('.')
+        # This handles both IPv4 and IPv6 validation
-        if len(parts) != 4:
+        ipaddress.ip_address(ip.strip())
-            return False
-
-        for part in parts:
-            num = int(part)
-            if not 0 <= num <= 255:
-                return False
-
         return True
-
     except (ValueError, AttributeError):
         return False
+
+def is_valid_target(target: str) -> bool:
+    """
+    Checks if the target is a valid domain or IP address (IPv4/IPv6).
+
+    Args:
+        target: The target string to validate.
+
+    Returns:
+        True if the target is a valid domain or IP, False otherwise.
+    """
+    return _is_valid_domain(target) or _is_valid_ip(target)
+
+def get_ip_version(ip: str) -> Union[int, None]:
+    """
+    Get the IP version (4 or 6) of a valid IP address.
+    
+    Args:
+        ip: IP address string
+        
+    Returns:
+        4 for IPv4, 6 for IPv6, None if invalid
+    """
+    try:
+        addr = ipaddress.ip_address(ip.strip())
+        return addr.version
+    except (ValueError, AttributeError):
+        return None
+
+def normalize_ip(ip: str) -> Union[str, None]:
+    """
+    Normalize an IP address to its canonical form.
+    
+    Args:
+        ip: IP address string
+        
+    Returns:
+        Normalized IP address string, None if invalid
+    """
+    try:
+        addr = ipaddress.ip_address(ip.strip())
+        return str(addr)
+    except (ValueError, AttributeError):
+        return None