try_db

.env.example

@@ -0,0 +1,34 @@
+# ===============================================
+# DNSRecon Environment Variables
+# ===============================================
+# Copy this file to .env and fill in your values.
+
+# --- API Keys ---
+# Add your Shodan API key for the Shodan provider to be enabled.
+SHODAN_API_KEY=
+
+# --- Flask & Session Settings ---
+# A strong, random secret key is crucial for session security.
+FLASK_SECRET_KEY=your-very-secret-and-random-key-here
+FLASK_HOST=127.0.0.1
+FLASK_PORT=5000
+FLASK_DEBUG=True
+# How long a user's session in the browser lasts (in hours).
+FLASK_PERMANENT_SESSION_LIFETIME_HOURS=2
+# How long inactive scanner data is stored in Redis (in minutes).
+SESSION_TIMEOUT_MINUTES=60
+
+
+# --- Application Core Settings ---
+# The default number of levels to recurse when scanning.
+DEFAULT_RECURSION_DEPTH=2
+# Default timeout for provider API requests in seconds.
+DEFAULT_TIMEOUT=30
+# The number of concurrent provider requests to make.
+MAX_CONCURRENT_REQUESTS=5
+# The number of results from a provider that triggers the "large entity" grouping.
+LARGE_ENTITY_THRESHOLD=100
+# The number of times to retry a target if a provider fails.
+MAX_RETRIES_PER_TARGET=3
+# How long cached provider responses are stored (in hours).
+CACHE_EXPIRY_HOURS=12

.gitignore

@@ -168,3 +168,4 @@ cython_debug/
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
 
+dump.rdb

README.md

@@ -1,107 +1,256 @@
-# DNS Reconnaissance Tool
+# DNSRecon - Passive Infrastructure Reconnaissance Tool
 
-A comprehensive DNS reconnaissance tool designed for investigators to gather intelligence on hostnames and IP addresses through multiple data sources.
+DNSRecon is an interactive, passive reconnaissance tool designed to map adversary infrastructure. It operates on a "free-by-default" model, ensuring core functionality without subscriptions, while allowing power users to enhance its capabilities with paid API keys.
+
+**Current Status: Phase 2 Implementation**
+
+  - ✅ Core infrastructure and graph engine
+  - ✅ Multi-provider support (crt.sh, DNS, Shodan)
+  - ✅ Session-based multi-user support
+  - ✅ Real-time web interface with interactive visualization
+  - ✅ Forensic logging system and JSON export
 
 ## Features
 
-- **DNS Resolution**: Query multiple DNS servers (1.1.1.1, 8.8.8.8, 9.9.9.9)
-- **TLD Expansion**: Automatically try all IANA TLDs for hostname-only inputs
-- **Certificate Transparency**: Query crt.sh for SSL certificate information
-- **Recursive Discovery**: Automatically discover and analyze subdomains
-- **External Intelligence**: Optional Shodan and VirusTotal integration
-- **Multiple Interfaces**: Both CLI and web interface available
-- **Comprehensive Reports**: JSON and text output formats
+  - **Passive Reconnaissance**: Gathers data without direct contact with target infrastructure.
+  - **In-Memory Graph Analysis**: Uses NetworkX for efficient relationship mapping.
+  - **Real-Time Visualization**: The graph updates dynamically as the scan progresses.
+  - **Forensic Logging**: A complete audit trail of all reconnaissance activities is maintained.
+  - **Confidence Scoring**: Relationships are weighted based on the reliability of the data source.
+  - **Session Management**: Supports concurrent user sessions with isolated scanner instances.
 
 ## Installation
 
-```bash
-# Clone or create the project structure
-mkdir dns-recon-tool && cd dns-recon-tool
+### Prerequisites
 
-# Install dependencies
+  - Python 3.8 or higher
+  - A modern web browser with JavaScript enabled
+  - (Recommended) A Linux host for running the application and the optional DNS cache.
+
+### 1\. Clone the Project
+
+```bash
+git clone https://github.com/your-repo/dnsrecon.git
+cd dnsrecon
+```
+
+### 2\. Install Python Dependencies
+
+It is highly recommended to use a virtual environment:
+
+```bash
+python3 -m venv venv
+source venv/bin/activate
 pip install -r requirements.txt
 ```
 
-## Usage
+### 3\. (Optional but Recommended) Set up a Local DNS Caching Resolver
 
-### Command Line Interface
+Running a local DNS caching resolver can significantly speed up DNS queries and reduce your network footprint. Here’s how to set up `unbound` on a Debian-based Linux distribution (like Ubuntu).
+
+**a. Install Unbound:**
 
 ```bash
-# Basic domain scan
-python -m src.main example.com
-
-# Try all TLDs for hostname
-python -m src.main example
-
-# With API keys and custom depth
-python -m src.main example.com --shodan-key YOUR_KEY --virustotal-key YOUR_KEY --max-depth 3
-
-# Save reports
-python -m src.main example.com --output results
-
-# JSON only output
-python -m src.main example.com --json-only
+sudo apt update
+sudo apt install unbound -y
 ```
 
-### Web Interface
+**b. Configure Unbound:**
+Create a new configuration file for DNSRecon:
 
 ```bash
-# Start web server
-python -m src.main --web
-
-# Custom port
-python -m src.main --web --port 8080
+sudo nano /etc/unbound/unbound.conf.d/dnsrecon.conf
 ```
 
-Then open http://localhost:5000 in your browser.
-
-## Configuration
-
-The tool uses the following default settings:
-- DNS Servers: 1.1.1.1, 8.8.8.8, 9.9.9.9
-- Max Recursion Depth: 2
-- Rate Limits: DNS (10/s), crt.sh (2/s), Shodan (0.5/s), VirusTotal (0.25/s)
-
-## API Keys
-
-For enhanced reconnaissance, obtain API keys from:
-- [Shodan](https://shodan.io) - Port scanning and service detection
-- [VirusTotal](https://virustotal.com) - Security analysis and reputation
-
-## Output
-
-The tool generates two types of reports:
-
-### JSON Report
-Complete machine-readable data including:
-- All discovered hostnames and IPs
-- DNS records by type
-- Certificate information
-- External service results
-- Metadata and timing
-
-### Text Report
-Human-readable summary with:
-- Executive summary
-- Hostnames by discovery depth
-- IP address analysis
-- DNS record details
-- Certificate analysis
-- Security findings
-
-## Architecture
+Add the following content to the file:
 
 ```
-src/
-├── main.py              # CLI entry point
-├── web_app.py           # Flask web interface  
-├── config.py            # Configuration management
-├── data_structures.py   # Data models
-├── dns_resolver.py      # DNS functionality
-├── certificate_checker.py # crt.sh integration
-├── shodan_client.py     # Shodan API
-├── virustotal_client.py # VirusTotal API
-├── tld_fetcher.py       # IANA TLD handling
-├── reconnaissance.py    # Main logic
-└── report_generator.py  # Report generation
-```
+server:
+    # Listen on localhost for all users
+    interface: 127.0.0.1
+    access-control: 0.0.0.0/0 refuse
+    access-control: 127.0.0.0/8 allow
+
+    # Enable prefetching of popular items
+    prefetch: yes
+```
+
+**c. Restart Unbound and set it as the default resolver:**
+
+```bash
+sudo systemctl restart unbound
+sudo systemctl enable unbound
+```
+
+To use this resolver for your system, you may need to update your network settings to point to `127.0.0.1` as your DNS server.
+
+**d. Update DNSProvider to use the local resolver:**
+In `dnsrecon/providers/dns_provider.py`, you can explicitly set the resolver's nameservers in the `__init__` method:
+
+```python
+# dnsrecon/providers/dns_provider.py
+
+class DNSProvider(BaseProvider):
+    def __init__(self, session_config=None):
+        """Initialize DNS provider with session-specific configuration."""
+        super().__init__(...)
+
+        # Configure DNS resolver
+        self.resolver = dns.resolver.Resolver()
+        self.resolver.nameservers = ['127.0.0.1'] # Use local caching resolver
+        self.resolver.timeout = 5
+        self.resolver.lifetime = 10
+```
+
+## Usage (Development)
+
+### 1\. Start the Application
+
+```bash
+python app.py
+```
+
+### 2\. Open Your Browser
+
+Navigate to `http://127.0.0.1:5000`.
+
+### 3\. Basic Reconnaissance Workflow
+
+1.  **Enter Target Domain**: Input a domain like `example.com`.
+2.  **Select Recursion Depth**: Depth 2 is recommended for most investigations.
+3.  **Start Reconnaissance**: Click "Start Reconnaissance" to begin.
+4.  **Monitor Progress**: Watch the real-time graph build as relationships are discovered.
+5.  **Analyze and Export**: Interact with the graph and download the results when the scan is complete.
+
+## Production Deployment
+
+To deploy DNSRecon in a production environment, follow these steps:
+
+### 1\. Use a Production WSGI Server
+
+Do not use the built-in Flask development server for production. Use a WSGI server like **Gunicorn**:
+
+```bash
+pip install gunicorn
+gunicorn --workers 4 --bind 0.0.0.0:5000 app:app
+```
+
+### 2\. Configure Environment Variables
+
+Set the following environment variables for a secure and configurable deployment:
+
+```bash
+# Generate a strong, random secret key
+export SECRET_KEY='your-super-secret-and-random-key'
+
+# Set Flask to production mode
+export FLASK_ENV='production'
+export FLASK_DEBUG=False
+
+# API keys (optional, but recommended for full functionality)
+export SHODAN_API_KEY="your_shodan_key"
+```
+
+### 3\. Use a Reverse Proxy
+
+Set up a reverse proxy like **Nginx** to sit in front of the Gunicorn server. This provides several benefits, including:
+
+  - **TLS/SSL Termination**: Securely handle HTTPS traffic.
+  - **Load Balancing**: Distribute traffic across multiple application instances.
+  - **Serving Static Files**: Efficiently serve CSS and JavaScript files.
+
+**Example Nginx Configuration:**
+
+```nginx
+server {
+    listen 80;
+    server_name your_domain.com;
+
+    location / {
+        return 301 https://$host$request_uri;
+    }
+}
+
+server {
+    listen 443 ssl;
+    server_name your_domain.com;
+
+    # SSL cert configuration
+    ssl_certificate /etc/letsencrypt/live/your_domain.com/fullchain.pem;
+    ssl_certificate_key /etc/letsencrypt/live/your_domain.com/privkey.pem;
+
+    location / {
+        proxy_pass http://127.0.0.1:5000;
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    }
+
+    location /static {
+        alias /path/to/your/dnsrecon/static;
+        expires 30d;
+    }
+}
+```
+
+## Autostart with systemd
+
+To run DNSRecon as a service that starts automatically on boot, you can use `systemd`.
+
+### 1\. Create a `.service` file
+
+Create a new service file in `/etc/systemd/system/`:
+
+```bash
+sudo nano /etc/systemd/system/dnsrecon.service
+```
+
+### 2\. Add the Service Configuration
+
+Paste the following configuration into the file. **Remember to replace `/path/to/your/dnsrecon` and `your_user` with your actual project path and username.**
+
+```ini
+[Unit]
+Description=DNSRecon Application
+After=network.target
+
+[Service]
+User=your_user
+Group=your_user
+WorkingDirectory=/path/to/your/dnsrecon
+ExecStart=/path/to/your/dnsrecon/venv/bin/gunicorn --workers 4 --bind 0.0.0.0:5000 app:app
+Restart=always
+Environment="SECRET_KEY=your-super-secret-and-random-key"
+Environment="FLASK_ENV=production"
+Environment="FLASK_DEBUG=False"
+Environment="SHODAN_API_KEY=your_shodan_key"
+
+[Install]
+WantedBy=multi-user.target
+```
+
+### 3\. Enable and Start the Service
+
+Reload the `systemd` daemon, enable the service to start on boot, and then start it immediately:
+
+```bash
+sudo systemctl daemon-reload
+sudo systemctl enable dnsrecon.service
+sudo systemctl start dnsrecon.service
+```
+
+You can check the status of the service at any time with:
+
+```bash
+sudo systemctl status dnsrecon.service
+```
+
+## Security Considerations
+
+  - **API Keys**: API keys are stored in memory for the duration of a user session and are not written to disk.
+  - **Rate Limiting**: DNSRecon includes built-in rate limiting to be respectful to data sources.
+  - **Local Use**: The application is designed for local or trusted network use and does not have built-in authentication. **Do not expose it directly to the internet without proper security controls.**
+
+## License
+
+This project is licensed under the terms of the license agreement found in the `LICENSE` file.

app.py

@@ -0,0 +1,446 @@
+# dnsrecon-reduced/app.py
+
+"""
+Flask application entry point for DNSRecon web interface.
+Provides REST API endpoints and serves the web interface with user session support.
+"""
+
+import json
+import traceback
+from flask import Flask, render_template, request, jsonify, send_file, session
+from datetime import datetime, timezone, timedelta
+import io
+
+from core.session_manager import session_manager
+from config import config
+
+
+app = Flask(__name__)
+# Use centralized configuration for Flask settings
+app.config['SECRET_KEY'] = config.flask_secret_key
+app.config['PERMANENT_SESSION_LIFETIME'] = timedelta(hours=config.flask_permanent_session_lifetime_hours)
+
+def get_user_scanner():
+    """
+    Retrieves the scanner for the current session, or creates a new
+    session and scanner if one doesn't exist.
+    """
+    # Get current Flask session info for debugging
+    current_flask_session_id = session.get('dnsrecon_session_id')
+    
+    # Try to get existing session
+    if current_flask_session_id:
+        existing_scanner = session_manager.get_session(current_flask_session_id)
+        if existing_scanner:
+            return current_flask_session_id, existing_scanner
+    
+    # Create new session if none exists
+    print("Creating new session as none was found...")
+    new_session_id = session_manager.create_session()
+    new_scanner = session_manager.get_session(new_session_id)
+    
+    if not new_scanner:
+        raise Exception("Failed to create new scanner session")
+    
+    # Store in Flask session
+    session['dnsrecon_session_id'] = new_session_id
+    session.permanent = True
+    
+    return new_session_id, new_scanner
+
+@app.route('/')
+def index():
+    """Serve the main web interface."""
+    return render_template('index.html')
+
+
+@app.route('/api/scan/start', methods=['POST'])
+def start_scan():
+    """
+    Start a new reconnaissance scan. Creates a new isolated scanner if
+    clear_graph is true, otherwise adds to the existing one.
+    """
+    print("=== API: /api/scan/start called ===")
+    
+    try:
+        data = request.get_json()
+        if not data or 'target_domain' not in data:
+            return jsonify({'success': False, 'error': 'Missing target_domain in request'}), 400
+        
+        target_domain = data['target_domain'].strip()
+        max_depth = data.get('max_depth', config.default_recursion_depth)
+        clear_graph = data.get('clear_graph', True)
+        
+        print(f"Parsed - target_domain: '{target_domain}', max_depth: {max_depth}, clear_graph: {clear_graph}")
+        
+        # Validation
+        if not target_domain:
+            return jsonify({'success': False, 'error': 'Target domain cannot be empty'}), 400
+        if not isinstance(max_depth, int) or not 1 <= max_depth <= 5:
+            return jsonify({'success': False, 'error': 'Max depth must be an integer between 1 and 5'}), 400
+        
+        user_session_id, scanner = None, None
+
+        if clear_graph:
+            print("Clear graph requested: Creating a new, isolated scanner session.")
+            old_session_id = session.get('dnsrecon_session_id')
+            if old_session_id:
+                session_manager.terminate_session(old_session_id)
+            
+            user_session_id = session_manager.create_session()
+            session['dnsrecon_session_id'] = user_session_id
+            session.permanent = True
+            scanner = session_manager.get_session(user_session_id)
+        else:
+            print("Adding to existing graph: Reusing the current scanner session.")
+            user_session_id, scanner = get_user_scanner()
+
+        if not scanner:
+            return jsonify({'success': False, 'error': 'Failed to get or create a scanner instance.'}), 500
+        
+        print(f"Using scanner {id(scanner)} in session {user_session_id}")
+        
+        success = scanner.start_scan(target_domain, max_depth, clear_graph=clear_graph)
+        
+        if success:
+            return jsonify({
+                'success': True,
+                'message': 'Scan started successfully',
+                'scan_id': scanner.logger.session_id,
+                'user_session_id': user_session_id,
+            })
+        else:
+            return jsonify({
+                'success': False,
+                'error': f'Failed to start scan (scanner status: {scanner.status})',
+            }), 409
+        
+    except Exception as e:
+        print(f"ERROR: Exception in start_scan endpoint: {e}")
+        traceback.print_exc()
+        return jsonify({'success': False, 'error': f'Internal server error: {str(e)}'}), 500
+
+@app.route('/api/scan/stop', methods=['POST'])
+def stop_scan():
+    """Stop the current scan with immediate GUI feedback."""
+    print("=== API: /api/scan/stop called ===")
+    
+    try:
+        # Get user-specific scanner
+        user_session_id, scanner = get_user_scanner()
+        print(f"Stopping scan for session: {user_session_id}")
+        
+        if not scanner:
+            return jsonify({
+                'success': False,
+                'error': 'No scanner found for session'
+            }), 404
+        
+        # Ensure session ID is set
+        if not scanner.session_id:
+            scanner.session_id = user_session_id
+        
+        # Use the stop mechanism
+        success = scanner.stop_scan()
+        
+        # Also set the Redis stop signal directly for extra reliability
+        session_manager.set_stop_signal(user_session_id)
+        
+        # Force immediate status update
+        session_manager.update_scanner_status(user_session_id, 'stopped')
+        
+        # Update the full scanner state
+        session_manager.update_session_scanner(user_session_id, scanner)
+        
+        print(f"Stop scan completed. Success: {success}, Scanner status: {scanner.status}")
+        
+        return jsonify({
+            'success': True,
+            'message': 'Scan stop requested - termination initiated',
+            'user_session_id': user_session_id,
+            'scanner_status': scanner.status,
+            'stop_method': 'cross_process'
+        })
+        
+    except Exception as e:
+        print(f"ERROR: Exception in stop_scan endpoint: {e}")
+        traceback.print_exc()
+        return jsonify({
+            'success': False,
+            'error': f'Internal server error: {str(e)}'
+        }), 500
+
+
+@app.route('/api/scan/status', methods=['GET'])
+def get_scan_status():
+    """Get current scan status with error handling."""
+    try:
+        # Get user-specific scanner
+        user_session_id, scanner = get_user_scanner()
+        
+        if not scanner:
+            # Return default idle status if no scanner
+            return jsonify({
+                'success': True,
+                'status': {
+                    'status': 'idle',
+                    'target_domain': None,
+                    'current_depth': 0,
+                    'max_depth': 0,
+                    'current_indicator': '',
+                    'total_indicators_found': 0,
+                    'indicators_processed': 0,
+                    'progress_percentage': 0.0,
+                    'enabled_providers': [],
+                    'graph_statistics': {},
+                    'user_session_id': user_session_id
+                }
+            })
+        
+        # Ensure session ID is set
+        if not scanner.session_id:
+            scanner.session_id = user_session_id
+        
+        status = scanner.get_scan_status()
+        status['user_session_id'] = user_session_id
+        
+        # Additional debug info
+        status['debug_info'] = {
+            'scanner_object_id': id(scanner),
+            'session_id_set': bool(scanner.session_id),
+            'has_scan_thread': bool(scanner.scan_thread and scanner.scan_thread.is_alive())
+        }
+        
+        return jsonify({
+            'success': True,
+            'status': status
+        })
+    
+    except Exception as e:
+        print(f"ERROR: Exception in get_scan_status endpoint: {e}")
+        traceback.print_exc()
+        return jsonify({
+            'success': False,
+            'error': f'Internal server error: {str(e)}',
+            'fallback_status': {
+                'status': 'error',
+                'target_domain': None,
+                'current_depth': 0,
+                'max_depth': 0,
+                'progress_percentage': 0.0
+            }
+        }), 500
+
+
+
+@app.route('/api/graph', methods=['GET'])
+def get_graph_data():
+    """Get current graph data with error handling."""
+    try:
+        # Get user-specific scanner
+        user_session_id, scanner = get_user_scanner()
+        
+        if not scanner:
+            # Return empty graph if no scanner
+            return jsonify({
+                'success': True,
+                'graph': {
+                    'nodes': [],
+                    'edges': [],
+                    'statistics': {
+                        'node_count': 0,
+                        'edge_count': 0,
+                        'creation_time': datetime.now(timezone.utc).isoformat(),
+                        'last_modified': datetime.now(timezone.utc).isoformat()
+                    }
+                },
+                'user_session_id': user_session_id
+            })
+        
+        graph_data = scanner.get_graph_data()
+        return jsonify({
+            'success': True,
+            'graph': graph_data,
+            'user_session_id': user_session_id
+        })
+    
+    except Exception as e:
+        print(f"ERROR: Exception in get_graph_data endpoint: {e}")
+        traceback.print_exc()
+        return jsonify({
+            'success': False,
+            'error': f'Internal server error: {str(e)}',
+            'fallback_graph': {
+                'nodes': [],
+                'edges': [],
+                'statistics': {'node_count': 0, 'edge_count': 0}
+            }
+        }), 500
+
+
+
+@app.route('/api/export', methods=['GET'])
+def export_results():
+    """Export complete scan results as downloadable JSON for the user session."""
+    try:
+        # Get user-specific scanner
+        user_session_id, scanner = get_user_scanner()
+        
+        # Get complete results
+        results = scanner.export_results()
+        
+        # Add session information to export
+        results['export_metadata'] = {
+            'user_session_id': user_session_id,
+            'export_timestamp': datetime.now(timezone.utc).isoformat(),
+            'export_type': 'user_session_results'
+        }
+        
+        # Create filename with timestamp
+        timestamp = datetime.now(timezone.utc).strftime('%Y%m%d_%H%M%S')
+        target = scanner.current_target or 'unknown'
+        filename = f"dnsrecon_{target}_{timestamp}_{user_session_id[:8]}.json"
+        
+        # Create in-memory file
+        json_data = json.dumps(results, indent=2, ensure_ascii=False)
+        file_obj = io.BytesIO(json_data.encode('utf-8'))
+        
+        return send_file(
+            file_obj,
+            as_attachment=True,
+            download_name=filename,
+            mimetype='application/json'
+        )
+    
+    except Exception as e:
+        print(f"ERROR: Exception in export_results endpoint: {e}")
+        traceback.print_exc()
+        return jsonify({
+            'success': False,
+            'error': f'Export failed: {str(e)}'
+        }), 500
+
+
+@app.route('/api/providers', methods=['GET'])
+def get_providers():
+    """Get information about available providers for the user session."""
+    
+    try:
+        # Get user-specific scanner
+        user_session_id, scanner = get_user_scanner()
+        
+        if scanner:
+            completed_tasks = scanner.indicators_completed
+            enqueued_tasks = len(scanner.task_queue)
+            print(f"DEBUG: Tasks - Completed: {completed_tasks}, Enqueued: {enqueued_tasks}")
+        else:
+            print("DEBUG: No active scanner session found.")
+
+        provider_info = scanner.get_provider_info()
+        
+        return jsonify({
+            'success': True,
+            'providers': provider_info,
+            'user_session_id': user_session_id
+        })
+    
+    except Exception as e:
+        print(f"ERROR: Exception in get_providers endpoint: {e}")
+        traceback.print_exc()
+        return jsonify({
+            'success': False,
+            'error': f'Internal server error: {str(e)}'
+        }), 500
+
+
+@app.route('/api/config/api-keys', methods=['POST'])
+def set_api_keys():
+    """
+    Set API keys for providers for the user session only.
+    """
+    try:
+        data = request.get_json()
+        
+        if data is None:
+            return jsonify({
+                'success': False,
+                'error': 'No API keys provided'
+            }), 400
+        
+        # Get user-specific scanner and config
+        user_session_id, scanner = get_user_scanner()
+        session_config = scanner.config
+        
+        updated_providers = []
+        
+        # Iterate over the API keys provided in the request data
+        for provider_name, api_key in data.items():
+            # This allows us to both set and clear keys. The config
+            # handles enabling/disabling based on if the key is empty.
+            api_key_value = str(api_key or '').strip()
+            success = session_config.set_api_key(provider_name.lower(), api_key_value)
+            
+            if success:
+                updated_providers.append(provider_name)
+        
+        if updated_providers:
+            # Reinitialize scanner providers to apply the new keys
+            scanner._initialize_providers()
+            
+            # Persist the updated scanner object back to the user's session
+            session_manager.update_session_scanner(user_session_id, scanner)
+            
+            return jsonify({
+                'success': True,
+                'message': f'API keys updated for session {user_session_id}: {", ".join(updated_providers)}',
+                'updated_providers': updated_providers,
+                'user_session_id': user_session_id
+            })
+        else:
+            return jsonify({
+                'success': False,
+                'error': 'No valid API keys were provided or provider names were incorrect.'
+            }), 400
+    
+    except Exception as e:
+        print(f"ERROR: Exception in set_api_keys endpoint: {e}")
+        traceback.print_exc()
+        return jsonify({
+            'success': False,
+            'error': f'Internal server error: {str(e)}'
+        }), 500
+
+@app.errorhandler(404)
+def not_found(error):
+    """Handle 404 errors."""
+    return jsonify({
+        'success': False,
+        'error': 'Endpoint not found'
+    }), 404
+
+
+@app.errorhandler(500)
+def internal_error(error):
+    """Handle 500 errors."""
+    print(f"ERROR: 500 Internal Server Error: {error}")
+    traceback.print_exc()
+    return jsonify({
+        'success': False,
+        'error': 'Internal server error'
+    }), 500
+
+
+if __name__ == '__main__':
+    print("Starting DNSRecon Flask application with user session support...")
+    
+    # Load configuration from environment
+    config.load_from_env()
+    
+    # Start Flask application
+    print(f"Starting server on {config.flask_host}:{config.flask_port}")
+    app.run(
+        host=config.flask_host,
+        port=config.flask_port,
+        debug=config.flask_debug,
+        threaded=True
+    )

config.py

@@ -0,0 +1,101 @@
+"""
+Configuration management for DNSRecon tool.
+Handles API key storage, rate limiting, and default settings.
+"""
+
+import os
+from typing import Dict, Optional
+from dotenv import load_dotenv
+
+# Load environment variables from .env file
+load_dotenv()
+
+class Config:
+    """Configuration manager for DNSRecon application."""
+    
+    def __init__(self):
+        """Initialize configuration with default values."""
+        self.api_keys: Dict[str, Optional[str]] = {}
+        
+        # --- General Settings ---
+        self.default_recursion_depth = 2
+        self.default_timeout = 15
+        self.max_concurrent_requests = 5
+        self.large_entity_threshold = 100
+        self.max_retries_per_target = 3
+        self.cache_expiry_hours = 12
+        
+        # --- Provider Caching Settings ---
+        self.cache_timeout_hours = 6  # Provider-specific cache timeout
+        
+        # --- Rate Limiting (requests per minute) ---
+        self.rate_limits = {
+            'crtsh': 30,
+            'shodan': 60,
+            'dns': 100
+        }
+        
+        # --- Provider Settings ---
+        self.enabled_providers = {
+            'crtsh': True,
+            'dns': True,
+            'shodan': False
+        }
+        
+        # --- Logging ---
+        self.log_level = 'INFO'
+        self.log_format = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+        
+        # --- Flask & Session Settings ---
+        self.flask_host = '127.0.0.1'
+        self.flask_port = 5000
+        self.flask_debug = True
+        self.flask_secret_key = 'default-secret-key-change-me'
+        self.flask_permanent_session_lifetime_hours = 2
+        self.session_timeout_minutes = 60
+        
+        # Load environment variables to override defaults
+        self.load_from_env()
+        
+    def load_from_env(self):
+        """Load configuration from environment variables."""
+        self.set_api_key('shodan', os.getenv('SHODAN_API_KEY'))
+
+        # Override settings from environment
+        self.default_recursion_depth = int(os.getenv('DEFAULT_RECURSION_DEPTH', self.default_recursion_depth))
+        self.default_timeout = int(os.getenv('DEFAULT_TIMEOUT', self.default_timeout))
+        self.max_concurrent_requests = int(os.getenv('MAX_CONCURRENT_REQUESTS', self.max_concurrent_requests))
+        self.large_entity_threshold = int(os.getenv('LARGE_ENTITY_THRESHOLD', self.large_entity_threshold))
+        self.max_retries_per_target = int(os.getenv('MAX_RETRIES_PER_TARGET', self.max_retries_per_target))
+        self.cache_expiry_hours = int(os.getenv('CACHE_EXPIRY_HOURS', self.cache_expiry_hours))
+        self.cache_timeout_hours = int(os.getenv('CACHE_TIMEOUT_HOURS', self.cache_timeout_hours))
+        
+        # Override Flask and session settings
+        self.flask_host = os.getenv('FLASK_HOST', self.flask_host)
+        self.flask_port = int(os.getenv('FLASK_PORT', self.flask_port))
+        self.flask_debug = os.getenv('FLASK_DEBUG', str(self.flask_debug)).lower() == 'true'
+        self.flask_secret_key = os.getenv('FLASK_SECRET_KEY', self.flask_secret_key)
+        self.flask_permanent_session_lifetime_hours = int(os.getenv('FLASK_PERMANENT_SESSION_LIFETIME_HOURS', self.flask_permanent_session_lifetime_hours))
+        self.session_timeout_minutes = int(os.getenv('SESSION_TIMEOUT_MINUTES', self.session_timeout_minutes))
+
+    def set_api_key(self, provider: str, api_key: Optional[str]) -> bool:
+        """Set API key for a provider."""
+        self.api_keys[provider] = api_key
+        if api_key:
+            self.enabled_providers[provider] = True
+        return True
+    
+    def get_api_key(self, provider: str) -> Optional[str]:
+        """Get API key for a provider."""
+        return self.api_keys.get(provider)
+    
+    def is_provider_enabled(self, provider: str) -> bool:
+        """Check if a provider is enabled."""
+        return self.enabled_providers.get(provider, False)
+    
+    def get_rate_limit(self, provider: str) -> int:
+        """Get rate limit for a provider."""
+        return self.rate_limits.get(provider, 60)
+
+# Global configuration instance
+config = Config()

core/__init__.py

@@ -0,0 +1,25 @@
+"""
+Core modules for DNSRecon passive reconnaissance tool.
+Contains graph management, scanning orchestration, and forensic logging.
+"""
+
+from .graph_manager import GraphManager, NodeType
+from .scanner import Scanner, ScanStatus  
+from .logger import ForensicLogger, get_forensic_logger, new_session
+from .session_manager import session_manager 
+from .session_config import SessionConfig, create_session_config 
+
+__all__ = [
+    'GraphManager',
+    'NodeType', 
+    'Scanner',
+    'ScanStatus',
+    'ForensicLogger',
+    'get_forensic_logger',
+    'new_session',
+    'session_manager', 
+    'SessionConfig',
+    'create_session_config'
+]
+
+__version__ = "1.0.0-phase2"

core/graph_manager.py

@@ -0,0 +1,527 @@
+# core/graph_manager.py
+
+"""
+Graph data model for DNSRecon using NetworkX.
+Manages in-memory graph storage with confidence scoring and forensic metadata.
+"""
+import re
+from datetime import datetime, timezone
+from enum import Enum
+from typing import Dict, List, Any, Optional, Tuple
+
+import networkx as nx
+
+
+class NodeType(Enum):
+    """Enumeration of supported node types."""
+    DOMAIN = "domain"
+    IP = "ip"
+    ASN = "asn"
+    LARGE_ENTITY = "large_entity"
+    CORRELATION_OBJECT = "correlation_object"
+
+    def __repr__(self):
+        return self.value
+
+
+class GraphManager:
+    """
+    Thread-safe graph manager for DNSRecon infrastructure mapping.
+    Uses NetworkX for in-memory graph storage with confidence scoring.
+    """
+
+    def __init__(self):
+        """Initialize empty directed graph."""
+        self.graph = nx.DiGraph()
+        self.creation_time = datetime.now(timezone.utc).isoformat()
+        self.last_modified = self.creation_time
+        self.correlation_index = {}
+        # Compile regex for date filtering for efficiency
+        self.date_pattern = re.compile(r'^\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}')
+
+    def __getstate__(self):
+        """Prepare GraphManager for pickling, excluding compiled regex."""
+        state = self.__dict__.copy()
+        # Compiled regex patterns are not always picklable
+        if 'date_pattern' in state:
+            del state['date_pattern']
+        return state
+
+    def __setstate__(self, state):
+        """Restore GraphManager state and recompile regex."""
+        self.__dict__.update(state)
+        self.date_pattern = re.compile(r'^\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}')
+
+    def _update_correlation_index(self, node_id: str, data: Any, path: List[str] = [], parent_attr: str = ""):
+        """Recursively traverse metadata and add hashable values to the index with better path tracking."""
+        if path is None:
+            path = []
+
+        if isinstance(data, dict):
+            for key, value in data.items():
+                self._update_correlation_index(node_id, value, path + [key], key)
+        elif isinstance(data, list):
+            for i, item in enumerate(data):
+                # Instead of just using [i], include the parent attribute context
+                list_path_component = f"[{i}]" if not parent_attr else f"{parent_attr}[{i}]"
+                self._update_correlation_index(node_id, item, path + [list_path_component], parent_attr)
+        else:
+            self._add_to_correlation_index(node_id, data, ".".join(path), parent_attr)
+
+    def _add_to_correlation_index(self, node_id: str, value: Any, path_str: str, parent_attr: str = ""):
+        """Add a hashable value to the correlation index, filtering out noise."""
+        if not isinstance(value, (str, int, float, bool)) or value is None:
+            return
+
+        # Ignore certain paths that contain noisy, non-unique identifiers
+        if any(keyword in path_str.lower() for keyword in ['count', 'total', 'timestamp', 'date']):
+            return
+
+        # Filter out common low-entropy values and date-like strings
+        if isinstance(value, str):
+            # FIXED: Prevent correlation on date/time strings.
+            if self.date_pattern.match(value):
+                return
+            if len(value) < 4 or value.lower() in ['true', 'false', 'unknown', 'none', 'crt.sh']:
+                return
+        elif isinstance(value, int) and (abs(value) < 1024 or abs(value) > 65535):
+            return  # Ignore small integers and common port numbers
+        elif isinstance(value, bool):
+            return  # Ignore boolean values
+
+        # Add the valuable correlation data to the index
+        if value not in self.correlation_index:
+            self.correlation_index[value] = {}
+        if node_id not in self.correlation_index[value]:
+            self.correlation_index[value][node_id] = []
+        
+        # Store both the full path and the parent attribute for better edge labeling
+        correlation_entry = {
+            'path': path_str,
+            'parent_attr': parent_attr,
+            'meaningful_attr': self._extract_meaningful_attribute(path_str, parent_attr)
+        }
+        
+        if correlation_entry not in self.correlation_index[value][node_id]:
+            self.correlation_index[value][node_id].append(correlation_entry)
+
+    def _extract_meaningful_attribute(self, path_str: str, parent_attr: str = "") -> str:
+        """Extract the most meaningful attribute name from a path string."""
+        if not path_str:
+            return "unknown"
+        
+        path_parts = path_str.split('.')
+        
+        # Look for the last non-array-index part
+        for part in reversed(path_parts):
+            # Skip array indices like [0], [1], etc.
+            if not (part.startswith('[') and part.endswith(']') and part[1:-1].isdigit()):
+                # Clean up compound names like "hostnames[0]" to just "hostnames"
+                clean_part = re.sub(r'\[\d+\]$', '', part)
+                if clean_part:
+                    return clean_part
+        
+        # Fallback to parent attribute if available
+        if parent_attr:
+            return parent_attr
+        
+        # Last resort - use the first meaningful part
+        for part in path_parts:
+            if not (part.startswith('[') and part.endswith(']') and part[1:-1].isdigit()):
+                clean_part = re.sub(r'\[\d+\]$', '', part)
+                if clean_part:
+                    return clean_part
+        
+        return "correlation"
+
+    def _check_for_correlations(self, new_node_id: str, data: Any, path: List[str] = [], parent_attr: str = "") -> List[Dict]:
+        """Recursively traverse metadata to find correlations with existing data."""
+        if path is None:
+            path = []
+
+        all_correlations = []
+        if isinstance(data, dict):
+            for key, value in data.items():
+                if key == 'source':  # Avoid correlating on the provider name
+                    continue
+                all_correlations.extend(self._check_for_correlations(new_node_id, value, path + [key], key))
+        elif isinstance(data, list):
+            for i, item in enumerate(data):
+                list_path_component = f"[{i}]" if not parent_attr else f"{parent_attr}[{i}]"
+                all_correlations.extend(self._check_for_correlations(new_node_id, item, path + [list_path_component], parent_attr))
+        else:
+            value = data
+            if value in self.correlation_index:
+                existing_nodes_with_paths = self.correlation_index[value]
+                unique_nodes = set(existing_nodes_with_paths.keys())
+                unique_nodes.add(new_node_id)
+
+                if len(unique_nodes) < 2:
+                    return all_correlations # Correlation must involve at least two distinct nodes
+
+                new_source = {
+                    'node_id': new_node_id, 
+                    'path': ".".join(path),
+                    'parent_attr': parent_attr,
+                    'meaningful_attr': self._extract_meaningful_attribute(".".join(path), parent_attr)
+                }
+                all_sources = [new_source]
+                
+                for node_id, path_entries in existing_nodes_with_paths.items():
+                    for entry in path_entries:
+                        if isinstance(entry, dict):
+                            all_sources.append({
+                                'node_id': node_id,
+                                'path': entry['path'],
+                                'parent_attr': entry.get('parent_attr', ''),
+                                'meaningful_attr': entry.get('meaningful_attr', self._extract_meaningful_attribute(entry['path'], entry.get('parent_attr', '')))
+                            })
+                        else:
+                            # Handle legacy string-only entries
+                            all_sources.append({
+                                'node_id': node_id,
+                                'path': str(entry),
+                                'parent_attr': '',
+                                'meaningful_attr': self._extract_meaningful_attribute(str(entry))
+                            })
+
+                all_correlations.append({
+                    'value': value,
+                    'sources': all_sources,
+                    'nodes': list(unique_nodes)
+                })
+        return all_correlations
+
+    def add_node(self, node_id: str, node_type: NodeType, attributes: Optional[Dict[str, Any]] = None,
+                description: str = "", metadata: Optional[Dict[str, Any]] = None) -> bool:
+        """Add a node to the graph, update attributes, and process correlations."""
+        is_new_node = not self.graph.has_node(node_id)
+        if is_new_node:
+            self.graph.add_node(node_id, type=node_type.value,
+                                added_timestamp=datetime.now(timezone.utc).isoformat(),
+                                attributes=attributes or {},
+                                description=description,
+                                metadata=metadata or {})
+        else:
+            # Safely merge new attributes into existing attributes
+            if attributes:
+                existing_attributes = self.graph.nodes[node_id].get('attributes', {})
+                existing_attributes.update(attributes)
+                self.graph.nodes[node_id]['attributes'] = existing_attributes
+            if description:
+                self.graph.nodes[node_id]['description'] = description
+            if metadata:
+                existing_metadata = self.graph.nodes[node_id].get('metadata', {})
+                existing_metadata.update(metadata)
+                self.graph.nodes[node_id]['metadata'] = existing_metadata
+
+        if attributes and node_type != NodeType.CORRELATION_OBJECT:
+            correlations = self._check_for_correlations(node_id, attributes)
+            for corr in correlations:
+                value = corr['value']
+                
+                # STEP 1: Substring check against all existing nodes
+                if self._correlation_value_matches_existing_node(value):
+                    # Skip creating correlation node - would be redundant
+                    continue
+                
+                eligible_nodes = set(corr['nodes'])
+                
+                if len(eligible_nodes) < 2:
+                    # Need at least 2 nodes to create a correlation
+                    continue
+                    
+                # STEP 3: Check for existing correlation node with same connection pattern
+                correlation_nodes_with_pattern = self._find_correlation_nodes_with_same_pattern(eligible_nodes)
+                
+                if correlation_nodes_with_pattern:
+                    # STEP 4: Merge with existing correlation node
+                    target_correlation_node = correlation_nodes_with_pattern[0]
+                    self._merge_correlation_values(target_correlation_node, value, corr)
+                else:
+                    # STEP 5: Create new correlation node for eligible nodes only
+                    correlation_node_id = f"corr_{abs(hash(str(sorted(eligible_nodes))))}"
+                    self.add_node(correlation_node_id, NodeType.CORRELATION_OBJECT,
+                                metadata={'values': [value], 'sources': corr['sources'],
+                                            'correlated_nodes': list(eligible_nodes)})
+                    
+                    # Create edges from eligible nodes to this correlation node with better labeling
+                    for c_node_id in eligible_nodes:
+                        if self.graph.has_node(c_node_id):
+                            # Find the best attribute name for this node
+                            meaningful_attr = self._find_best_attribute_name_for_node(c_node_id, corr['sources'])
+                            relationship_type = f"c_{meaningful_attr}"
+                            self.add_edge(c_node_id, correlation_node_id, relationship_type, confidence_score=0.9)
+
+            self._update_correlation_index(node_id, attributes)
+
+        self.last_modified = datetime.now(timezone.utc).isoformat()
+        return is_new_node
+
+    def _find_best_attribute_name_for_node(self, node_id: str, sources: List[Dict]) -> str:
+        """Find the best attribute name for a correlation edge by looking at the sources."""
+        node_sources = [s for s in sources if s['node_id'] == node_id]
+        
+        if not node_sources:
+            return "correlation"
+        
+        # Use the meaningful_attr if available
+        for source in node_sources:
+            meaningful_attr = source.get('meaningful_attr')
+            if meaningful_attr and meaningful_attr != "unknown":
+                return meaningful_attr
+        
+        # Fallback to parent_attr
+        for source in node_sources:
+            parent_attr = source.get('parent_attr')
+            if parent_attr:
+                return parent_attr
+        
+        # Last resort - extract from path
+        for source in node_sources:
+            path = source.get('path', '')
+            if path:
+                extracted = self._extract_meaningful_attribute(path)
+                if extracted != "unknown":
+                    return extracted
+        
+        return "correlation"
+
+    def _has_direct_edge_bidirectional(self, node_a: str, node_b: str) -> bool:
+        """
+        Check if there's a direct edge between two nodes in either direction.
+        Returns True if node_a→node_b OR node_b→node_a exists.
+        """
+        return (self.graph.has_edge(node_a, node_b) or 
+                self.graph.has_edge(node_b, node_a))
+
+    def _correlation_value_matches_existing_node(self, correlation_value: str) -> bool:
+        """
+        Check if correlation value contains any existing node ID as substring.
+        Returns True if match found (correlation node should NOT be created).
+        """
+        correlation_str = str(correlation_value).lower()
+        
+        # Check against all existing nodes
+        for existing_node_id in self.graph.nodes():
+            if existing_node_id.lower() in correlation_str:
+                return True
+        
+        return False
+
+    def _find_correlation_nodes_with_same_pattern(self, node_set: set) -> List[str]:
+        """
+        Find existing correlation nodes that have the exact same pattern of connected nodes.
+        Returns list of correlation node IDs with matching patterns.
+        """
+        correlation_nodes = self.get_nodes_by_type(NodeType.CORRELATION_OBJECT)
+        matching_nodes = []
+        
+        for corr_node_id in correlation_nodes:
+            # Get all nodes connected to this correlation node
+            connected_nodes = set()
+            
+            # Add all predecessors (nodes pointing TO the correlation node)
+            connected_nodes.update(self.graph.predecessors(corr_node_id))
+            
+            # Add all successors (nodes pointed TO by the correlation node)  
+            connected_nodes.update(self.graph.successors(corr_node_id))
+            
+            # Check if the pattern matches exactly
+            if connected_nodes == node_set:
+                matching_nodes.append(corr_node_id)
+        
+        return matching_nodes
+
+    def _merge_correlation_values(self, target_node_id: str, new_value: Any, corr_data: Dict) -> None:
+        """
+        Merge a new correlation value into an existing correlation node.
+        Uses same logic as large entity merging.
+        """
+        if not self.graph.has_node(target_node_id):
+            return
+            
+        target_metadata = self.graph.nodes[target_node_id]['metadata']
+        
+        # Get existing values (ensure it's a list)
+        existing_values = target_metadata.get('values', [])
+        if not isinstance(existing_values, list):
+            existing_values = [existing_values]
+        
+        # Add new value if not already present
+        if new_value not in existing_values:
+            existing_values.append(new_value)
+            
+        # Merge sources
+        existing_sources = target_metadata.get('sources', [])
+        new_sources = corr_data.get('sources', [])
+        
+        # Create set of unique sources based on (node_id, path) tuples
+        source_set = set()
+        for source in existing_sources + new_sources:
+            source_tuple = (source['node_id'], source.get('path', ''))
+            source_set.add(source_tuple)
+        
+        # Convert back to list of dictionaries
+        merged_sources = [{'node_id': nid, 'path': path} for nid, path in source_set]
+        
+        # Update metadata
+        target_metadata.update({
+            'values': existing_values,
+            'sources': merged_sources,
+            'correlated_nodes': list(set(target_metadata.get('correlated_nodes', []) + corr_data.get('nodes', []))),
+            'merge_count': len(existing_values),
+            'last_merge_timestamp': datetime.now(timezone.utc).isoformat()
+        })
+        
+        # Update description to reflect merged nature
+        value_count = len(existing_values)
+        node_count = len(target_metadata['correlated_nodes'])
+        self.graph.nodes[target_node_id]['description'] = (
+            f"Correlation container with {value_count} merged values "
+            f"across {node_count} nodes"
+        )
+        
+    def add_edge(self, source_id: str, target_id: str, relationship_type: str,
+                 confidence_score: float = 0.5, source_provider: str = "unknown",
+                 raw_data: Optional[Dict[str, Any]] = None) -> bool:
+        """Add or update an edge between two nodes, ensuring nodes exist."""
+        if not self.graph.has_node(source_id) or not self.graph.has_node(target_id):
+            return False
+
+        new_confidence = confidence_score
+        
+        if relationship_type.startswith("c_"):
+            edge_label = relationship_type
+        else:
+            edge_label = f"{source_provider}_{relationship_type}"
+        
+        if self.graph.has_edge(source_id, target_id):
+            # If edge exists, update confidence if the new score is higher.
+            if new_confidence > self.graph.edges[source_id, target_id].get('confidence_score', 0):
+                self.graph.edges[source_id, target_id]['confidence_score'] = new_confidence
+                self.graph.edges[source_id, target_id]['updated_timestamp'] = datetime.now(timezone.utc).isoformat()
+                self.graph.edges[source_id, target_id]['updated_by'] = source_provider
+            return False
+
+        # Add a new edge with all attributes.
+        self.graph.add_edge(source_id, target_id,
+                            relationship_type=edge_label,
+                            confidence_score=new_confidence,
+                            source_provider=source_provider,
+                            discovery_timestamp=datetime.now(timezone.utc).isoformat(),
+                            raw_data=raw_data or {})
+        self.last_modified = datetime.now(timezone.utc).isoformat()
+        return True
+
+    def get_node_count(self) -> int:
+        """Get total number of nodes in the graph."""
+        return self.graph.number_of_nodes()
+
+    def get_edge_count(self) -> int:
+        """Get total number of edges in the graph."""
+        return self.graph.number_of_edges()
+
+    def get_nodes_by_type(self, node_type: NodeType) -> List[str]:
+        """Get all nodes of a specific type."""
+        return [n for n, d in self.graph.nodes(data=True) if d.get('type') == node_type.value]
+
+    def get_neighbors(self, node_id: str) -> List[str]:
+        """Get all unique neighbors (predecessors and successors) for a node."""
+        if not self.graph.has_node(node_id):
+            return []
+        return list(set(self.graph.predecessors(node_id)) | set(self.graph.successors(node_id)))
+
+    def get_high_confidence_edges(self, min_confidence: float = 0.8) -> List[Tuple[str, str, Dict]]:
+        """Get edges with confidence score above a given threshold."""
+        return [(u, v, d) for u, v, d in self.graph.edges(data=True)
+                if d.get('confidence_score', 0) >= min_confidence]
+
+    def get_graph_data(self) -> Dict[str, Any]:
+        """Export graph data formatted for frontend visualization."""
+        nodes = []
+        for node_id, attrs in self.graph.nodes(data=True):
+            node_data = {'id': node_id, 'label': node_id, 'type': attrs.get('type', 'unknown'),
+                         'attributes': attrs.get('attributes', {}),
+                         'description': attrs.get('description', ''),
+                         'metadata': attrs.get('metadata', {}),
+                         'added_timestamp': attrs.get('added_timestamp')}
+            # Customize node appearance based on type and attributes
+            node_type = node_data['type']
+            attributes = node_data['attributes']
+            if node_type == 'domain' and attributes.get('certificates', {}).get('has_valid_cert') is False:
+                node_data['color'] = {'background': '#c7c7c7', 'border': '#999'} # Gray for invalid cert
+            
+            # Add incoming and outgoing edges to node data
+            if self.graph.has_node(node_id):
+                node_data['incoming_edges'] = [{'from': u, 'data': d} for u, _, d in self.graph.in_edges(node_id, data=True)]
+                node_data['outgoing_edges'] = [{'to': v, 'data': d} for _, v, d in self.graph.out_edges(node_id, data=True)]
+            
+            nodes.append(node_data)
+
+        edges = []
+        for source, target, attrs in self.graph.edges(data=True):
+            edges.append({'from': source, 'to': target,
+                          'label': attrs.get('relationship_type', ''),
+                          'confidence_score': attrs.get('confidence_score', 0),
+                          'source_provider': attrs.get('source_provider', ''),
+                          'discovery_timestamp': attrs.get('discovery_timestamp')})
+        return {
+            'nodes': nodes, 'edges': edges,
+            'statistics': self.get_statistics()['basic_metrics']
+        }
+
+    def export_json(self) -> Dict[str, Any]:
+        """Export complete graph data as a JSON-serializable dictionary."""
+        graph_data = nx.node_link_data(self.graph) # Use NetworkX's built-in robust serializer
+        return {
+            'export_metadata': {
+                'export_timestamp': datetime.now(timezone.utc).isoformat(),
+                'graph_creation_time': self.creation_time,
+                'last_modified': self.last_modified,
+                'total_nodes': self.get_node_count(),
+                'total_edges': self.get_edge_count(),
+                'graph_format': 'dnsrecon_v1_nodeling'
+            },
+            'graph': graph_data,
+            'statistics': self.get_statistics()
+        }
+
+    def _get_confidence_distribution(self) -> Dict[str, int]:
+        """Get distribution of edge confidence scores."""
+        distribution = {'high': 0, 'medium': 0, 'low': 0}
+        for _, _, data in self.graph.edges(data=True):
+            confidence = data.get('confidence_score', 0)
+            if confidence >= 0.8:
+                distribution['high'] += 1
+            elif confidence >= 0.6:
+                distribution['medium'] += 1
+            else:
+                distribution['low'] += 1
+        return distribution
+
+    def get_statistics(self) -> Dict[str, Any]:
+        """Get comprehensive statistics about the graph."""
+        stats = {'basic_metrics': {'total_nodes': self.get_node_count(),
+                                   'total_edges': self.get_edge_count(),
+                                   'creation_time': self.creation_time,
+                                   'last_modified': self.last_modified},
+                 'node_type_distribution': {}, 'relationship_type_distribution': {},
+                 'confidence_distribution': self._get_confidence_distribution(),
+                 'provider_distribution': {}}
+        # Calculate distributions
+        for node_type in NodeType:
+            stats['node_type_distribution'][node_type.value] = self.get_nodes_by_type(node_type).__len__()
+        for _, _, data in self.graph.edges(data=True):
+            rel_type = data.get('relationship_type', 'unknown')
+            stats['relationship_type_distribution'][rel_type] = stats['relationship_type_distribution'].get(rel_type, 0) + 1
+            provider = data.get('source_provider', 'unknown')
+            stats['provider_distribution'][provider] = stats['provider_distribution'].get(provider, 0) + 1
+        return stats
+
+    def clear(self) -> None:
+        """Clear all nodes, edges, and indices from the graph."""
+        self.graph.clear()
+        self.correlation_index.clear()
+        self.creation_time = datetime.now(timezone.utc).isoformat()
+        self.last_modified = self.creation_time

core/logger.py

@@ -0,0 +1,281 @@
+# dnsrecon/core/logger.py
+
+import logging
+import threading
+from datetime import datetime
+from typing import Dict, Any, Optional, List
+from dataclasses import dataclass, asdict
+from datetime import timezone
+
+
+@dataclass
+class APIRequest:
+    """Structured representation of an API request for forensic logging."""
+    timestamp: str
+    provider: str
+    url: str
+    method: str
+    status_code: Optional[int]
+    response_size: Optional[int]
+    duration_ms: Optional[float]
+    error: Optional[str]
+    target_indicator: str
+    discovery_context: Optional[str]
+
+
+@dataclass
+class RelationshipDiscovery:
+    """Structured representation of a discovered relationship."""
+    timestamp: str
+    source_node: str
+    target_node: str
+    relationship_type: str
+    confidence_score: float
+    provider: str
+    raw_data: Dict[str, Any]
+    discovery_method: str
+
+
+class ForensicLogger:
+    """
+    Thread-safe forensic logging system for DNSRecon.
+    Maintains detailed audit trail of all reconnaissance activities.
+    """
+    
+    def __init__(self, session_id: str = ""):
+        """
+        Initialize forensic logger.
+        
+        Args:
+            session_id: Unique identifier for this reconnaissance session
+        """
+        self.session_id = session_id or self._generate_session_id()
+        #self.lock = threading.Lock()
+        
+        # Initialize audit trail storage
+        self.api_requests: List[APIRequest] = []
+        self.relationships: List[RelationshipDiscovery] = []
+        self.session_metadata = {
+            'session_id': self.session_id,
+            'start_time': datetime.now(timezone.utc).isoformat(),
+            'end_time': None,
+            'total_requests': 0,
+            'total_relationships': 0,
+            'providers_used': set(),
+            'target_domains': set()
+        }
+        
+        # Configure standard logger
+        self.logger = logging.getLogger(f'dnsrecon.{self.session_id}')
+        self.logger.setLevel(logging.INFO)
+        
+        # Create formatter for structured logging
+        formatter = logging.Formatter(
+            '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+        )
+        
+        # Add console handler if not already present
+        if not self.logger.handlers:
+            console_handler = logging.StreamHandler()
+            console_handler.setFormatter(formatter)
+            self.logger.addHandler(console_handler)
+
+    def __getstate__(self):
+        """Prepare ForensicLogger for pickling by excluding unpicklable objects."""
+        state = self.__dict__.copy()
+        # Remove the unpickleable 'logger' attribute
+        if 'logger' in state:
+            del state['logger']
+        return state
+
+    def __setstate__(self, state):
+        """Restore ForensicLogger after unpickling by reconstructing logger."""
+        self.__dict__.update(state)
+        # Re-initialize the 'logger' attribute
+        self.logger = logging.getLogger(f'dnsrecon.{self.session_id}')
+        self.logger.setLevel(logging.INFO)
+        formatter = logging.Formatter(
+            '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+        )
+        if not self.logger.handlers:
+            console_handler = logging.StreamHandler()
+            console_handler.setFormatter(formatter)
+            self.logger.addHandler(console_handler)
+
+    def _generate_session_id(self) -> str:
+        """Generate unique session identifier."""
+        return f"dnsrecon_{datetime.now(timezone.utc).strftime('%Y%m%d_%H%M%S')}"
+    
+    def log_api_request(self, provider: str, url: str, method: str = "GET", 
+                       status_code: Optional[int] = None, 
+                       response_size: Optional[int] = None,
+                       duration_ms: Optional[float] = None,
+                       error: Optional[str] = None,
+                       target_indicator: str = "",
+                       discovery_context: Optional[str] = None) -> None:
+        """
+        Log an API request for forensic audit trail.
+        
+        Args:
+            provider: Name of the data provider
+            url: Request URL
+            method: HTTP method
+            status_code: HTTP response status code
+            response_size: Size of response in bytes
+            duration_ms: Request duration in milliseconds
+            error: Error message if request failed
+            target_indicator: The indicator being investigated
+            discovery_context: Context of how this indicator was discovered
+        """
+        api_request = APIRequest(
+            timestamp=datetime.now(timezone.utc).isoformat(),
+            provider=provider,
+            url=url,
+            method=method,
+            status_code=status_code,
+            response_size=response_size,
+            duration_ms=duration_ms,
+            error=error,
+            target_indicator=target_indicator,
+            discovery_context=discovery_context
+        )
+        
+        self.api_requests.append(api_request)
+        self.session_metadata['total_requests'] += 1
+        self.session_metadata['providers_used'].add(provider)
+        
+        if target_indicator:
+            self.session_metadata['target_domains'].add(target_indicator)
+        
+        # Log to standard logger
+        if error:
+            self.logger.error(f"API Request Failed - {provider}: {url} - {error}")
+        else:
+            self.logger.info(f"API Request - {provider}: {url} - Status: {status_code}")
+    
+    def log_relationship_discovery(self, source_node: str, target_node: str,
+                                 relationship_type: str, confidence_score: float,
+                                 provider: str, raw_data: Dict[str, Any],
+                                 discovery_method: str) -> None:
+        """
+        Log discovery of a new relationship between indicators.
+        
+        Args:
+            source_node: Source node identifier
+            target_node: Target node identifier
+            relationship_type: Type of relationship (e.g., 'SAN', 'A_Record')
+            confidence_score: Confidence score (0.0 to 1.0)
+            provider: Provider that discovered this relationship
+            raw_data: Raw data from provider response
+            discovery_method: Method used to discover relationship
+        """
+        relationship = RelationshipDiscovery(
+            timestamp=datetime.now(timezone.utc).isoformat(),
+            source_node=source_node,
+            target_node=target_node,
+            relationship_type=relationship_type,
+            confidence_score=confidence_score,
+            provider=provider,
+            raw_data=raw_data,
+            discovery_method=discovery_method
+        )
+        
+        self.relationships.append(relationship)
+        self.session_metadata['total_relationships'] += 1
+        
+        self.logger.info(
+            f"Relationship Discovered - {source_node} -> {target_node} "
+            f"({relationship_type}) - Confidence: {confidence_score:.2f} - Provider: {provider}"
+        )
+    
+    def log_scan_start(self, target_domain: str, recursion_depth: int, 
+                      enabled_providers: List[str]) -> None:
+        """Log the start of a reconnaissance scan."""
+        self.logger.info(f"Scan Started - Target: {target_domain}, Depth: {recursion_depth}")
+        self.logger.info(f"Enabled Providers: {', '.join(enabled_providers)}")
+        
+        self.session_metadata['target_domains'].add(target_domain)
+    
+    def log_scan_complete(self) -> None:
+        """Log the completion of a reconnaissance scan."""
+        self.session_metadata['end_time'] = datetime.now(timezone.utc).isoformat()
+        self.session_metadata['providers_used'] = list(self.session_metadata['providers_used'])
+        self.session_metadata['target_domains'] = list(self.session_metadata['target_domains'])
+        
+        self.logger.info(f"Scan Complete - Session: {self.session_id}")
+    
+    def export_audit_trail(self) -> Dict[str, Any]:
+        """
+        Export complete audit trail for forensic analysis.
+        
+        Returns:
+            Dictionary containing complete session audit trail
+        """
+        return {
+            'session_metadata': self.session_metadata.copy(),
+            'api_requests': [asdict(req) for req in self.api_requests],
+            'relationships': [asdict(rel) for rel in self.relationships],
+            'export_timestamp': datetime.now(timezone.utc).isoformat()
+        }
+    
+    def get_forensic_summary(self) -> Dict[str, Any]:
+        """
+        Get summary statistics for forensic reporting.
+        
+        Returns:
+            Dictionary containing summary statistics
+        """
+        provider_stats = {}
+        for provider in self.session_metadata['providers_used']:
+            provider_requests = [req for req in self.api_requests if req.provider == provider]
+            provider_relationships = [rel for rel in self.relationships if rel.provider == provider]
+            
+            provider_stats[provider] = {
+                'total_requests': len(provider_requests),
+                'successful_requests': len([req for req in provider_requests if req.error is None]),
+                'failed_requests': len([req for req in provider_requests if req.error is not None]),
+                'relationships_discovered': len(provider_relationships),
+                'avg_confidence': sum(rel.confidence_score for rel in provider_relationships) / len(provider_relationships) if provider_relationships else 0
+            }
+        
+        return {
+            'session_id': self.session_id,
+            'duration_minutes': self._calculate_session_duration(),
+            'total_requests': self.session_metadata['total_requests'],
+            'total_relationships': self.session_metadata['total_relationships'],
+            'unique_indicators': len(set([rel.source_node for rel in self.relationships] + [rel.target_node for rel in self.relationships])),
+            'provider_statistics': provider_stats
+        }
+    
+    def _calculate_session_duration(self) -> float:
+        """Calculate session duration in minutes."""
+        if not self.session_metadata['end_time']:
+            end_time = datetime.now(timezone.utc)
+        else:
+            end_time = datetime.fromisoformat(self.session_metadata['end_time'])
+        
+        start_time = datetime.fromisoformat(self.session_metadata['start_time'])
+        duration = (end_time - start_time).total_seconds() / 60
+        return round(duration, 2)
+
+
+# Global logger instance for the current session
+_current_logger: Optional[ForensicLogger] = None
+_logger_lock = threading.Lock()
+
+
+def get_forensic_logger() -> ForensicLogger:
+    """Get or create the current forensic logger instance."""
+    global _current_logger
+    with _logger_lock:
+        if _current_logger is None:
+            _current_logger = ForensicLogger()
+        return _current_logger
+
+
+def new_session() -> ForensicLogger:
+    """Start a new forensic logging session."""
+    global _current_logger
+    with _logger_lock:
+        _current_logger = ForensicLogger()
+        return _current_logger

core/scanner.py

@@ -0,0 +1,921 @@
+# dnsrecon/core/scanner.py
+
+import threading
+import traceback
+import time
+import os
+import importlib
+from typing import List, Set, Dict, Any, Tuple, Optional
+from concurrent.futures import ThreadPoolExecutor, as_completed, CancelledError, Future
+from collections import defaultdict, deque
+from datetime import datetime, timezone
+
+from core.graph_manager import GraphManager, NodeType
+from core.logger import get_forensic_logger, new_session
+from utils.helpers import _is_valid_ip, _is_valid_domain
+from providers.base_provider import BaseProvider
+
+
+class ScanStatus:
+    """Enumeration of scan statuses."""
+    IDLE = "idle"
+    RUNNING = "running"
+    COMPLETED = "completed"
+    FAILED = "failed"
+    STOPPED = "stopped"
+
+
+class Scanner:
+    """
+    Main scanning orchestrator for DNSRecon passive reconnaissance.
+    """
+
+    def __init__(self, session_config=None):
+        """Initialize scanner with session-specific configuration."""
+        print("Initializing Scanner instance...")
+
+        try:
+            # Use provided session config or create default
+            if session_config is None:
+                from core.session_config import create_session_config
+                session_config = create_session_config()
+            
+            self.config = session_config
+            self.graph = GraphManager()
+            self.providers = []
+            self.status = ScanStatus.IDLE
+            self.current_target = None
+            self.current_depth = 0
+            self.max_depth = 2
+            self.stop_event = threading.Event()
+            self.scan_thread = None
+            self.session_id: Optional[str] = None  # Will be set by session manager
+            self.task_queue = deque([])
+            self.target_retries = defaultdict(int)
+            self.scan_failed_due_to_retries = False
+
+            # **NEW**: Track currently processing tasks to prevent processing after stop
+            self.currently_processing = set()
+            self.processing_lock = threading.Lock()
+
+            # Scanning progress tracking
+            self.total_indicators_found = 0
+            self.indicators_processed = 0
+            self.indicators_completed = 0
+            self.tasks_re_enqueued = 0
+            self.current_indicator = ""
+
+            # Concurrent processing configuration
+            self.max_workers = self.config.max_concurrent_requests
+            self.executor = None
+
+            # Initialize providers with session config
+            print("Calling _initialize_providers with session config...")
+            self._initialize_providers()
+
+            # Initialize logger
+            print("Initializing forensic logger...")
+            self.logger = get_forensic_logger()
+
+            print("Scanner initialization complete")
+
+        except Exception as e:
+            print(f"ERROR: Scanner initialization failed: {e}")
+            traceback.print_exc()
+            raise
+
+    def _is_stop_requested(self) -> bool:
+        """
+        Check if stop is requested using both local and Redis-based signals.
+        This ensures reliable termination across process boundaries.
+        """
+        # Check local threading event first (fastest)
+        if self.stop_event.is_set():
+            return True
+        
+        # Check Redis-based stop signal if session ID is available
+        if self.session_id:
+            try:
+                from core.session_manager import session_manager
+                return session_manager.is_stop_requested(self.session_id)
+            except Exception as e:
+                print(f"Error checking Redis stop signal: {e}")
+                # Fall back to local event
+                return self.stop_event.is_set()
+        
+        return False
+
+    def _set_stop_signal(self) -> None:
+        """
+        Set stop signal both locally and in Redis.
+        """
+        # Set local event
+        self.stop_event.set()
+        
+        # Set Redis signal if session ID is available
+        if self.session_id:
+            try:
+                from core.session_manager import session_manager
+                session_manager.set_stop_signal(self.session_id)
+            except Exception as e:
+                print(f"Error setting Redis stop signal: {e}")
+
+    def __getstate__(self):
+        """Prepare object for pickling by excluding unpicklable attributes."""
+        state = self.__dict__.copy()
+        
+        # Remove unpicklable threading objects
+        unpicklable_attrs = [
+            'stop_event',
+            'scan_thread', 
+            'executor',
+            'processing_lock'  # **NEW**: Exclude the processing lock
+        ]
+        
+        for attr in unpicklable_attrs:
+            if attr in state:
+                del state[attr]
+        
+        # Handle providers separately to ensure they're picklable
+        if 'providers' in state:
+            # The providers should be picklable now, but let's ensure clean state
+            for provider in state['providers']:
+                if hasattr(provider, '_stop_event'):
+                    provider._stop_event = None
+        
+        return state
+
+    def __setstate__(self, state):
+        """Restore object after unpickling by reconstructing threading objects."""
+        self.__dict__.update(state)
+        
+        # Reconstruct threading objects
+        self.stop_event = threading.Event()
+        self.scan_thread = None
+        self.executor = None
+        self.processing_lock = threading.Lock()  # **NEW**: Recreate processing lock
+        
+        # **NEW**: Reset processing tracking
+        if not hasattr(self, 'currently_processing'):
+            self.currently_processing = set()
+        
+        # Re-set stop events for providers
+        if hasattr(self, 'providers'):
+            for provider in self.providers:
+                if hasattr(provider, 'set_stop_event'):
+                    provider.set_stop_event(self.stop_event)
+
+    def _initialize_providers(self) -> None:
+        """Initialize all available providers based on session configuration."""
+        self.providers = []
+        print("Initializing providers with session config...")
+
+        provider_dir = os.path.join(os.path.dirname(__file__), '..', 'providers')
+        for filename in os.listdir(provider_dir):
+            if filename.endswith('_provider.py') and not filename.startswith('base'):
+                module_name = f"providers.{filename[:-3]}"
+                try:
+                    module = importlib.import_module(module_name)
+                    for attribute_name in dir(module):
+                        attribute = getattr(module, attribute_name)
+                        if isinstance(attribute, type) and issubclass(attribute, BaseProvider) and attribute is not BaseProvider:
+                            provider_class = attribute
+                            provider = provider_class(name=attribute_name, session_config=self.config)
+                            provider_name = provider.get_name()
+
+                            if self.config.is_provider_enabled(provider_name):
+                                if provider.is_available():
+                                    provider.set_stop_event(self.stop_event)
+                                    self.providers.append(provider)
+                                    print(f"✓ {provider.get_display_name()} provider initialized successfully for session")
+                                else:
+                                    print(f"✗ {provider.get_display_name()} provider is not available")
+                except Exception as e:
+                    print(f"✗ Failed to initialize provider from {filename}: {e}")
+                    traceback.print_exc()
+
+        print(f"Initialized {len(self.providers)} providers for session")
+
+    def update_session_config(self, new_config) -> None:
+        """Update session configuration and reinitialize providers."""
+        print("Updating session configuration...")
+        self.config = new_config
+        self.max_workers = self.config.max_concurrent_requests
+        self._initialize_providers()
+        print("Session configuration updated")
+
+    def start_scan(self, target_domain: str, max_depth: int = 2, clear_graph: bool = True) -> bool:
+        """Start a new reconnaissance scan with proper cleanup of previous scans."""
+        print(f"=== STARTING SCAN IN SCANNER {id(self)} ===")
+        print(f"Session ID: {self.session_id}")
+        print(f"Initial scanner status: {self.status}")
+
+        # **IMPROVED**: More aggressive cleanup of previous scan
+        if self.scan_thread and self.scan_thread.is_alive():
+            print("A previous scan thread is still alive. Forcing termination...")
+            
+            # Set stop signals immediately
+            self._set_stop_signal()
+            self.status = ScanStatus.STOPPED
+            
+            # Clear all processing state
+            with self.processing_lock:
+                self.currently_processing.clear()
+            self.task_queue.clear()
+            
+            # Shutdown executor aggressively
+            if self.executor:
+                print("Shutting down executor forcefully...")
+                self.executor.shutdown(wait=False, cancel_futures=True)
+                self.executor = None
+            
+            # Wait for thread termination with shorter timeout
+            print("Waiting for previous scan thread to terminate...")
+            self.scan_thread.join(5.0)  # Reduced from 10 seconds
+            
+            if self.scan_thread.is_alive():
+                print("WARNING: Previous scan thread is still alive after 5 seconds")
+                # Continue anyway, but log the issue
+                self.logger.logger.warning("Previous scan thread failed to terminate cleanly")
+
+        # Reset state for new scan with proper forensic logging
+        print("Resetting scanner state for new scan...")
+        self.status = ScanStatus.IDLE
+        self.stop_event.clear()
+        
+        # **NEW**: Clear Redis stop signal explicitly
+        if self.session_id:
+            from core.session_manager import session_manager
+            session_manager.clear_stop_signal(self.session_id)
+        
+        with self.processing_lock:
+            self.currently_processing.clear()
+        
+        self.task_queue.clear()
+        self.target_retries.clear()
+        self.scan_failed_due_to_retries = False
+        
+        # Update session state immediately for GUI feedback
+        self._update_session_state()
+        print("Scanner state reset complete.")
+
+        try:
+            if not hasattr(self, 'providers') or not self.providers:
+                print(f"ERROR: No providers available in scanner {id(self)}, cannot start scan")
+                return False
+            
+            print(f"Scanner {id(self)} validation passed, providers available: {[p.get_name() for p in self.providers]}")
+
+            if clear_graph:
+                self.graph.clear()
+            self.current_target = target_domain.lower().strip()
+            self.max_depth = max_depth
+            self.current_depth = 0
+            
+            self.total_indicators_found = 0
+            self.indicators_processed = 0
+            self.indicators_completed = 0
+            self.tasks_re_enqueued = 0
+            self.current_indicator = self.current_target
+
+            # Update GUI with scan preparation state
+            self._update_session_state()
+
+            # Start new forensic session
+            print(f"Starting new forensic session for scanner {id(self)}...")
+            self.logger = new_session()
+
+            # Start scan in a separate thread
+            print(f"Starting scan thread for scanner {id(self)}...")
+            self.scan_thread = threading.Thread(
+                target=self._execute_scan,
+                args=(self.current_target, max_depth),
+                daemon=True
+            )
+            self.scan_thread.start()
+
+            print(f"=== SCAN STARTED SUCCESSFULLY IN SCANNER {id(self)} ===")
+            return True
+
+        except Exception as e:
+            print(f"ERROR: Exception in start_scan for scanner {id(self)}: {e}")
+            traceback.print_exc()
+            self.status = ScanStatus.FAILED
+            self._update_session_state() 
+            return False
+
+    def _execute_scan(self, target_domain: str, max_depth: int) -> None:
+        """Execute the reconnaissance scan with proper termination handling."""
+        print(f"_execute_scan started for {target_domain} with depth {max_depth}")
+        self.executor = ThreadPoolExecutor(max_workers=self.max_workers)
+        processed_targets = set()
+        
+        self.task_queue.append((target_domain, 0, False))
+        
+        try:
+            self.status = ScanStatus.RUNNING
+            self._update_session_state()
+            
+            enabled_providers = [provider.get_name() for provider in self.providers]
+            self.logger.log_scan_start(target_domain, max_depth, enabled_providers)
+            self.graph.add_node(target_domain, NodeType.DOMAIN)
+            self._initialize_provider_states(target_domain)
+
+            # **IMPROVED**: Better termination checking in main loop
+            while self.task_queue and not self._is_stop_requested():
+                try:
+                    target, depth, is_large_entity_member = self.task_queue.popleft()
+                except IndexError:
+                    # Queue became empty during processing
+                    break
+
+                if target in processed_targets:
+                    continue
+
+                if depth > max_depth:
+                    continue
+
+                # **NEW**: Track this target as currently processing
+                with self.processing_lock:
+                    if self._is_stop_requested():
+                        print(f"Stop requested before processing {target}")
+                        break
+                    self.currently_processing.add(target)
+
+                try:
+                    self.current_depth = depth
+                    self.current_indicator = target
+                    self._update_session_state()
+                    
+                    # **IMPROVED**: More frequent stop checking during processing
+                    if self._is_stop_requested():
+                        print(f"Stop requested during processing setup for {target}")
+                        break
+                    
+                    new_targets, large_entity_members, success = self._query_providers_for_target(target, depth, is_large_entity_member)
+                    
+                    # **NEW**: Check stop signal after provider queries
+                    if self._is_stop_requested():
+                        print(f"Stop requested after querying providers for {target}")
+                        break
+                    
+                    if not success:
+                        self.target_retries[target] += 1
+                        if self.target_retries[target] <= self.config.max_retries_per_target:
+                            print(f"Re-queueing target {target} (attempt {self.target_retries[target]})")
+                            self.task_queue.append((target, depth, is_large_entity_member))
+                            self.tasks_re_enqueued += 1
+                        else:
+                            print(f"ERROR: Max retries exceeded for target {target}")
+                            self.scan_failed_due_to_retries = True
+                            self._log_target_processing_error(target, "Max retries exceeded")
+                    else:
+                        processed_targets.add(target)
+                        self.indicators_completed += 1
+                    
+                    # **NEW**: Only add new targets if not stopped
+                    if not self._is_stop_requested():
+                        for new_target in new_targets:
+                            if new_target not in processed_targets:
+                                self.task_queue.append((new_target, depth + 1, False))
+                        
+                        for member in large_entity_members:
+                            if member not in processed_targets:
+                                self.task_queue.append((member, depth, True))
+                
+                finally:
+                    # **NEW**: Always remove from processing set
+                    with self.processing_lock:
+                        self.currently_processing.discard(target)
+
+            # **NEW**: Log termination reason
+            if self._is_stop_requested():
+                print("Scan terminated due to stop request")
+                self.logger.logger.info("Scan terminated by user request")
+            elif not self.task_queue:
+                print("Scan completed - no more targets to process")
+                self.logger.logger.info("Scan completed - all targets processed")
+
+        except Exception as e:
+            print(f"ERROR: Scan execution failed with error: {e}")
+            traceback.print_exc()
+            self.status = ScanStatus.FAILED
+            self.logger.logger.error(f"Scan failed: {e}")
+        finally:
+            # **NEW**: Clear processing state on exit
+            with self.processing_lock:
+                self.currently_processing.clear()
+            
+            if self._is_stop_requested():
+                self.status = ScanStatus.STOPPED
+            elif self.scan_failed_due_to_retries:
+                self.status = ScanStatus.FAILED
+            else:
+                self.status = ScanStatus.COMPLETED
+                
+            self._update_session_state()
+            self.logger.log_scan_complete()
+            if self.executor:
+                self.executor.shutdown(wait=False, cancel_futures=True)
+                self.executor = None
+            stats = self.graph.get_statistics()
+            print("Final scan statistics:")
+            print(f"  - Total nodes: {stats['basic_metrics']['total_nodes']}")
+            print(f"  - Total edges: {stats['basic_metrics']['total_edges']}")
+            print(f"  - Targets processed: {len(processed_targets)}")
+
+    def _query_providers_for_target(self, target: str, depth: int, dns_only: bool = False) -> Tuple[Set[str], Set[str], bool]:
+        """Query providers for a single target with enhanced stop checking."""
+        # **NEW**: Early termination check
+        if self._is_stop_requested():
+            print(f"Stop requested before querying providers for {target}")
+            return set(), set(), False
+
+        is_ip = _is_valid_ip(target)
+        target_type = NodeType.IP if is_ip else NodeType.DOMAIN
+        print(f"Querying providers for {target_type.value}: {target} at depth {depth}")
+
+        self.graph.add_node(target, target_type)
+        self._initialize_provider_states(target)
+        
+        new_targets = set()
+        large_entity_members = set()
+        node_attributes = defaultdict(lambda: defaultdict(list))
+        all_providers_successful = True
+
+        eligible_providers = self._get_eligible_providers(target, is_ip, dns_only)
+        
+        if not eligible_providers:
+            self._log_no_eligible_providers(target, is_ip)
+            return new_targets, large_entity_members, True
+
+        # **IMPROVED**: Check stop signal before each provider
+        for i, provider in enumerate(eligible_providers):
+            if self._is_stop_requested():
+                print(f"Stop requested while querying provider {i+1}/{len(eligible_providers)} for {target}")
+                all_providers_successful = False
+                break
+            
+            try:
+                provider_results = self._query_single_provider_forensic(provider, target, is_ip, depth)
+                if provider_results is None:
+                    all_providers_successful = False
+                elif not self._is_stop_requested():
+                    discovered, is_large_entity = self._process_provider_results_forensic(
+                        target, provider, provider_results, node_attributes, depth
+                    )
+                    if is_large_entity:
+                        large_entity_members.update(discovered)
+                    else:
+                        new_targets.update(discovered)
+                else:
+                    print(f"Stop requested after processing results from {provider.get_name()}")
+                    break
+            except Exception as e:
+                all_providers_successful = False
+                self._log_provider_error(target, provider.get_name(), str(e))
+
+        # **NEW**: Only update node attributes if not stopped
+        if not self._is_stop_requested():
+            for node_id, attributes in node_attributes.items():
+                if self.graph.graph.has_node(node_id):
+                    node_is_ip = _is_valid_ip(node_id)
+                    node_type_to_add = NodeType.IP if node_is_ip else NodeType.DOMAIN
+                    self.graph.add_node(node_id, node_type_to_add, attributes=attributes)
+
+        return new_targets, large_entity_members, all_providers_successful
+
+    def stop_scan(self) -> bool:
+        """Request immediate scan termination with proper cleanup."""
+        try:
+            print("=== INITIATING IMMEDIATE SCAN TERMINATION ===")
+            self.logger.logger.info("Scan termination requested by user")
+            
+            # **IMPROVED**: More aggressive stop signal setting
+            self._set_stop_signal()
+            self.status = ScanStatus.STOPPED
+            
+            # **NEW**: Clear processing state immediately
+            with self.processing_lock:
+                currently_processing_copy = self.currently_processing.copy()
+                self.currently_processing.clear()
+                print(f"Cleared {len(currently_processing_copy)} currently processing targets: {currently_processing_copy}")
+            
+            # **IMPROVED**: Clear task queue and log what was discarded
+            discarded_tasks = list(self.task_queue)
+            self.task_queue.clear()
+            print(f"Discarded {len(discarded_tasks)} pending tasks")
+            
+            # **IMPROVED**: Aggressively shut down executor
+            if self.executor:
+                print("Shutting down executor with immediate cancellation...")
+                try:
+                    # Cancel all pending futures
+                    self.executor.shutdown(wait=False, cancel_futures=True)
+                    print("Executor shutdown completed")
+                except Exception as e:
+                    print(f"Error during executor shutdown: {e}")
+
+            # Immediately update GUI with stopped status
+            self._update_session_state()
+
+            print("Termination signals sent. The scan will stop as soon as possible.")
+            return True
+            
+        except Exception as e:
+            print(f"ERROR: Exception in stop_scan: {e}")
+            self.logger.logger.error(f"Error during scan termination: {e}")
+            traceback.print_exc()
+            return False
+
+    def _update_session_state(self) -> None:
+        """
+        Update the scanner state in Redis for GUI updates.
+        This ensures the web interface sees real-time updates.
+        """
+        if self.session_id:
+            try:
+                from core.session_manager import session_manager
+                success = session_manager.update_session_scanner(self.session_id, self)
+                if not success:
+                    print(f"WARNING: Failed to update session state for {self.session_id}")
+            except Exception as e:
+                print(f"ERROR: Failed to update session state: {e}")
+
+    def get_scan_status(self) -> Dict[str, Any]:
+        """Get current scan status with processing information."""
+        try:
+            with self.processing_lock:
+                currently_processing_count = len(self.currently_processing)
+                currently_processing_list = list(self.currently_processing)
+            
+            return {
+                'status': self.status,
+                'target_domain': self.current_target,
+                'current_depth': self.current_depth,
+                'max_depth': self.max_depth,
+                'current_indicator': self.current_indicator,
+                'indicators_processed': self.indicators_processed,
+                'indicators_completed': self.indicators_completed,
+                'tasks_re_enqueued': self.tasks_re_enqueued,
+                'progress_percentage': self._calculate_progress(),
+                'enabled_providers': [provider.get_name() for provider in self.providers],
+                'graph_statistics': self.graph.get_statistics(),
+                'task_queue_size': len(self.task_queue),
+                'currently_processing_count': currently_processing_count,  # **NEW**
+                'currently_processing': currently_processing_list[:5]  # **NEW**: Show first 5 for debugging
+            }
+        except Exception as e:
+            print(f"ERROR: Exception in get_scan_status: {e}")
+            traceback.print_exc()
+            return {
+                'status': 'error',
+                'target_domain': None,
+                'current_depth': 0,
+                'max_depth': 0,
+                'current_indicator': '',
+                'indicators_processed': 0,
+                'indicators_completed': 0,
+                'tasks_re_enqueued': 0,
+                'progress_percentage': 0.0,
+                'enabled_providers': [],
+                'graph_statistics': {},
+                'task_queue_size': 0,
+                'currently_processing_count': 0,
+                'currently_processing': []
+            }
+
+    def _initialize_provider_states(self, target: str) -> None:
+        """Initialize provider states for forensic tracking."""
+        if not self.graph.graph.has_node(target):
+            return
+            
+        node_data = self.graph.graph.nodes[target]
+        if 'metadata' not in node_data:
+            node_data['metadata'] = {}
+        if 'provider_states' not in node_data['metadata']:
+            node_data['metadata']['provider_states'] = {}
+
+    def _get_eligible_providers(self, target: str, is_ip: bool, dns_only: bool) -> List:
+        """Get providers eligible for querying this target."""
+        if dns_only:
+            return [p for p in self.providers if p.get_name() == 'dns']
+
+        eligible = []
+        target_key = 'ips' if is_ip else 'domains'
+        
+        for provider in self.providers:
+            if provider.get_eligibility().get(target_key):
+                if not self._already_queried_provider(target, provider.get_name()):
+                    eligible.append(provider)
+                else:
+                    print(f"Skipping {provider.get_name()} for {target} - already queried")
+        
+        return eligible
+
+    def _already_queried_provider(self, target: str, provider_name: str) -> bool:
+        """Check if we already successfully queried a provider for a target."""
+        if not self.graph.graph.has_node(target):
+            return False
+            
+        node_data = self.graph.graph.nodes[target]
+        provider_states = node_data.get('metadata', {}).get('provider_states', {})
+        
+        # A provider has been successfully queried if a state exists and its status is 'success'
+        provider_state = provider_states.get(provider_name)
+        return provider_state is not None and provider_state.get('status') == 'success'
+
+    def _query_single_provider_forensic(self, provider, target: str, is_ip: bool, current_depth: int) -> Optional[List]:
+        """Query a single provider with stop signal checking."""
+        provider_name = provider.get_name()
+        start_time = datetime.now(timezone.utc)
+        
+        if self._is_stop_requested():
+            print(f"Stop requested before querying {provider_name} for {target}")
+            return None
+        
+        print(f"Querying {provider_name} for {target}")
+        
+        self.logger.logger.info(f"Attempting {provider_name} query for {target} at depth {current_depth}")
+        
+        try:
+            if is_ip:
+                results = provider.query_ip(target)
+            else:
+                results = provider.query_domain(target)
+            
+            if self._is_stop_requested():
+                print(f"Stop requested after querying {provider_name} for {target}")
+                return None
+            
+            self._update_provider_state(target, provider_name, 'success', len(results), None, start_time)
+            
+            print(f"✓ {provider_name} returned {len(results)} results for {target}")
+            return results
+            
+        except Exception as e:
+            self._update_provider_state(target, provider_name, 'failed', 0, str(e), start_time)
+            print(f"✗ {provider_name} failed for {target}: {e}")
+            return None
+
+    def _update_provider_state(self, target: str, provider_name: str, status: str, 
+                              results_count: int, error: Optional[str], start_time: datetime) -> None:
+        """Update provider state in node metadata for forensic tracking."""
+        if not self.graph.graph.has_node(target):
+            return
+            
+        node_data = self.graph.graph.nodes[target]
+        if 'metadata' not in node_data:
+            node_data['metadata'] = {}
+        if 'provider_states' not in node_data['metadata']:
+            node_data['metadata']['provider_states'] = {}
+        
+        node_data['metadata']['provider_states'][provider_name] = {
+            'status': status,
+            'timestamp': start_time.isoformat(),
+            'results_count': results_count,
+            'error': error,
+            'duration_ms': (datetime.now(timezone.utc) - start_time).total_seconds() * 1000
+        }
+        
+        self.logger.logger.info(f"Provider state updated: {target} -> {provider_name} -> {status} ({results_count} results)")
+
+    def _process_provider_results_forensic(self, target: str, provider, results: List,
+                                        node_attributes: Dict, current_depth: int) -> Tuple[Set[str], bool]:
+        """Process provider results, returns (discovered_targets, is_large_entity)."""
+        provider_name = provider.get_name()
+        discovered_targets = set()
+
+        if self._is_stop_requested():
+            print(f"Stop requested before processing results from {provider_name} for {target}")
+            return discovered_targets, False
+
+        if len(results) > self.config.large_entity_threshold:
+            print(f"Large entity detected: {provider_name} returned {len(results)} results for {target}")
+            members = self._create_large_entity(target, provider_name, results, current_depth)
+            return members, True
+
+        for i, (source, rel_target, rel_type, confidence, raw_data) in enumerate(results):
+            if i % 5 == 0 and self._is_stop_requested():  # Check more frequently
+                print(f"Stop requested while processing results from {provider_name} for {target}")
+                break
+
+            self.logger.log_relationship_discovery(
+                source_node=source,
+                target_node=rel_target,
+                relationship_type=rel_type,
+                confidence_score=confidence,
+                provider=provider_name,
+                raw_data=raw_data,
+                discovery_method=f"{provider_name}_query_depth_{current_depth}"
+            )
+
+            self._collect_node_attributes(source, provider_name, rel_type, rel_target, raw_data, node_attributes[source])
+
+            if isinstance(rel_target, list):
+                # If the target is a list, iterate and process each item
+                for single_target in rel_target:
+                    if _is_valid_ip(single_target):
+                        self.graph.add_node(single_target, NodeType.IP)
+                        if self.graph.add_edge(source, single_target, rel_type, confidence, provider_name, raw_data):
+                            print(f"Added IP relationship: {source} -> {single_target} ({rel_type})")
+                        discovered_targets.add(single_target)
+                    elif _is_valid_domain(single_target):
+                        self.graph.add_node(single_target, NodeType.DOMAIN)
+                        if self.graph.add_edge(source, single_target, rel_type, confidence, provider_name, raw_data):
+                            print(f"Added domain relationship: {source} -> {single_target} ({rel_type})")
+                        discovered_targets.add(single_target)
+                        self._collect_node_attributes(single_target, provider_name, rel_type, source, raw_data, node_attributes[single_target])
+
+            elif _is_valid_ip(rel_target):
+                self.graph.add_node(rel_target, NodeType.IP)
+                if self.graph.add_edge(source, rel_target, rel_type, confidence, provider_name, raw_data):
+                    print(f"Added IP relationship: {source} -> {rel_target} ({rel_type})")
+                discovered_targets.add(rel_target)
+
+            elif rel_target.startswith('AS') and rel_target[2:].isdigit():
+                self.graph.add_node(rel_target, NodeType.ASN)
+                if self.graph.add_edge(source, rel_target, rel_type, confidence, provider_name, raw_data):
+                    print(f"Added ASN relationship: {source} -> {rel_target} ({rel_type})")
+
+            elif _is_valid_domain(rel_target):
+                self.graph.add_node(rel_target, NodeType.DOMAIN)
+                if self.graph.add_edge(source, rel_target, rel_type, confidence, provider_name, raw_data):
+                    print(f"Added domain relationship: {source} -> {rel_target} ({rel_type})")
+                discovered_targets.add(rel_target)
+                self._collect_node_attributes(rel_target, provider_name, rel_type, source, raw_data, node_attributes[rel_target])
+            
+            else:
+                self._collect_node_attributes(source, provider_name, rel_type, rel_target, raw_data, node_attributes[source])
+
+        return discovered_targets, False
+
+    def _create_large_entity(self, source: str, provider_name: str, results: List, current_depth: int) -> Set[str]:
+        """Create a large entity node and returns the members for DNS processing."""
+        entity_id = f"large_entity_{provider_name}_{hash(source) & 0x7FFFFFFF}"
+        
+        targets = [rel[1] for rel in results if len(rel) > 1]
+        node_type = 'unknown'
+        
+        if targets:
+            if _is_valid_domain(targets[0]):
+                node_type = 'domain'
+            elif _is_valid_ip(targets[0]):
+                node_type = 'ip'
+        
+        for target in targets:
+            self.graph.add_node(target, NodeType.DOMAIN if node_type == 'domain' else NodeType.IP)
+
+        attributes = {
+            'count': len(targets),
+            'nodes': targets,
+            'node_type': node_type,
+            'source_provider': provider_name,
+            'discovery_depth': current_depth,
+            'threshold_exceeded': self.config.large_entity_threshold,
+        }
+        description = f'Large entity created due to {len(targets)} results from {provider_name}'
+        
+        self.graph.add_node(entity_id, NodeType.LARGE_ENTITY, attributes=attributes, description=description)
+        
+        if results:
+            rel_type = results[0][2]
+            self.graph.add_edge(source, entity_id, rel_type, 0.9, provider_name, 
+                              {'large_entity_info': f'Contains {len(targets)} {node_type}s'})
+        
+        self.logger.logger.warning(f"Large entity created: {entity_id} contains {len(targets)} targets from {provider_name}")
+        print(f"Created large entity {entity_id} for {len(targets)} {node_type}s from {provider_name}")
+        
+        return set(targets)
+
+    def _collect_node_attributes(self, node_id: str, provider_name: str, rel_type: str,
+                                    target: str, raw_data: Dict[str, Any], attributes: Dict[str, Any]) -> None:
+        """Collect and organize attributes for a node."""
+        self.logger.logger.debug(f"Collecting attributes for {node_id} from {provider_name}: {rel_type}")
+
+        if provider_name == 'dns':
+            record_type = raw_data.get('query_type', 'UNKNOWN')
+            value = raw_data.get('value', target)
+            dns_entry = f"{record_type}: {value}"
+            if dns_entry not in attributes.get('dns_records', []):
+                attributes.setdefault('dns_records', []).append(dns_entry)
+
+        elif provider_name == 'crtsh':
+            if rel_type == "san_certificate":
+                domain_certs = raw_data.get('domain_certificates', {})
+                if node_id in domain_certs:
+                    cert_summary = domain_certs[node_id]
+                    attributes['certificates'] = cert_summary
+                    if target not in attributes.get('related_domains_san', []):
+                        attributes.setdefault('related_domains_san', []).append(target)
+
+        elif provider_name == 'shodan':
+            shodan_attributes = attributes.setdefault('shodan', {})
+            for key, value in raw_data.items():
+                if key not in shodan_attributes or not shodan_attributes.get(key):
+                    shodan_attributes[key] = value
+
+        if rel_type == "asn_membership":
+            attributes['asn'] = {
+                'id': target,
+                'description': raw_data.get('org', ''),
+                'isp': raw_data.get('isp', ''),
+                'country': raw_data.get('country', '')
+            }
+        
+        record_type_name = rel_type
+        if record_type_name not in attributes:
+            attributes[record_type_name] = []
+        
+        if isinstance(target, list):
+            attributes[record_type_name].extend(target)
+        else:
+            if target not in attributes[record_type_name]:
+                attributes[record_type_name].append(target)
+
+    def _log_target_processing_error(self, target: str, error: str) -> None:
+        """Log target processing errors for forensic trail."""
+        self.logger.logger.error(f"Target processing failed for {target}: {error}")
+
+    def _log_provider_error(self, target: str, provider_name: str, error: str) -> None:
+        """Log provider query errors for forensic trail."""
+        self.logger.logger.error(f"Provider {provider_name} failed for {target}: {error}")
+
+    def _log_no_eligible_providers(self, target: str, is_ip: bool) -> None:
+        """Log when no providers are eligible for a target."""
+        target_type = 'IP' if is_ip else 'domain'
+        self.logger.logger.warning(f"No eligible providers for {target_type}: {target}")
+
+    def _calculate_progress(self) -> float:
+        """Calculate scan progress percentage based on task completion."""
+        total_tasks = self.indicators_completed + len(self.task_queue)
+        if total_tasks == 0:
+            return 0.0
+        return min(100.0, (self.indicators_completed / total_tasks) * 100)
+
+    def get_graph_data(self) -> Dict[str, Any]:
+        """Get current graph data for visualization."""
+        return self.graph.get_graph_data()
+
+    def export_results(self) -> Dict[str, Any]:
+        """Export complete scan results with forensic audit trail."""
+        graph_data = self.graph.export_json()
+        audit_trail = self.logger.export_audit_trail()
+        provider_stats = {}
+        for provider in self.providers:
+            provider_stats[provider.get_name()] = provider.get_statistics()
+        
+        export_data = {
+            'scan_metadata': {
+                'target_domain': self.current_target,
+                'max_depth': self.max_depth,
+                'final_status': self.status,
+                'total_indicators_processed': self.indicators_processed,
+                'enabled_providers': list(provider_stats.keys()),
+                'session_id': self.session_id
+            },
+            'graph_data': graph_data,
+            'forensic_audit': audit_trail,
+            'provider_statistics': provider_stats,
+            'scan_summary': self.logger.get_forensic_summary()
+        }
+        return export_data
+
+    def get_provider_statistics(self) -> Dict[str, Dict[str, Any]]:
+        """Get statistics for all providers with forensic information."""
+        stats = {}
+        for provider in self.providers:
+            stats[provider.get_name()] = provider.get_statistics()
+        return stats
+
+    def get_provider_info(self) -> Dict[str, Dict[str, Any]]:
+        """Get information about all available providers."""
+        info = {}
+        provider_dir = os.path.join(os.path.dirname(__file__), '..', 'providers')
+        for filename in os.listdir(provider_dir):
+            if filename.endswith('_provider.py') and not filename.startswith('base'):
+                module_name = f"providers.{filename[:-3]}"
+                try:
+                    module = importlib.import_module(module_name)
+                    for attribute_name in dir(module):
+                        attribute = getattr(module, attribute_name)
+                        if isinstance(attribute, type) and issubclass(attribute, BaseProvider) and attribute is not BaseProvider:
+                            provider_class = attribute
+                            # Instantiate to get metadata, even if not fully configured
+                            temp_provider = provider_class(name=attribute_name, session_config=self.config)
+                            provider_name = temp_provider.get_name()
+                            
+                            # Find the actual provider instance if it exists, to get live stats
+                            live_provider = next((p for p in self.providers if p.get_name() == provider_name), None)
+                            
+                            info[provider_name] = {
+                                'display_name': temp_provider.get_display_name(),
+                                'requires_api_key': temp_provider.requires_api_key(),
+                                'statistics': live_provider.get_statistics() if live_provider else temp_provider.get_statistics(),
+                                'enabled': self.config.is_provider_enabled(provider_name),
+                                'rate_limit': self.config.get_rate_limit(provider_name),
+                            }
+                except Exception as e:
+                    print(f"✗ Failed to get info for provider from {filename}: {e}")
+                    traceback.print_exc()
+        return info

core/session_config.py

@@ -0,0 +1,20 @@
+"""
+Per-session configuration management for DNSRecon.
+Provides isolated configuration instances for each user session.
+"""
+
+from config import Config
+
+class SessionConfig(Config):
+    """
+    Session-specific configuration that inherits from global config
+    but maintains isolated API keys and provider settings.
+    """
+    
+    def __init__(self):
+        """Initialize session config with global defaults."""
+        super().__init__()
+
+def create_session_config() -> 'SessionConfig':
+    """Create a new session configuration instance."""
+    return SessionConfig()

core/session_manager.py

@@ -0,0 +1,391 @@
+# dnsrecon/core/session_manager.py
+
+import threading
+import time
+import uuid
+import redis
+import pickle
+from typing import Dict, Optional, Any, List
+
+from core.scanner import Scanner
+from config import config
+
+# WARNING: Using pickle can be a security risk if the data source is not trusted.
+# In this case, we are only serializing/deserializing our own trusted Scanner objects,
+# which is generally safe. Do not unpickle data from untrusted sources.
+
+class SessionManager:
+    """
+    Manages multiple scanner instances for concurrent user sessions using Redis.
+    """
+    
+    def __init__(self, session_timeout_minutes: int = 0):
+        """
+        Initialize session manager with a Redis backend.
+        """
+        if session_timeout_minutes is None:
+            session_timeout_minutes = config.session_timeout_minutes
+
+        self.redis_client = redis.StrictRedis(db=0, decode_responses=False)
+        self.session_timeout = session_timeout_minutes * 60  # Convert to seconds
+        self.lock = threading.Lock() # Lock for local operations, Redis handles atomic ops
+        
+        # Start cleanup thread
+        self.cleanup_thread = threading.Thread(target=self._cleanup_loop, daemon=True)
+        self.cleanup_thread.start()
+        
+        print(f"SessionManager initialized with Redis backend and {session_timeout_minutes}min timeout")
+    
+    def __getstate__(self):
+        """Prepare SessionManager for pickling."""
+        state = self.__dict__.copy()
+        # Exclude unpickleable attributes - Redis client and threading objects
+        unpicklable_attrs = ['lock', 'cleanup_thread', 'redis_client']
+        for attr in unpicklable_attrs:
+            if attr in state:
+                del state[attr]
+        return state
+
+    def __setstate__(self, state):
+        """Restore SessionManager after unpickling."""
+        self.__dict__.update(state)
+        # Re-initialize unpickleable attributes
+        import redis
+        self.redis_client = redis.StrictRedis(db=0, decode_responses=False)
+        self.lock = threading.Lock()
+        self.cleanup_thread = threading.Thread(target=self._cleanup_loop, daemon=True)
+        self.cleanup_thread.start()
+
+    def _get_session_key(self, session_id: str) -> str:
+        """Generates the Redis key for a session."""
+        return f"dnsrecon:session:{session_id}"
+
+    def _get_stop_signal_key(self, session_id: str) -> str:
+        """Generates the Redis key for a session's stop signal."""
+        return f"dnsrecon:stop:{session_id}"
+
+    def create_session(self) -> str:
+        """
+        Create a new user session and store it in Redis.
+        """
+        session_id = str(uuid.uuid4())
+        print(f"=== CREATING SESSION {session_id} IN REDIS ===")
+        
+        try:
+            from core.session_config import create_session_config
+            session_config = create_session_config()
+            scanner_instance = Scanner(session_config=session_config)
+            
+            # Set the session ID on the scanner for cross-process stop signal management
+            scanner_instance.session_id = session_id
+            
+            session_data = {
+                'scanner': scanner_instance,
+                'config': session_config,
+                'created_at': time.time(),
+                'last_activity': time.time(),
+                'status': 'active'
+            }
+            
+            # Serialize the entire session data dictionary using pickle
+            serialized_data = pickle.dumps(session_data)
+            
+            # Store in Redis
+            session_key = self._get_session_key(session_id)
+            self.redis_client.setex(session_key, self.session_timeout, serialized_data)
+            
+            # Initialize stop signal as False
+            stop_key = self._get_stop_signal_key(session_id)
+            self.redis_client.setex(stop_key, self.session_timeout, b'0')
+            
+            print(f"Session {session_id} stored in Redis with stop signal initialized")
+            return session_id
+            
+        except Exception as e:
+            print(f"ERROR: Failed to create session {session_id}: {e}")
+            raise
+
+    def set_stop_signal(self, session_id: str) -> bool:
+        """
+        Set the stop signal for a session (cross-process safe).
+        
+        Args:
+            session_id: Session identifier
+            
+        Returns:
+            bool: True if signal was set successfully
+        """
+        try:
+            stop_key = self._get_stop_signal_key(session_id)
+            # Set stop signal to '1' with the same TTL as the session
+            self.redis_client.setex(stop_key, self.session_timeout, b'1')
+            print(f"Stop signal set for session {session_id}")
+            return True
+        except Exception as e:
+            print(f"ERROR: Failed to set stop signal for session {session_id}: {e}")
+            return False
+
+    def is_stop_requested(self, session_id: str) -> bool:
+        """
+        Check if stop is requested for a session (cross-process safe).
+        
+        Args:
+            session_id: Session identifier
+            
+        Returns:
+            bool: True if stop is requested
+        """
+        try:
+            stop_key = self._get_stop_signal_key(session_id)
+            value = self.redis_client.get(stop_key)
+            return value == b'1' if value is not None else False
+        except Exception as e:
+            print(f"ERROR: Failed to check stop signal for session {session_id}: {e}")
+            return False
+
+    def clear_stop_signal(self, session_id: str) -> bool:
+        """
+        Clear the stop signal for a session.
+        
+        Args:
+            session_id: Session identifier
+            
+        Returns:
+            bool: True if signal was cleared successfully
+        """
+        try:
+            stop_key = self._get_stop_signal_key(session_id)
+            self.redis_client.setex(stop_key, self.session_timeout, b'0')
+            print(f"Stop signal cleared for session {session_id}")
+            return True
+        except Exception as e:
+            print(f"ERROR: Failed to clear stop signal for session {session_id}: {e}")
+            return False
+
+    def _get_session_data(self, session_id: str) -> Optional[Dict[str, Any]]:
+        """Retrieves and deserializes session data from Redis."""
+        try:
+            session_key = self._get_session_key(session_id)
+            serialized_data = self.redis_client.get(session_key)
+            if serialized_data:
+                session_data = pickle.loads(serialized_data)
+                # Ensure the scanner has the correct session ID for stop signal checking
+                if 'scanner' in session_data and session_data['scanner']:
+                    session_data['scanner'].session_id = session_id
+                return session_data
+            return None
+        except Exception as e:
+            print(f"ERROR: Failed to get session data for {session_id}: {e}")
+            return None
+
+    def _save_session_data(self, session_id: str, session_data: Dict[str, Any]) -> bool:
+        """
+        Serializes and saves session data back to Redis with updated TTL.
+        
+        Returns:
+            bool: True if save was successful
+        """
+        try:
+            session_key = self._get_session_key(session_id)
+            serialized_data = pickle.dumps(session_data)
+            result = self.redis_client.setex(session_key, self.session_timeout, serialized_data)
+            return result
+        except Exception as e:
+            print(f"ERROR: Failed to save session data for {session_id}: {e}")
+            return False
+
+    def update_session_scanner(self, session_id: str, scanner: 'Scanner') -> bool:
+        """
+        Updates just the scanner object in a session with immediate persistence.
+        
+        Returns:
+            bool: True if update was successful
+        """
+        try:
+            session_data = self._get_session_data(session_id)
+            if session_data:
+                # Ensure scanner has the session ID
+                scanner.session_id = session_id
+                session_data['scanner'] = scanner
+                session_data['last_activity'] = time.time()
+                
+                # Immediately save to Redis for GUI updates
+                success = self._save_session_data(session_id, session_data)
+                if success:
+                    print(f"Scanner state updated for session {session_id} (status: {scanner.status})")
+                else:
+                    print(f"WARNING: Failed to save scanner state for session {session_id}")
+                return success
+            else:
+                print(f"WARNING: Session {session_id} not found for scanner update")
+                return False
+        except Exception as e:
+            print(f"ERROR: Failed to update scanner for session {session_id}: {e}")
+            return False
+
+    def update_scanner_status(self, session_id: str, status: str) -> bool:
+        """
+        Quickly update just the scanner status for immediate GUI feedback.
+        
+        Args:
+            session_id: Session identifier
+            status: New scanner status
+            
+        Returns:
+            bool: True if update was successful
+        """
+        try:
+            session_data = self._get_session_data(session_id)
+            if session_data and 'scanner' in session_data:
+                session_data['scanner'].status = status
+                session_data['last_activity'] = time.time()
+                
+                success = self._save_session_data(session_id, session_data)
+                if success:
+                    print(f"Scanner status updated to '{status}' for session {session_id}")
+                else:
+                    print(f"WARNING: Failed to save status update for session {session_id}")
+                return success
+            return False
+        except Exception as e:
+            print(f"ERROR: Failed to update scanner status for session {session_id}: {e}")
+            return False
+
+    def get_session(self, session_id: str) -> Optional[Scanner]:
+        """
+        Get scanner instance for a session from Redis with session ID management.
+        """
+        if not session_id:
+            return None
+            
+        session_data = self._get_session_data(session_id)
+        
+        if not session_data or session_data.get('status') != 'active':
+            return None
+        
+        # Update last activity and save back to Redis
+        session_data['last_activity'] = time.time()
+        self._save_session_data(session_id, session_data)
+        
+        scanner = session_data.get('scanner')
+        if scanner:
+            # Ensure the scanner can check the Redis-based stop signal
+            scanner.session_id = session_id
+        
+        return scanner
+
+    def get_session_status_only(self, session_id: str) -> Optional[str]:
+        """
+        Get just the scanner status without full session retrieval (for performance).
+        
+        Args:
+            session_id: Session identifier
+            
+        Returns:
+            Scanner status string or None if not found
+        """
+        try:
+            session_data = self._get_session_data(session_id)
+            if session_data and 'scanner' in session_data:
+                return session_data['scanner'].status
+            return None
+        except Exception as e:
+            print(f"ERROR: Failed to get session status for {session_id}: {e}")
+            return None
+
+    def terminate_session(self, session_id: str) -> bool:
+        """
+        Terminate a specific session in Redis with reliable stop signal and immediate status update.
+        """
+        print(f"=== TERMINATING SESSION {session_id} ===")
+        
+        try:
+            # First, set the stop signal
+            self.set_stop_signal(session_id)
+            
+            # Update scanner status to stopped immediately for GUI feedback
+            self.update_scanner_status(session_id, 'stopped')
+            
+            session_data = self._get_session_data(session_id)
+            if not session_data:
+                print(f"Session {session_id} not found")
+                return False
+
+            scanner = session_data.get('scanner')
+            if scanner and scanner.status == 'running':
+                print(f"Stopping scan for session: {session_id}")
+                # The scanner will check the Redis stop signal
+                scanner.stop_scan()
+                
+                # Update the scanner state immediately
+                self.update_session_scanner(session_id, scanner)
+            
+            # Wait a moment for graceful shutdown
+            time.sleep(0.5)
+            
+            # Delete session data and stop signal from Redis
+            session_key = self._get_session_key(session_id)
+            stop_key = self._get_stop_signal_key(session_id)
+            self.redis_client.delete(session_key)
+            self.redis_client.delete(stop_key)
+            
+            print(f"Terminated and removed session from Redis: {session_id}")
+            return True
+            
+        except Exception as e:
+            print(f"ERROR: Failed to terminate session {session_id}: {e}")
+            return False
+
+    def _cleanup_loop(self) -> None:
+        """
+        Background thread to cleanup inactive sessions and orphaned stop signals.
+        """
+        while True:
+            try:
+                # Clean up orphaned stop signals
+                stop_keys = self.redis_client.keys("dnsrecon:stop:*")
+                for stop_key in stop_keys:
+                    # Extract session ID from stop key
+                    session_id = stop_key.decode('utf-8').split(':')[-1]
+                    session_key = self._get_session_key(session_id)
+                    
+                    # If session doesn't exist but stop signal does, clean it up
+                    if not self.redis_client.exists(session_key):
+                        self.redis_client.delete(stop_key)
+                        print(f"Cleaned up orphaned stop signal for session {session_id}")
+                        
+            except Exception as e:
+                print(f"Error in cleanup loop: {e}")
+            
+            time.sleep(300)  # Sleep for 5 minutes
+
+    def get_statistics(self) -> Dict[str, Any]:
+        """Get session manager statistics."""
+        try:
+            session_keys = self.redis_client.keys("dnsrecon:session:*")
+            stop_keys = self.redis_client.keys("dnsrecon:stop:*")
+            
+            active_sessions = len(session_keys)
+            running_scans = 0
+            
+            for session_key in session_keys:
+                session_id = session_key.decode('utf-8').split(':')[-1]
+                status = self.get_session_status_only(session_id)
+                if status == 'running':
+                    running_scans += 1
+            
+            return {
+                'total_active_sessions': active_sessions,
+                'running_scans': running_scans,
+                'total_stop_signals': len(stop_keys)
+            }
+        except Exception as e:
+            print(f"ERROR: Failed to get statistics: {e}")
+            return {
+                'total_active_sessions': 0,
+                'running_scans': 0,
+                'total_stop_signals': 0
+            }
+
+# Global session manager instance
+session_manager = SessionManager(session_timeout_minutes=60)

dnsrecon.py

@@ -1,976 +0,0 @@
-#!/usr/bin/env python3
-"""
-Enhanced DNS Reconnaissance Tool with Recursive Analysis
-
-Copyright (c) 2025 mstoeck3. 
-
-Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
-
-1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-"""
-
-import subprocess
-import json
-import requests
-import argparse
-import sys
-import time
-import os
-import re
-import ipaddress
-from datetime import datetime
-from typing import Dict, List, Optional, Any, Set
-from urllib.parse import urlparse
-import threading
-from queue import Queue, Empty
-
-class EnhancedDNSReconTool:
-    def __init__(self, shodan_api_key: Optional[str] = None, virustotal_api_key: Optional[str] = None):
-        self.shodan_api_key = shodan_api_key
-        self.virustotal_api_key = virustotal_api_key
-        self.output_dir = "dns_recon_results"
-        self.session = requests.Session()
-        self.session.headers.update({
-            'User-Agent': 'EnhancedDNSReconTool/2.0 (Educational/Research Purpose)'
-        })
-        
-        # Track processed items to avoid infinite recursion
-        self.processed_domains: Set[str] = set()
-        self.processed_ips: Set[str] = set()
-        
-        # Results storage for recursive analysis
-        self.all_results: Dict[str, Any] = {}
-        
-        # Rate limiting
-        self.last_vt_request = 0
-        self.last_shodan_request = 0
-        self.vt_rate_limit = 4  # 4 requests per minute for free tier
-        self.shodan_rate_limit = 1  # 1 request per second for free tier
-        
-    def check_dependencies(self) -> bool:
-        """Check if required system tools are available."""
-        required_tools = ['dig', 'whois']
-        missing_tools = []
-        
-        for tool in required_tools:
-            try:
-                subprocess.run([tool, '--help'], 
-                             capture_output=True, check=False, timeout=5)
-            except (subprocess.TimeoutExpired, FileNotFoundError):
-                missing_tools.append(tool)
-        
-        if missing_tools:
-            print(f"❌ Missing required tools: {', '.join(missing_tools)}")
-            print("Install with: apt install dnsutils whois (Ubuntu/Debian)")
-            return False
-        return True
-    
-    def run_command(self, cmd: str, timeout: int = 30) -> str:
-        """Run shell command with timeout and error handling."""
-        try:
-            result = subprocess.run(
-                cmd, shell=True, capture_output=True, 
-                text=True, timeout=timeout
-            )
-            return result.stdout.strip() if result.stdout else result.stderr.strip()
-        except subprocess.TimeoutExpired:
-            return "Error: Command timed out"
-        except Exception as e:
-            return f"Error: {str(e)}"
-    
-    def rate_limit_virustotal(self):
-        """Implement rate limiting for VirusTotal API."""
-        current_time = time.time()
-        time_since_last = current_time - self.last_vt_request
-        min_interval = 60 / self.vt_rate_limit  # seconds between requests
-        
-        if time_since_last < min_interval:
-            sleep_time = min_interval - time_since_last
-            print(f"  Rate limiting: waiting {sleep_time:.1f}s for VirusTotal...")
-            time.sleep(sleep_time)
-        
-        self.last_vt_request = time.time()
-    
-    def rate_limit_shodan(self):
-        """Implement rate limiting for Shodan API."""
-        current_time = time.time()
-        time_since_last = current_time - self.last_shodan_request
-        min_interval = 1 / self.shodan_rate_limit  # seconds between requests
-        
-        if time_since_last < min_interval:
-            sleep_time = min_interval - time_since_last
-            time.sleep(sleep_time)
-        
-        self.last_shodan_request = time.time()
-
-    def query_virustotal_domain(self, domain: str) -> Dict[str, Any]:
-        """Query VirusTotal API for domain information."""
-        if not self.virustotal_api_key:
-            return {
-                'success': False,
-                'message': 'No VirusTotal API key provided'
-            }
-        
-        print(f"🔍 Querying VirusTotal for domain: {domain}")
-        
-        try:
-            self.rate_limit_virustotal()
-            
-            url = f"https://www.virustotal.com/vtapi/v2/domain/report"
-            params = {
-                'apikey': self.virustotal_api_key,
-                'domain': domain
-            }
-            
-            response = self.session.get(url, params=params, timeout=30)
-            
-            if response.status_code == 200:
-                data = response.json()
-                
-                # Extract key information
-                result = {
-                    'success': True,
-                    'domain': domain,
-                    'response_code': data.get('response_code', 0),
-                    'verbose_msg': data.get('verbose_msg', ''),
-                    'detection_ratio': f"{data.get('positives', 0)}/{data.get('total', 0)}"
-                }
-                
-                # Add scan results if available
-                if 'scans' in data:
-                    result['scan_engines'] = len(data['scans'])
-                    result['malicious_engines'] = sum(1 for scan in data['scans'].values() if scan.get('detected', False))
-                    result['scan_summary'] = {}
-                    
-                    # Categorize detections
-                    for engine, scan_result in data['scans'].items():
-                        if scan_result.get('detected', False):
-                            category = scan_result.get('result', 'malicious')
-                            if category not in result['scan_summary']:
-                                result['scan_summary'][category] = []
-                            result['scan_summary'][category].append(engine)
-                
-                # Add additional data if available
-                for key in ['subdomains', 'detected_urls', 'undetected_urls', 'resolutions']:
-                    if key in data:
-                        result[key] = data[key]
-                
-                return result
-            else:
-                return {
-                    'success': False,
-                    'error': f"HTTP {response.status_code}",
-                    'message': response.text[:200]
-                }
-                
-        except Exception as e:
-            return {
-                'success': False,
-                'error': str(e),
-                'message': 'VirusTotal domain query failed'
-            }
-
-    def query_virustotal_ip(self, ip: str) -> Dict[str, Any]:
-        """Query VirusTotal API for IP information."""
-        if not self.virustotal_api_key:
-            return {
-                'success': False,
-                'message': 'No VirusTotal API key provided'
-            }
-        
-        print(f"🔍 Querying VirusTotal for IP: {ip}")
-        
-        try:
-            self.rate_limit_virustotal()
-            
-            url = f"https://www.virustotal.com/vtapi/v2/ip-address/report"
-            params = {
-                'apikey': self.virustotal_api_key,
-                'ip': ip
-            }
-            
-            response = self.session.get(url, params=params, timeout=30)
-            
-            if response.status_code == 200:
-                data = response.json()
-                
-                result = {
-                    'success': True,
-                    'ip': ip,
-                    'response_code': data.get('response_code', 0),
-                    'verbose_msg': data.get('verbose_msg', ''),
-                    'detection_ratio': f"{data.get('positives', 0)}/{data.get('total', 0)}"
-                }
-                
-                # Add scan results if available
-                if 'scans' in data:
-                    result['scan_engines'] = len(data['scans'])
-                    result['malicious_engines'] = sum(1 for scan in data['scans'].values() if scan.get('detected', False))
-                
-                # Add additional data
-                for key in ['detected_urls', 'undetected_urls', 'resolutions', 'asn', 'country']:
-                    if key in data:
-                        result[key] = data[key]
-                
-                return result
-            else:
-                return {
-                    'success': False,
-                    'error': f"HTTP {response.status_code}",
-                    'message': response.text[:200]
-                }
-                
-        except Exception as e:
-            return {
-                'success': False,
-                'error': str(e),
-                'message': 'VirusTotal IP query failed'
-            }
-    
-    def get_dns_records(self, domain: str, record_type: str, 
-                       server: Optional[str] = None) -> Dict[str, Any]:
-        """Fetch DNS records with comprehensive error handling and proper parsing."""
-        server_flag = f"@{server}" if server else ""
-        cmd = f"dig {domain} {record_type} {server_flag} +noall +answer"
-        
-        output = self.run_command(cmd)
-        
-        # Parse the output into structured data
-        records = []
-        if output and not output.startswith("Error:"):
-            for line in output.split('\n'):
-                line = line.strip()
-                if line and not line.startswith(';') and not line.startswith('>>'):
-                    # Split on any whitespace (handles both tabs and spaces)
-                    parts = line.split()
-                    
-                    if len(parts) >= 4:
-                        name = parts[0].rstrip('.')
-                        
-                        # Check if second field is numeric (TTL)
-                        if len(parts) >= 5 and parts[1].isdigit():
-                            # Format: name TTL class type data
-                            ttl = parts[1]
-                            dns_class = parts[2]
-                            dns_type = parts[3]
-                            data = ' '.join(parts[4:])
-                        else:
-                            # Format: name class type data (no TTL shown)
-                            ttl = ''
-                            dns_class = parts[1]
-                            dns_type = parts[2]
-                            data = ' '.join(parts[3:]) if len(parts) > 3 else ''
-                        
-                        # Validate that we have the expected record type
-                        if dns_type.upper() == record_type.upper():
-                            records.append({
-                                'name': name,
-                                'ttl': ttl,
-                                'class': dns_class,
-                                'type': dns_type,
-                                'data': data
-                            })
-        
-        return {
-            'query': f"{domain} {record_type}",
-            'server': server or 'system',
-            'raw_output': output,
-            'records': records,
-            'record_count': len(records)
-        }
-    
-    def get_comprehensive_dns(self, domain: str) -> Dict[str, Any]:
-        """Get comprehensive DNS information."""
-        print(f"🔍 Gathering DNS records for {domain}...")
-        
-        # Standard record types
-        record_types = ['A', 'AAAA', 'MX', 'NS', 'SOA', 'TXT', 'CNAME', 
-                       'CAA', 'SRV', 'PTR']
-        
-        # DNS servers to query
-        dns_servers = [
-            None,           # System default
-            '1.1.1.1',      # Cloudflare
-            '8.8.8.8',      # Google
-            '9.9.9.9',      # Quad9
-        ]
-        
-        dns_results = {}
-        
-        for record_type in record_types:
-            dns_results[record_type] = {}
-            for server in dns_servers:
-                server_name = server or 'system'
-                result = self.get_dns_records(domain, record_type, server)
-                dns_results[record_type][server_name] = result
-                
-                time.sleep(0.1)  # Rate limiting
-        
-        # Try DNSSEC validation
-        dnssec_cmd = f"dig {domain} +dnssec +noall +answer"
-        dns_results['DNSSEC'] = {
-            'system': {
-                'query': f"{domain} +dnssec",
-                'raw_output': self.run_command(dnssec_cmd),
-                'records': [],
-                'record_count': 0
-            }
-        }
-        
-        return dns_results
-    
-    def perform_reverse_dns(self, ip: str) -> Dict[str, Any]:
-        """Perform reverse DNS lookup on IP address."""
-        print(f"🔄 Reverse DNS lookup for {ip}")
-        
-        try:
-            # Validate IP address
-            ipaddress.ip_address(ip)
-            
-            # Perform reverse DNS lookup
-            cmd = f"dig -x {ip} +short"
-            output = self.run_command(cmd)
-            
-            hostnames = []
-            if output and not output.startswith("Error:"):
-                hostnames = [line.strip().rstrip('.') for line in output.split('\n') if line.strip()]
-            
-            return {
-                'success': True,
-                'ip': ip,
-                'hostnames': hostnames,
-                'hostname_count': len(hostnames),
-                'raw_output': output
-            }
-            
-        except Exception as e:
-            return {
-                'success': False,
-                'ip': ip,
-                'error': str(e),
-                'hostnames': [],
-                'hostname_count': 0
-            }
-    
-    def extract_subdomains_from_certificates(self, domain: str) -> Set[str]:
-        """Extract subdomains from certificate transparency logs."""
-        print(f"📋 Extracting subdomains from certificates for {domain}")
-        
-        try:
-            url = f"https://crt.sh/?q=%.{domain}&output=json"
-            response = self.session.get(url, timeout=30)
-            
-            subdomains = set()
-            
-            if response.status_code == 200:
-                cert_data = response.json()
-                
-                for cert in cert_data:
-                    name_value = cert.get('name_value', '')
-                    if name_value:
-                        # Handle multiple domains in one certificate
-                        domains_in_cert = [d.strip() for d in name_value.split('\n')]
-                        for subdomain in domains_in_cert:
-                            # Clean up the subdomain
-                            subdomain = subdomain.lower().strip()
-                            if subdomain and '.' in subdomain:
-                                # Only include subdomains of the target domain
-                                if subdomain.endswith(f".{domain}") or subdomain == domain:
-                                    subdomains.add(subdomain)
-                                elif subdomain.startswith("*."):
-                                    # Handle wildcard certificates
-                                    clean_subdomain = subdomain[2:]
-                                    if clean_subdomain.endswith(f".{domain}") or clean_subdomain == domain:
-                                        subdomains.add(clean_subdomain)
-            
-            return subdomains
-            
-        except Exception as e:
-            print(f"  Error extracting subdomains: {e}")
-            return set()
-    
-    def extract_ips_from_dns(self, dns_data: Dict[str, Any]) -> Set[str]:
-        """Extract IP addresses from DNS records."""
-        ips = set()
-        
-        # Extract from A records
-        for server_data in dns_data.get('A', {}).values():
-            for record in server_data.get('records', []):
-                ip = record.get('data', '')
-                if ip and self.is_valid_ip(ip):
-                    ips.add(ip)
-        
-        # Extract from AAAA records
-        for server_data in dns_data.get('AAAA', {}).values():
-            for record in server_data.get('records', []):
-                ipv6 = record.get('data', '')
-                if ipv6 and self.is_valid_ip(ipv6):
-                    ips.add(ipv6)
-        
-        return ips
-    
-    def is_valid_ip(self, ip: str) -> bool:
-        """Check if string is a valid IP address."""
-        try:
-            ipaddress.ip_address(ip)
-            return True
-        except ValueError:
-            return False
-    
-    def get_whois_data(self, domain: str) -> Dict[str, Any]:
-        """Fetch and parse WHOIS data with improved parsing."""
-        print(f"📋 Fetching WHOIS data for {domain}...")
-        
-        raw_whois = self.run_command(f"whois {domain}")
-        
-        # Basic parsing of common WHOIS fields
-        whois_data = {
-            'raw': raw_whois,
-            'parsed': {}
-        }
-        
-        if not raw_whois.startswith("Error:"):
-            lines = raw_whois.split('\n')
-            for line in lines:
-                line = line.strip()
-                if ':' in line and not line.startswith('%') and not line.startswith('#') and not line.startswith('>>>'):
-                    # Handle different WHOIS formats
-                    if line.count(':') == 1:
-                        key, value = line.split(':', 1)
-                    else:
-                        # Multiple colons - take first as key, rest as value
-                        parts = line.split(':', 2)
-                        key, value = parts[0], ':'.join(parts[1:])
-                    
-                    key = key.strip().lower().replace(' ', '_').replace('-', '_')
-                    value = value.strip()
-                    if value and key:
-                        # Handle multiple values for same key (like name servers)
-                        if key in whois_data['parsed']:
-                            # Convert to list if not already
-                            if not isinstance(whois_data['parsed'][key], list):
-                                whois_data['parsed'][key] = [whois_data['parsed'][key]]
-                            whois_data['parsed'][key].append(value)
-                        else:
-                            whois_data['parsed'][key] = value
-        
-        return whois_data
-    
-    def get_certificate_transparency(self, domain: str) -> Dict[str, Any]:
-        """Query certificate transparency logs via crt.sh."""
-        print(f"🔐 Querying certificate transparency logs for {domain}...")
-        
-        try:
-            # Query crt.sh API
-            url = f"https://crt.sh/?q=%.{domain}&output=json"
-            response = self.session.get(url, timeout=30)
-            
-            if response.status_code == 200:
-                cert_data = response.json()
-                
-                # Extract unique subdomains
-                subdomains = set()
-                cert_details = []
-                
-                for cert in cert_data:
-                    # Extract subdomains from name_value
-                    name_value = cert.get('name_value', '')
-                    if name_value:
-                        # Handle multiple domains in one certificate
-                        domains_in_cert = [d.strip() for d in name_value.split('\n')]
-                        subdomains.update(domains_in_cert)
-                    
-                    cert_details.append({
-                        'id': cert.get('id'),
-                        'issuer': cert.get('issuer_name'),
-                        'common_name': cert.get('common_name'),
-                        'name_value': cert.get('name_value'),
-                        'not_before': cert.get('not_before'),
-                        'not_after': cert.get('not_after'),
-                        'serial_number': cert.get('serial_number')
-                    })
-                
-                return {
-                    'success': True,
-                    'total_certificates': len(cert_data),
-                    'unique_subdomains': sorted(list(subdomains)),
-                    'subdomain_count': len(subdomains),
-                    'certificates': cert_details[:50]  # Limit for output size
-                }
-            else:
-                return {
-                    'success': False,
-                    'error': f"HTTP {response.status_code}",
-                    'message': 'Failed to fetch certificate data'
-                }
-                
-        except Exception as e:
-            return {
-                'success': False,
-                'error': str(e),
-                'message': 'Request to crt.sh failed'
-            }
-    
-    def query_shodan(self, domain: str) -> Dict[str, Any]:
-        """Query Shodan API for domain information."""
-        if not self.shodan_api_key:
-            return {
-                'success': False,
-                'message': 'No Shodan API key provided'
-            }
-        
-        print(f"🔎 Querying Shodan for {domain}...")
-        
-        try:
-            self.rate_limit_shodan()
-            
-            # Search for the domain
-            url = f"https://api.shodan.io/shodan/host/search"
-            params = {
-                'key': self.shodan_api_key,
-                'query': f'hostname:{domain}'
-            }
-            
-            response = self.session.get(url, params=params, timeout=30)
-            
-            if response.status_code == 200:
-                data = response.json()
-                return {
-                    'success': True,
-                    'total_results': data.get('total', 0),
-                    'matches': data.get('matches', [])[:10],  # Limit results
-                    'facets': data.get('facets', {})
-                }
-            else:
-                return {
-                    'success': False,
-                    'error': f"HTTP {response.status_code}",
-                    'message': response.text[:200]
-                }
-                
-        except Exception as e:
-            return {
-                'success': False,
-                'error': str(e),
-                'message': 'Shodan query failed'
-            }
-
-    def query_shodan_ip(self, ip: str) -> Dict[str, Any]:
-        """Query Shodan API for IP information."""
-        if not self.shodan_api_key:
-            return {
-                'success': False,
-                'message': 'No Shodan API key provided'
-            }
-        
-        print(f"🔎 Querying Shodan for IP {ip}...")
-        
-        try:
-            self.rate_limit_shodan()
-            
-            url = f"https://api.shodan.io/shodan/host/{ip}"
-            params = {'key': self.shodan_api_key}
-            
-            response = self.session.get(url, params=params, timeout=30)
-            
-            if response.status_code == 200:
-                data = response.json()
-                return {
-                    'success': True,
-                    'ip': ip,
-                    'data': data
-                }
-            else:
-                return {
-                    'success': False,
-                    'error': f"HTTP {response.status_code}",
-                    'message': response.text[:200]
-                }
-                
-        except Exception as e:
-            return {
-                'success': False,
-                'error': str(e),
-                'message': 'Shodan IP query failed'
-            }
-    
-    def analyze_domain_recursively(self, domain: str, depth: int = 0, max_depth: int = 2) -> Dict[str, Any]:
-        """Perform comprehensive analysis on a domain with recursive subdomain discovery."""
-        if domain in self.processed_domains or depth > max_depth:
-            return {}
-        
-        self.processed_domains.add(domain)
-        
-        print(f"\n{'  ' * depth}🎯 Analyzing domain: {domain} (depth {depth})")
-        
-        results = {
-            'domain': domain,
-            'timestamp': datetime.now().isoformat(),
-            'depth': depth,
-            'dns_records': {},
-            'whois': {},
-            'certificate_transparency': {},
-            'virustotal_domain': {},
-            'shodan': {},
-            'discovered_ips': {},
-            'discovered_subdomains': {}
-        }
-        
-        # DNS Records
-        results['dns_records'] = self.get_comprehensive_dns(domain)
-        
-        # Extract IP addresses from DNS records
-        discovered_ips = self.extract_ips_from_dns(results['dns_records'])
-        
-        # WHOIS (only for primary domain to avoid rate limiting)
-        if depth == 0:
-            results['whois'] = self.get_whois_data(domain)
-        
-        # Certificate Transparency
-        results['certificate_transparency'] = self.get_certificate_transparency(domain)
-        
-        # VirusTotal Domain Analysis
-        results['virustotal_domain'] = self.query_virustotal_domain(domain)
-        
-        # Shodan Domain Analysis
-        results['shodan'] = self.query_shodan(domain)
-        
-        # Extract subdomains from certificate transparency
-        if depth < max_depth:
-            subdomains = self.extract_subdomains_from_certificates(domain)
-            
-            # Filter out already processed subdomains
-            new_subdomains = subdomains - self.processed_domains
-            new_subdomains.discard(domain)  # Remove the current domain itself
-            
-            print(f"{'  ' * depth}📋 Found {len(new_subdomains)} new subdomains to analyze")
-            
-            # Recursively analyze subdomains (limit to prevent excessive recursion)
-            for subdomain in list(new_subdomains)[:20]:  # Limit to 20 subdomains per domain
-                if subdomain not in self.processed_domains:
-                    subdomain_results = self.analyze_domain_recursively(subdomain, depth + 1, max_depth)
-                    if subdomain_results:
-                        results['discovered_subdomains'][subdomain] = subdomain_results
-        
-        # Analyze discovered IP addresses
-        for ip in discovered_ips:
-            if ip not in self.processed_ips:
-                ip_results = self.analyze_ip_recursively(ip, depth)
-                if ip_results:
-                    results['discovered_ips'][ip] = ip_results
-        
-        # Store in global results
-        self.all_results[domain] = results
-        
-        return results
-    
-    def analyze_ip_recursively(self, ip: str, depth: int = 0) -> Dict[str, Any]:
-        """Perform comprehensive analysis on an IP address."""
-        if ip in self.processed_ips:
-            return {}
-        
-        self.processed_ips.add(ip)
-        
-        print(f"{'  ' * depth}🌐 Analyzing IP: {ip}")
-        
-        results = {
-            'ip': ip,
-            'timestamp': datetime.now().isoformat(),
-            'reverse_dns': {},
-            'virustotal_ip': {},
-            'shodan_ip': {},
-            'discovered_domains': {}
-        }
-        
-        # Reverse DNS lookup
-        results['reverse_dns'] = self.perform_reverse_dns(ip)
-        
-        # VirusTotal IP Analysis
-        results['virustotal_ip'] = self.query_virustotal_ip(ip)
-        
-        # Shodan IP Analysis
-        results['shodan_ip'] = self.query_shodan_ip(ip)
-        
-        # Analyze discovered domains from reverse DNS
-        reverse_dns = results['reverse_dns']
-        if reverse_dns.get('success') and reverse_dns.get('hostnames'):
-            for hostname in reverse_dns['hostnames'][:5]:  # Limit to 5 hostnames
-                if hostname not in self.processed_domains and hostname.count('.') >= 1:
-                    # Only analyze if it's a reasonable hostname and not already processed
-                    domain_results = self.analyze_domain_recursively(hostname, depth + 1, max_depth=1)
-                    if domain_results:
-                        results['discovered_domains'][hostname] = domain_results
-        
-        return results
-    
-    def create_comprehensive_summary(self, filename: str) -> None:
-        """Create comprehensive summary report with recursive analysis results."""
-        with open(filename, 'w', encoding='utf-8') as f:
-            f.write("Enhanced DNS Reconnaissance Report with Recursive Analysis\n")
-            f.write("=" * 65 + "\n")
-            f.write(f"Analysis completed at: {datetime.now().isoformat()}\n")
-            f.write(f"Total domains analyzed: {len(self.processed_domains)}\n")
-            f.write(f"Total IP addresses analyzed: {len(self.processed_ips)}\n\n")
-            
-            # Executive Summary
-            f.write("EXECUTIVE SUMMARY\n")
-            f.write("-" * 17 + "\n")
-            
-            total_threats = 0
-            domains_with_issues = []
-            ips_with_issues = []
-            
-            # Count threats across all analyzed domains and IPs
-            for domain, domain_data in self.all_results.items():
-                # Check VirusTotal results for domain
-                vt_domain = domain_data.get('virustotal_domain', {})
-                if vt_domain.get('success') and vt_domain.get('malicious_engines', 0) > 0:
-                    total_threats += 1
-                    domains_with_issues.append(domain)
-                
-                # Check discovered IPs
-                for ip, ip_data in domain_data.get('discovered_ips', {}).items():
-                    vt_ip = ip_data.get('virustotal_ip', {})
-                    if vt_ip.get('success') and vt_ip.get('malicious_engines', 0) > 0:
-                        total_threats += 1
-                        ips_with_issues.append(ip)
-            
-            f.write(f"Security Status: {'⚠️  THREATS DETECTED' if total_threats > 0 else '✅ NO THREATS DETECTED'}\n")
-            f.write(f"Total Security Issues: {total_threats}\n")
-            if domains_with_issues:
-                f.write(f"Domains with issues: {', '.join(domains_with_issues[:5])}\n")
-            if ips_with_issues:
-                f.write(f"IPs with issues: {', '.join(ips_with_issues[:5])}\n")
-            f.write("\n")
-            
-            # Process each domain in detail
-            for domain, domain_data in self.all_results.items():
-                if domain_data.get('depth', 0) == 0:  # Only show primary domains in detail
-                    self._write_domain_analysis(f, domain, domain_data)
-            
-            # Summary of all discovered assets
-            f.write("\nASSET DISCOVERY SUMMARY\n")
-            f.write("-" * 23 + "\n")
-            f.write(f"All Discovered Domains ({len(self.processed_domains)}):\n")
-            for domain in sorted(self.processed_domains):
-                f.write(f"  {domain}\n")
-            
-            f.write(f"\nAll Discovered IP Addresses ({len(self.processed_ips)}):\n")
-            for ip in sorted(self.processed_ips, key=ipaddress.IPv4Address):
-                f.write(f"  {ip}\n")
-            
-            f.write(f"\n{'=' * 65}\n")
-            f.write("Report Generation Complete\n")
-
-    def _write_domain_analysis(self, f, domain: str, domain_data: Dict[str, Any]) -> None:
-        """Write detailed domain analysis to file."""
-        f.write(f"\nDETAILED ANALYSIS: {domain.upper()}\n")
-        f.write("=" * (20 + len(domain)) + "\n")
-        
-        # DNS Records Summary
-        dns_data = domain_data.get('dns_records', {})
-        f.write("DNS Records Summary:\n")
-        for record_type in ['A', 'AAAA', 'MX', 'NS', 'TXT']:
-            system_records = dns_data.get(record_type, {}).get('system', {}).get('records', [])
-            f.write(f"  {record_type}: {len(system_records)} records\n")
-        
-        # Security Analysis
-        f.write(f"\nSecurity Analysis:\n")
-        
-        # VirusTotal Domain Results
-        vt_domain = domain_data.get('virustotal_domain', {})
-        if vt_domain.get('success'):
-            detection_ratio = vt_domain.get('detection_ratio', '0/0')
-            malicious_engines = vt_domain.get('malicious_engines', 0)
-            f.write(f"  VirusTotal Domain: {detection_ratio} ({malicious_engines} flagged as malicious)\n")
-            
-            if malicious_engines > 0:
-                f.write(f"  ⚠️  SECURITY ALERT: Domain flagged by {malicious_engines} security engines\n")
-                scan_summary = vt_domain.get('scan_summary', {})
-                for category, engines in scan_summary.items():
-                    f.write(f"    {category}: {', '.join(engines[:3])}\n")
-        else:
-            f.write(f"  VirusTotal Domain: {vt_domain.get('message', 'Not available')}\n")
-        
-        # Certificate Information
-        cert_data = domain_data.get('certificate_transparency', {})
-        if cert_data.get('success'):
-            f.write(f"  SSL Certificates: {cert_data.get('total_certificates', 0)} found\n")
-            f.write(f"  Subdomains from Certificates: {cert_data.get('subdomain_count', 0)}\n")
-        
-        # Discovered Assets
-        discovered_ips = domain_data.get('discovered_ips', {})
-        discovered_subdomains = domain_data.get('discovered_subdomains', {})
-        
-        if discovered_ips:
-            f.write(f"\nDiscovered IP Addresses ({len(discovered_ips)}):\n")
-            for ip, ip_data in discovered_ips.items():
-                vt_ip = ip_data.get('virustotal_ip', {})
-                reverse_dns = ip_data.get('reverse_dns', {})
-                
-                f.write(f"  {ip}:\n")
-                
-                # Reverse DNS
-                if reverse_dns.get('success') and reverse_dns.get('hostnames'):
-                    f.write(f"    Reverse DNS: {', '.join(reverse_dns['hostnames'][:3])}\n")
-                
-                # VirusTotal IP results
-                if vt_ip.get('success'):
-                    detection_ratio = vt_ip.get('detection_ratio', '0/0')
-                    malicious_engines = vt_ip.get('malicious_engines', 0)
-                    f.write(f"    VirusTotal: {detection_ratio}")
-                    if malicious_engines > 0:
-                        f.write(f" ⚠️  FLAGGED BY {malicious_engines} ENGINES")
-                    f.write("\n")
-                
-                # Shodan IP results
-                shodan_ip = ip_data.get('shodan_ip', {})
-                if shodan_ip.get('success'):
-                    shodan_data = shodan_ip.get('data', {})
-                    ports = shodan_data.get('ports', [])
-                    if ports:
-                        f.write(f"    Shodan Ports: {', '.join(map(str, ports[:10]))}\n")
-                
-                f.write("\n")
-        
-        if discovered_subdomains:
-            f.write(f"Discovered Subdomains ({len(discovered_subdomains)}):\n")
-            for subdomain, subdomain_data in discovered_subdomains.items():
-                f.write(f"  {subdomain}\n")
-                
-                # Quick security check for subdomain
-                vt_subdomain = subdomain_data.get('virustotal_domain', {})
-                if vt_subdomain.get('success') and vt_subdomain.get('malicious_engines', 0) > 0:
-                    f.write(f"    ⚠️  Security Issue: Flagged by VirusTotal\n")
-                
-                subdomain_ips = subdomain_data.get('discovered_ips', {})
-                if subdomain_ips:
-                    f.write(f"    IPs: {', '.join(list(subdomain_ips.keys())[:3])}\n")
-        
-        f.write("\n")
-
-    def save_results(self, domain: str) -> None:
-        """Save results in multiple formats."""
-        if not os.path.exists(self.output_dir):
-            os.makedirs(self.output_dir)
-        
-        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
-        base_filename = f"{self.output_dir}/{domain}_{timestamp}"
-        
-        # Save complete JSON (all recursive data)
-        json_file = f"{base_filename}_complete.json"
-        with open(json_file, 'w', encoding='utf-8') as f:
-            json.dump(self.all_results, f, indent=2, ensure_ascii=False, default=str)
-        
-        # Save comprehensive summary
-        summary_file = f"{base_filename}_analysis.txt"
-        self.create_comprehensive_summary(summary_file)
-        
-        # Save asset list (domains and IPs)
-        assets_file = f"{base_filename}_assets.txt"
-        with open(assets_file, 'w', encoding='utf-8') as f:
-            f.write("Discovered Assets Summary\n")
-            f.write("=" * 25 + "\n\n")
-            
-            f.write(f"Domains ({len(self.processed_domains)}):\n")
-            for domain in sorted(self.processed_domains):
-                f.write(f"{domain}\n")
-            
-            f.write(f"\nIP Addresses ({len(self.processed_ips)}):\n")
-            for ip in sorted(self.processed_ips, key=lambda x: ipaddress.IPv4Address(x)):
-                f.write(f"{ip}\n")
-        
-        print(f"\n📄 Results saved:")
-        print(f"   Complete JSON: {json_file}")
-        print(f"   Analysis Report: {summary_file}")
-        print(f"   Asset List: {assets_file}")
-
-    def run_enhanced_reconnaissance(self, domain: str, max_depth: int = 2) -> Dict[str, Any]:
-        """Run enhanced recursive DNS reconnaissance."""
-        print(f"\n🚀 Starting enhanced DNS reconnaissance for: {domain}")
-        print(f"   Max recursion depth: {max_depth}")
-        print(f"   APIs enabled: VirusTotal={bool(self.virustotal_api_key)}, Shodan={bool(self.shodan_api_key)}")
-        
-        start_time = time.time()
-        
-        # Clear previous results
-        self.processed_domains.clear()
-        self.processed_ips.clear()
-        self.all_results.clear()
-        
-        # Start recursive analysis
-        results = self.analyze_domain_recursively(domain, depth=0, max_depth=max_depth)
-        
-        end_time = time.time()
-        duration = end_time - start_time
-        
-        print(f"\n✅ Enhanced reconnaissance completed in {duration:.1f} seconds")
-        print(f"   Domains analyzed: {len(self.processed_domains)}")
-        print(f"   IP addresses analyzed: {len(self.processed_ips)}")
-        
-        return results
-
-def main():
-    parser = argparse.ArgumentParser(
-        description="Enhanced DNS Reconnaissance Tool with Recursive Analysis - Use only on domains you own or have permission to test",
-        epilog="LEGAL NOTICE: Unauthorized reconnaissance may violate applicable laws. Use responsibly."
-    )
-    parser.add_argument('domain', help='Target domain (e.g., example.com)')
-    parser.add_argument('--shodan-key', help='Shodan API key for additional reconnaissance')
-    parser.add_argument('--virustotal-key', help='VirusTotal API key for threat intelligence')
-    parser.add_argument('--max-depth', type=int, default=2, 
-                       help='Maximum recursion depth for subdomain analysis (default: 2)')
-    parser.add_argument('--output-dir', default='dns_recon_results', 
-                       help='Output directory for results')
-    
-    args = parser.parse_args()
-    
-    # Validate domain format
-    if not re.match(r'^[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$', args.domain):
-        print("❌ Invalid domain format. Please provide a valid domain (e.g., example.com)")
-        sys.exit(1)
-    
-    # Initialize tool
-    tool = EnhancedDNSReconTool(
-        shodan_api_key=args.shodan_key,
-        virustotal_api_key=args.virustotal_key
-    )
-    tool.output_dir = args.output_dir
-    
-    # Check dependencies
-    if not tool.check_dependencies():
-        sys.exit(1)
-    
-    # Warn about API keys
-    if not args.virustotal_key:
-        print("⚠️  No VirusTotal API key provided. Threat intelligence will be limited.")
-    if not args.shodan_key:
-        print("⚠️  No Shodan API key provided. Host intelligence will be limited.")
-    
-    try:
-        # Run enhanced reconnaissance
-        results = tool.run_enhanced_reconnaissance(args.domain, args.max_depth)
-        
-        # Save results
-        tool.save_results(args.domain)
-        
-        print(f"\n🎯 Enhanced reconnaissance completed for {args.domain}")
-        
-    except KeyboardInterrupt:
-        print("\n⏹️  Reconnaissance interrupted by user")
-        sys.exit(0)
-    except Exception as e:
-        print(f"❌ Error during reconnaissance: {e}")
-        import traceback
-        traceback.print_exc()
-        sys.exit(1)
-
-if __name__ == "__main__":
-    main()

providers/__init__.py

@@ -0,0 +1,19 @@
+"""
+Data provider modules for DNSRecon.
+Contains implementations for various reconnaissance data sources.
+"""
+
+from .base_provider import BaseProvider, RateLimiter
+from .crtsh_provider import CrtShProvider
+from .dns_provider import DNSProvider
+from .shodan_provider import ShodanProvider
+
+__all__ = [
+    'BaseProvider',
+    'RateLimiter', 
+    'CrtShProvider',
+    'DNSProvider',
+    'ShodanProvider'
+]
+
+__version__ = "0.0.0-rc"

providers/base_provider.py

@@ -0,0 +1,301 @@
+# dnsrecon/providers/base_provider.py
+
+import time
+import requests
+import threading
+from abc import ABC, abstractmethod
+from typing import List, Dict, Any, Optional, Tuple
+
+from core.logger import get_forensic_logger
+
+
+class RateLimiter:
+    """Simple rate limiter for API calls."""
+
+    def __init__(self, requests_per_minute: int):
+        """
+        Initialize rate limiter.
+
+        Args:
+            requests_per_minute: Maximum requests allowed per minute
+        """
+        self.requests_per_minute = requests_per_minute
+        self.min_interval = 60.0 / requests_per_minute
+        self.last_request_time = 0
+
+    def __getstate__(self):
+        """RateLimiter is fully picklable, return full state."""
+        return self.__dict__.copy()
+
+    def __setstate__(self, state):
+        """Restore RateLimiter state."""
+        self.__dict__.update(state)
+
+    def wait_if_needed(self) -> None:
+        """Wait if necessary to respect rate limits."""
+        current_time = time.time()
+        time_since_last = current_time - self.last_request_time
+
+        if time_since_last < self.min_interval:
+            sleep_time = self.min_interval - time_since_last
+            time.sleep(sleep_time)
+
+        self.last_request_time = time.time()
+
+
+class BaseProvider(ABC):
+    """
+    Abstract base class for all DNSRecon data providers.
+    Now supports session-specific configuration.
+    """
+
+    def __init__(self, name: str, rate_limit: int = 60, timeout: int = 30, session_config=None):
+        """
+        Initialize base provider with session-specific configuration.
+
+        Args:
+            name: Provider name for logging
+            rate_limit: Requests per minute limit (default override)
+            timeout: Request timeout in seconds
+            session_config: Session-specific configuration
+        """
+        # Use session config if provided, otherwise fall back to global config
+        if session_config is not None:
+            self.config = session_config
+            actual_rate_limit = self.config.get_rate_limit(name)
+            actual_timeout = self.config.default_timeout
+        else:
+            # Fallback to global config for backwards compatibility
+            from config import config as global_config
+            self.config = global_config
+            actual_rate_limit = rate_limit
+            actual_timeout = timeout
+
+        self.name = name
+        self.rate_limiter = RateLimiter(actual_rate_limit)
+        self.timeout = actual_timeout
+        self._local = threading.local()
+        self.logger = get_forensic_logger()
+        self._stop_event = None
+
+        # Statistics (per provider instance)
+        self.total_requests = 0
+        self.successful_requests = 0
+        self.failed_requests = 0
+        self.total_relationships_found = 0
+
+    def __getstate__(self):
+        """Prepare BaseProvider for pickling by excluding unpicklable objects."""
+        state = self.__dict__.copy()
+        # Exclude the unpickleable '_local' attribute and stop event
+        unpicklable_attrs = ['_local', '_stop_event']
+        for attr in unpicklable_attrs:
+            if attr in state:
+                del state[attr]
+        return state
+
+    def __setstate__(self, state):
+        """Restore BaseProvider after unpickling by reconstructing threading objects."""
+        self.__dict__.update(state)
+        # Re-initialize the '_local' attribute and stop event
+        self._local = threading.local()
+        self._stop_event = None
+
+    @property
+    def session(self):
+        if not hasattr(self._local, 'session'):
+            self._local.session = requests.Session()
+            self._local.session.headers.update({
+                'User-Agent': 'DNSRecon/1.0 (Passive Reconnaissance Tool)'
+            })
+        return self._local.session
+
+    @abstractmethod
+    def get_name(self) -> str:
+        """Return the provider name."""
+        pass
+
+    @abstractmethod
+    def get_display_name(self) -> str:
+        """Return the provider display name for the UI."""
+        pass
+
+    @abstractmethod
+    def requires_api_key(self) -> bool:
+        """Return True if the provider requires an API key."""
+        pass
+
+    @abstractmethod
+    def get_eligibility(self) -> Dict[str, bool]:
+        """Return a dictionary indicating if the provider can query domains and/or IPs."""
+        pass
+
+    @abstractmethod
+    def is_available(self) -> bool:
+        """Check if the provider is available and properly configured."""
+        pass
+
+    @abstractmethod
+    def query_domain(self, domain: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]:
+        """
+        Query the provider for information about a domain.
+
+        Args:
+            domain: Domain to investigate
+
+        Returns:
+            List of tuples: (source_node, target_node, relationship_type, confidence, raw_data)
+        """
+        pass
+
+    @abstractmethod
+    def query_ip(self, ip: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]:
+        """
+        Query the provider for information about an IP address.
+
+        Args:
+            ip: IP address to investigate
+
+        Returns:
+            List of tuples: (source_node, target_node, relationship_type, confidence, raw_data)
+        """
+        pass
+
+    def make_request(self, url: str, method: str = "GET",
+                    params: Optional[Dict[str, Any]] = None,
+                    headers: Optional[Dict[str, str]] = None,
+                    target_indicator: str = "") -> Optional[requests.Response]:
+        """
+        Make a rate-limited HTTP request.
+        """
+        if self._is_stop_requested():
+            print(f"Request cancelled before start: {url}")
+            return None
+
+        self.rate_limiter.wait_if_needed()
+
+        start_time = time.time()
+        response = None
+        error = None
+
+        try:
+            self.total_requests += 1
+
+            request_headers = dict(self.session.headers).copy()
+            if headers:
+                request_headers.update(headers)
+
+            print(f"Making {method} request to: {url}")
+
+            if method.upper() == "GET":
+                response = self.session.get(
+                    url,
+                    params=params,
+                    headers=request_headers,
+                    timeout=self.timeout
+                )
+            elif method.upper() == "POST":
+                response = self.session.post(
+                    url,
+                    json=params,
+                    headers=request_headers,
+                    timeout=self.timeout
+                )
+            else:
+                raise ValueError(f"Unsupported HTTP method: {method}")
+
+            print(f"Response status: {response.status_code}")
+            response.raise_for_status()
+            self.successful_requests += 1
+            
+            duration_ms = (time.time() - start_time) * 1000
+            self.logger.log_api_request(
+                provider=self.name,
+                url=url,
+                method=method.upper(),
+                status_code=response.status_code,
+                response_size=len(response.content),
+                duration_ms=duration_ms,
+                error=None,
+                target_indicator=target_indicator
+            )
+            
+            return response
+
+        except requests.exceptions.RequestException as e:
+            error = str(e)
+            self.failed_requests += 1
+            duration_ms = (time.time() - start_time) * 1000
+            self.logger.log_api_request(
+                provider=self.name,
+                url=url,
+                method=method.upper(),
+                status_code=response.status_code if response else None,
+                response_size=len(response.content) if response else None,
+                duration_ms=duration_ms,
+                error=error,
+                target_indicator=target_indicator
+            )
+            raise e
+
+    def _is_stop_requested(self) -> bool:
+        """
+        Enhanced stop signal checking that handles both local and Redis-based signals.
+        """
+        if hasattr(self, '_stop_event') and self._stop_event and self._stop_event.is_set():
+            return True
+        return False
+
+    def set_stop_event(self, stop_event: threading.Event) -> None:
+        """
+        Set the stop event for this provider to enable cancellation.
+        
+        Args:
+            stop_event: Threading event to signal cancellation
+        """
+        self._stop_event = stop_event
+
+    def log_relationship_discovery(self, source_node: str, target_node: str,
+                                 relationship_type: str,
+                                 confidence_score: float,
+                                 raw_data: Dict[str, Any],
+                                 discovery_method: str) -> None:
+        """
+        Log discovery of a new relationship.
+
+        Args:
+            source_node: Source node identifier
+            target_node: Target node identifier
+            relationship_type: Type of relationship
+            confidence_score: Confidence score
+            raw_data: Raw data from provider
+            discovery_method: Method used for discovery
+        """
+        self.total_relationships_found += 1
+
+        self.logger.log_relationship_discovery(
+            source_node=source_node,
+            target_node=target_node,
+            relationship_type=relationship_type,
+            confidence_score=confidence_score,
+            provider=self.name,
+            raw_data=raw_data,
+            discovery_method=discovery_method
+        )
+
+    def get_statistics(self) -> Dict[str, Any]:
+        """
+        Get provider statistics.
+
+        Returns:
+            Dictionary containing provider performance metrics
+        """
+        return {
+            'name': self.name,
+            'total_requests': self.total_requests,
+            'successful_requests': self.successful_requests,
+            'failed_requests': self.failed_requests,
+            'success_rate': (self.successful_requests / self.total_requests * 100) if self.total_requests > 0 else 0,
+            'relationships_found': self.total_relationships_found,
+            'rate_limit': self.rate_limiter.requests_per_minute
+        }

providers/crtsh_provider.py

@@ -0,0 +1,513 @@
+# dnsrecon/providers/crtsh_provider.py
+
+import json
+import re
+import os
+from pathlib import Path
+from typing import List, Dict, Any, Tuple, Set
+from datetime import datetime, timezone
+
+# New dependency required for this provider
+try:
+    import psycopg2
+    import psycopg2.extras
+    PSYCOPG2_AVAILABLE = True
+except ImportError:
+    PSYCOPG2_AVAILABLE = False
+
+from .base_provider import BaseProvider
+from utils.helpers import _is_valid_domain
+
+# We use requests only to raise the same exception type for compatibility with core retry logic
+import requests
+
+
+class CrtShProvider(BaseProvider):
+    """
+    Provider for querying crt.sh certificate transparency database via its public PostgreSQL endpoint.
+    This version is designed to be a drop-in, high-performance replacement for the API-based provider.
+    It preserves the same caching and data processing logic.
+    """
+
+    def __init__(self, name=None, session_config=None):
+        """Initialize CrtShDB provider with session-specific configuration."""
+        super().__init__(
+            name="crtsh",
+            rate_limit=0,  # No rate limit for direct DB access
+            timeout=60,    # Increased timeout for potentially long DB queries
+            session_config=session_config
+        )
+        # Database connection details
+        self.db_host = "crt.sh"
+        self.db_port = 5432
+        self.db_name = "certwatch"
+        self.db_user = "guest"
+        self._stop_event = None
+
+        # Initialize cache directory (same as original provider)
+        self.cache_dir = Path('cache') / 'crtsh'
+        self.cache_dir.mkdir(parents=True, exist_ok=True)
+
+    def get_name(self) -> str:
+        """Return the provider name."""
+        return "crtsh"
+
+    def get_display_name(self) -> str:
+        """Return the provider display name for the UI."""
+        return "crt.sh (DB)"
+
+    def requires_api_key(self) -> bool:
+        """Return True if the provider requires an API key."""
+        return False
+
+    def get_eligibility(self) -> Dict[str, bool]:
+        """Return a dictionary indicating if the provider can query domains and/or IPs."""
+        return {'domains': True, 'ips': False}
+
+    def is_available(self) -> bool:
+        """
+        Check if the provider can be used. Requires the psycopg2 library.
+        """
+        if not PSYCOPG2_AVAILABLE:
+            self.logger.logger.warning("psycopg2 library not found. CrtShDBProvider is unavailable. "
+                                      "Please run 'pip install psycopg2-binary'.")
+            return False
+        return True
+
+    def _query_crtsh(self, domain: str) -> List[Dict[str, Any]]:
+        """
+        Query the crt.sh PostgreSQL database for raw certificate data.
+        Raises exceptions for DB/network errors to allow core logic to retry.
+        """
+        conn = None
+        certificates = []
+        
+        # SQL Query to find all certificate IDs related to the domain (including subdomains),
+        # then retrieve comprehensive details for each certificate, mimicking the JSON API structure.
+        sql_query = """
+        WITH certificates_of_interest AS (
+            SELECT DISTINCT ci.certificate_id
+            FROM certificate_identity ci
+            WHERE ci.name_value ILIKE %(domain_wildcard)s OR ci.name_value = %(domain)s
+        )
+        SELECT
+            c.id,
+            c.serial_number,
+            c.not_before,
+            c.not_after,
+            (SELECT min(entry_timestamp) FROM ct_log_entry cle WHERE cle.certificate_id = c.id) as entry_timestamp,
+            ca.id as issuer_ca_id,
+            ca.name as issuer_name,
+            (SELECT array_to_string(array_agg(DISTINCT ci.name_value), E'\n') FROM certificate_identity ci WHERE ci.certificate_id = c.id) as name_value,
+            (SELECT name_value FROM certificate_identity ci WHERE ci.certificate_id = c.id AND ci.name_type = 'commonName' LIMIT 1) as common_name
+        FROM
+            certificate c
+        JOIN ca ON c.issuer_ca_id = ca.id
+        WHERE c.id IN (SELECT certificate_id FROM certificates_of_interest);
+        """
+        
+        try:
+            conn = psycopg2.connect(
+                dbname=self.db_name,
+                user=self.db_user,
+                host=self.db_host,
+                port=self.db_port,
+                connect_timeout=self.timeout
+            )
+            
+            with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cursor:
+                cursor.execute(sql_query, {'domain': domain, 'domain_wildcard': f'%.{domain}'})
+                results = cursor.fetchall()
+                certificates = [dict(row) for row in results]
+                
+            self.logger.logger.info(f"crt.sh DB query for '{domain}' returned {len(certificates)} certificates.")
+
+        except psycopg2.Error as e:
+            self.logger.logger.error(f"PostgreSQL query failed for {domain}: {e}")
+            # Raise a RequestException to be compatible with the existing retry logic in the core application
+            raise requests.exceptions.RequestException(f"PostgreSQL query failed: {e}") from e
+        finally:
+            if conn:
+                conn.close()
+        
+        return certificates
+
+    def query_domain(self, domain: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]:
+        """
+        Query crt.sh for certificates containing the domain with caching support.
+        Properly raises exceptions for network errors to allow core logic retries.
+        """
+        if not _is_valid_domain(domain):
+            return []
+        
+        if self._stop_event and self._stop_event.is_set():
+            return []
+        
+        cache_file = self._get_cache_file_path(domain)
+        cache_status = self._get_cache_status(cache_file)
+        
+        certificates = []
+        
+        try:
+            if cache_status == "fresh":
+                certificates = self._load_cached_certificates(cache_file)
+                self.logger.logger.info(f"Using cached data for {domain} ({len(certificates)} certificates)")
+                
+            elif cache_status == "not_found":
+                # Fresh query from DB, create new cache
+                certificates = self._query_crtsh(domain)
+                if certificates:
+                    self._create_cache_file(cache_file, domain, self._serialize_certs_for_cache(certificates))
+                else:
+                    self.logger.logger.info(f"No certificates found for {domain}, not caching")
+                    
+            elif cache_status == "stale":
+                try:
+                    new_certificates = self._query_crtsh(domain)
+                    if new_certificates:
+                        certificates = self._append_to_cache(cache_file, self._serialize_certs_for_cache(new_certificates))
+                    else:
+                        certificates = self._load_cached_certificates(cache_file)
+                except requests.exceptions.RequestException:
+                    certificates = self._load_cached_certificates(cache_file)
+                    if certificates:
+                        self.logger.logger.warning(f"DB query failed for {domain}, using stale cache data.")
+                    else:
+                        raise
+            
+        except requests.exceptions.RequestException as e:
+            # Re-raise so core logic can retry
+            self.logger.logger.error(f"DB query failed for {domain}: {e}")
+            raise e
+        except json.JSONDecodeError as e:
+            # JSON parsing errors from cache should also be handled
+            self.logger.logger.error(f"Failed to parse JSON from cache for {domain}: {e}")
+            raise e
+        
+        if self._stop_event and self._stop_event.is_set():
+            return []
+        
+        if not certificates:
+            return []
+        
+        return self._process_certificates_to_relationships(domain, certificates)
+        
+    def _serialize_certs_for_cache(self, certificates: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        """
+        Serialize certificate data for JSON caching, converting datetime objects to ISO strings.
+        """
+        serialized_certs = []
+        for cert in certificates:
+            serialized_cert = cert.copy()
+            for key in ['not_before', 'not_after', 'entry_timestamp']:
+                if isinstance(serialized_cert.get(key), datetime):
+                    # Ensure datetime is timezone-aware before converting
+                    dt_obj = serialized_cert[key]
+                    if dt_obj.tzinfo is None:
+                        dt_obj = dt_obj.replace(tzinfo=timezone.utc)
+                    serialized_cert[key] = dt_obj.isoformat()
+            serialized_certs.append(serialized_cert)
+        return serialized_certs
+
+    # --- All methods below are copied directly from the original CrtShProvider ---
+    # They are compatible because _query_crtsh returns data in the same format
+    # as the original _query_crtsh_api method. A small adjustment is made to
+    # _parse_certificate_date to handle datetime objects directly from the DB.
+
+    def _get_cache_file_path(self, domain: str) -> Path:
+        """Generate cache file path for a domain."""
+        safe_domain = domain.replace('.', '_').replace('/', '_').replace('\\', '_')
+        return self.cache_dir / f"{safe_domain}.json"
+    
+    def _get_cache_status(self, cache_file_path: Path) -> str:
+        """Check cache status for a domain."""
+        if not cache_file_path.exists():
+            return "not_found"
+        
+        try:
+            with open(cache_file_path, 'r') as f:
+                cache_data = json.load(f)
+            
+            last_query_str = cache_data.get("last_upstream_query")
+            if not last_query_str:
+                return "stale"
+            
+            last_query = datetime.fromisoformat(last_query_str.replace('Z', '+00:00'))
+            hours_since_query = (datetime.now(timezone.utc) - last_query).total_seconds() / 3600
+            
+            cache_timeout = self.config.cache_timeout_hours
+            if hours_since_query < cache_timeout:
+                return "fresh"
+            else:
+                return "stale"
+                
+        except (json.JSONDecodeError, ValueError, KeyError) as e:
+            self.logger.logger.warning(f"Invalid cache file format for {cache_file_path}: {e}")
+            return "stale"
+    
+    def _load_cached_certificates(self, cache_file_path: Path) -> List[Dict[str, Any]]:
+        """Load certificates from cache file."""
+        try:
+            with open(cache_file_path, 'r') as f:
+                cache_data = json.load(f)
+            return cache_data.get('certificates', [])
+        except (json.JSONDecodeError, FileNotFoundError, KeyError) as e:
+            self.logger.logger.error(f"Failed to load cached certificates from {cache_file_path}: {e}")
+            return []
+    
+    def _create_cache_file(self, cache_file_path: Path, domain: str, certificates: List[Dict[str, Any]]) -> None:
+        """Create new cache file with certificates."""
+        try:
+            cache_data = {
+                "domain": domain,
+                "first_cached": datetime.now(timezone.utc).isoformat(),
+                "last_upstream_query": datetime.now(timezone.utc).isoformat(),
+                "upstream_query_count": 1,
+                "certificates": certificates
+            }
+            cache_file_path.parent.mkdir(parents=True, exist_ok=True)
+            with open(cache_file_path, 'w') as f:
+                json.dump(cache_data, f, separators=(',', ':'))
+            self.logger.logger.info(f"Created cache file for {domain} with {len(certificates)} certificates")
+        except Exception as e:
+            self.logger.logger.warning(f"Failed to create cache file for {domain}: {e}")
+    
+    def _append_to_cache(self, cache_file_path: Path, new_certificates: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        """Append new certificates to existing cache and return all certificates."""
+        try:
+            with open(cache_file_path, 'r') as f:
+                cache_data = json.load(f)
+            
+            existing_ids = {cert.get('id') for cert in cache_data.get('certificates', [])}
+            added_count = 0
+            for cert in new_certificates:
+                cert_id = cert.get('id')
+                if cert_id and cert_id not in existing_ids:
+                    cache_data['certificates'].append(cert)
+                    existing_ids.add(cert_id)
+                    added_count += 1
+            
+            cache_data['last_upstream_query'] = datetime.now(timezone.utc).isoformat()
+            cache_data['upstream_query_count'] = cache_data.get('upstream_query_count', 0) + 1
+            
+            with open(cache_file_path, 'w') as f:
+                json.dump(cache_data, f, separators=(',', ':'))
+            
+            total_certs = len(cache_data['certificates'])
+            self.logger.logger.info(f"Appended {added_count} new certificates to cache. Total: {total_certs}")
+            return cache_data['certificates']
+        except Exception as e:
+            self.logger.logger.warning(f"Failed to append to cache: {e}")
+            return new_certificates
+    
+    def _parse_issuer_organization(self, issuer_dn: str) -> str:
+        """Parse the issuer Distinguished Name to extract just the organization name."""
+        if not issuer_dn: return issuer_dn
+        try:
+            components = [comp.strip() for comp in issuer_dn.split(',')]
+            for component in components:
+                if component.startswith('O='):
+                    org_name = component[2:].strip()
+                    if org_name.startswith('"') and org_name.endswith('"'):
+                        org_name = org_name[1:-1]
+                    return org_name
+            return issuer_dn
+        except Exception as e:
+            self.logger.logger.debug(f"Failed to parse issuer DN '{issuer_dn}': {e}")
+            return issuer_dn
+    
+    def _parse_certificate_date(self, date_input: Any) -> datetime:
+        """
+        Parse certificate date from various formats (string from cache, datetime from DB).
+        """
+        if isinstance(date_input, datetime):
+            # If it's already a datetime object from the DB, just ensure it's UTC
+            if date_input.tzinfo is None:
+                return date_input.replace(tzinfo=timezone.utc)
+            return date_input
+
+        date_string = str(date_input)
+        if not date_string:
+            raise ValueError("Empty date string")
+
+        try:
+            if 'Z' in date_string:
+                return datetime.fromisoformat(date_string.replace('Z', '+00:00'))
+            # Handle standard ISO format with or without timezone
+            dt = datetime.fromisoformat(date_string)
+            if dt.tzinfo is None:
+                return dt.replace(tzinfo=timezone.utc)
+            return dt
+        except ValueError as e:
+            try:
+                # Fallback for other formats
+                return datetime.strptime(date_string[:19], "%Y-%m-%dT%H:%M:%S").replace(tzinfo=timezone.utc)
+            except Exception:
+                raise ValueError(f"Unable to parse date: {date_string}") from e
+
+    def _is_cert_valid(self, cert_data: Dict[str, Any]) -> bool:
+        """Check if a certificate is currently valid based on its expiry date."""
+        try:
+            not_after_str = cert_data.get('not_after')
+            if not not_after_str: return False
+
+            not_after_date = self._parse_certificate_date(not_after_str)
+            not_before_str = cert_data.get('not_before')
+            now = datetime.now(timezone.utc)
+            is_not_expired = not_after_date > now
+
+            if not_before_str:
+                not_before_date = self._parse_certificate_date(not_before_str)
+                is_not_before_valid = not_before_date <= now
+                return is_not_expired and is_not_before_valid
+            return is_not_expired
+        except Exception as e:
+            self.logger.logger.debug(f"Certificate validity check failed: {e}")
+            return False
+
+    def _extract_certificate_metadata(self, cert_data: Dict[str, Any]) -> Dict[str, Any]:
+        # This method works as-is.
+        raw_issuer_name = cert_data.get('issuer_name', '')
+        parsed_issuer_name = self._parse_issuer_organization(raw_issuer_name)
+        metadata = {
+            'certificate_id': cert_data.get('id'),
+            'serial_number': cert_data.get('serial_number'),
+            'issuer_name': parsed_issuer_name,
+            'issuer_ca_id': cert_data.get('issuer_ca_id'),
+            'common_name': cert_data.get('common_name'),
+            'not_before': cert_data.get('not_before'),
+            'not_after': cert_data.get('not_after'),
+            'entry_timestamp': cert_data.get('entry_timestamp'),
+            'source': 'crt.sh (DB)'
+        }
+        try:
+            if metadata['not_before'] and metadata['not_after']:
+                not_before = self._parse_certificate_date(metadata['not_before'])
+                not_after = self._parse_certificate_date(metadata['not_after'])
+                metadata['validity_period_days'] = (not_after - not_before).days
+                metadata['is_currently_valid'] = self._is_cert_valid(cert_data)
+                metadata['expires_soon'] = (not_after - datetime.now(timezone.utc)).days <= 30
+                metadata['not_before'] = not_before.strftime('%Y-%m-%d %H:%M:%S UTC')
+                metadata['not_after'] = not_after.strftime('%Y-%m-%d %H:%M:%S UTC')
+        except Exception as e:
+            self.logger.logger.debug(f"Error computing certificate metadata: {e}")
+            metadata['is_currently_valid'] = False
+            metadata['expires_soon'] = False
+        return metadata
+
+    def _process_certificates_to_relationships(self, domain: str, certificates: List[Dict[str, Any]]) -> List[Tuple[str, str, str, float, Dict[str, Any]]]:
+        # This method works as-is.
+        relationships = []
+        if self._stop_event and self._stop_event.is_set(): return []
+        domain_certificates = {}
+        all_discovered_domains = set()
+        for i, cert_data in enumerate(certificates):
+            if i % 5 == 0 and self._stop_event and self._stop_event.is_set(): break
+            cert_metadata = self._extract_certificate_metadata(cert_data)
+            cert_domains = self._extract_domains_from_certificate(cert_data)
+            all_discovered_domains.update(cert_domains)
+            for cert_domain in cert_domains:
+                if not _is_valid_domain(cert_domain): continue
+                if cert_domain not in domain_certificates:
+                    domain_certificates[cert_domain] = []
+                domain_certificates[cert_domain].append(cert_metadata)
+        if self._stop_event and self._stop_event.is_set(): return []
+        for i, discovered_domain in enumerate(all_discovered_domains):
+            if discovered_domain == domain: continue
+            if i % 10 == 0 and self._stop_event and self._stop_event.is_set(): break
+            if not _is_valid_domain(discovered_domain): continue
+            query_domain_certs = domain_certificates.get(domain, [])
+            discovered_domain_certs = domain_certificates.get(discovered_domain, [])
+            shared_certificates = self._find_shared_certificates(query_domain_certs, discovered_domain_certs)
+            confidence = self._calculate_domain_relationship_confidence(
+                domain, discovered_domain, shared_certificates, all_discovered_domains
+            )
+            relationship_raw_data = {
+                'relationship_type': 'certificate_discovery',
+                'shared_certificates': shared_certificates,
+                'total_shared_certs': len(shared_certificates),
+                'discovery_context': self._determine_relationship_context(discovered_domain, domain),
+                'domain_certificates': {
+                    domain: self._summarize_certificates(query_domain_certs),
+                    discovered_domain: self._summarize_certificates(discovered_domain_certs)
+                }
+            }
+            relationships.append((
+                domain, discovered_domain, 'san_certificate', confidence, relationship_raw_data
+            ))
+            self.log_relationship_discovery(
+                source_node=domain, target_node=discovered_domain, relationship_type='san_certificate',
+                confidence_score=confidence, raw_data=relationship_raw_data,
+                discovery_method="certificate_transparency_analysis"
+            )
+        return relationships
+    
+    # --- All remaining helper methods are identical to the original and fully compatible ---
+    # They are included here for completeness.
+
+    def _find_shared_certificates(self, certs1: List[Dict[str, Any]], certs2: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        cert1_ids = {cert.get('certificate_id') for cert in certs1 if cert.get('certificate_id')}
+        return [cert for cert in certs2 if cert.get('certificate_id') in cert1_ids]
+
+    def _summarize_certificates(self, certificates: List[Dict[str, Any]]) -> Dict[str, Any]:
+        if not certificates: return {'total_certificates': 0, 'valid_certificates': 0, 'expired_certificates': 0, 'expires_soon_count': 0, 'unique_issuers': [], 'latest_certificate': None, 'has_valid_cert': False}
+        valid_count = sum(1 for cert in certificates if cert.get('is_currently_valid'))
+        expires_soon_count = sum(1 for cert in certificates if cert.get('expires_soon'))
+        unique_issuers = list(set(cert.get('issuer_name') for cert in certificates if cert.get('issuer_name')))
+        latest_cert, latest_date = None, None
+        for cert in certificates:
+            try:
+                if cert.get('not_before'):
+                    cert_date = self._parse_certificate_date(cert['not_before'])
+                    if latest_date is None or cert_date > latest_date:
+                        latest_date, latest_cert = cert_date, cert
+            except Exception: continue
+        return {'total_certificates': len(certificates), 'valid_certificates': valid_count, 'expired_certificates': len(certificates) - valid_count, 'expires_soon_count': expires_soon_count, 'unique_issuers': unique_issuers, 'latest_certificate': latest_cert, 'has_valid_cert': valid_count > 0, 'certificate_details': certificates}
+
+    def _calculate_domain_relationship_confidence(self, domain1: str, domain2: str, shared_certificates: List[Dict[str, Any]], all_discovered_domains: Set[str]) -> float:
+        base_confidence, context_bonus, shared_bonus, validity_bonus, issuer_bonus = 0.9, 0.0, 0.0, 0.0, 0.0
+        relationship_context = self._determine_relationship_context(domain2, domain1)
+        if relationship_context == 'subdomain': context_bonus = 0.1
+        elif relationship_context == 'parent_domain': context_bonus = 0.05
+        if shared_certificates:
+            if len(shared_certificates) >= 3: shared_bonus = 0.1
+            elif len(shared_certificates) >= 2: shared_bonus = 0.05
+            else: shared_bonus = 0.02
+            if any(cert.get('is_currently_valid') for cert in shared_certificates): validity_bonus = 0.05
+            for cert in shared_certificates:
+                if any(ca in cert.get('issuer_name', '').lower() for ca in ['let\'s encrypt', 'digicert', 'sectigo', 'globalsign']):
+                    issuer_bonus = max(issuer_bonus, 0.03)
+                    break
+        return max(0.1, min(1.0, base_confidence + context_bonus + shared_bonus + validity_bonus + issuer_bonus))
+
+    def _determine_relationship_context(self, cert_domain: str, query_domain: str) -> str:
+        if cert_domain == query_domain: return 'exact_match'
+        if cert_domain.endswith(f'.{query_domain}'): return 'subdomain'
+        if query_domain.endswith(f'.{cert_domain}'): return 'parent_domain'
+        return 'related_domain'
+    
+    def query_ip(self, ip: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]:
+        return []
+    
+    def _extract_domains_from_certificate(self, cert_data: Dict[str, Any]) -> Set[str]:
+        domains = set()
+        if cn := cert_data.get('common_name'):
+            if cleaned := self._clean_domain_name(cn):
+                domains.update(cleaned)
+        if nv := cert_data.get('name_value'):
+            for line in nv.split('\n'):
+                if cleaned := self._clean_domain_name(line.strip()):
+                    domains.update(cleaned)
+        return domains
+    
+    def _clean_domain_name(self, domain_name: str) -> List[str]:
+        if not domain_name: return []
+        domain = domain_name.strip().lower().split('://', 1)[-1].split('/', 1)[0]
+        if ':' in domain and not domain.count(':') > 1: domain = domain.split(':', 1)[0]
+        cleaned_domains = [domain, domain[2:]] if domain.startswith('*.') else [domain]
+        final_domains = []
+        for d in cleaned_domains:
+            d = re.sub(r'[^\w\-\.]', '', d)
+            if d and not d.startswith(('.', '-')) and not d.endswith(('.', '-')):
+                final_domains.append(d)
+        return [d for d in final_domains if _is_valid_domain(d)]

providers/dns_provider.py

@@ -0,0 +1,198 @@
+# dnsrecon/providers/dns_provider.py
+
+from dns import resolver, reversename
+from typing import List, Dict, Any, Tuple
+from .base_provider import BaseProvider
+from utils.helpers import _is_valid_ip, _is_valid_domain
+
+
+class DNSProvider(BaseProvider):
+    """
+    Provider for standard DNS resolution and reverse DNS lookups.
+    Now uses session-specific configuration.
+    """
+
+    def __init__(self, name=None, session_config=None):
+        """Initialize DNS provider with session-specific configuration."""
+        super().__init__(
+            name="dns",
+            rate_limit=100,
+            timeout=10,
+            session_config=session_config
+        )
+
+        # Configure DNS resolver
+        self.resolver = resolver.Resolver()
+        self.resolver.timeout = 5
+        self.resolver.lifetime = 10
+        #self.resolver.nameservers = ['127.0.0.1']
+
+    def get_name(self) -> str:
+        """Return the provider name."""
+        return "dns"
+
+    def get_display_name(self) -> str:
+        """Return the provider display name for the UI."""
+        return "DNS"
+
+    def requires_api_key(self) -> bool:
+        """Return True if the provider requires an API key."""
+        return False
+
+    def get_eligibility(self) -> Dict[str, bool]:
+        """Return a dictionary indicating if the provider can query domains and/or IPs."""
+        return {'domains': True, 'ips': True}
+
+    def is_available(self) -> bool:
+        """DNS is always available - no API key required."""
+        return True
+
+    def query_domain(self, domain: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]:
+        """
+        Query DNS records for the domain to discover relationships.
+        ...
+        """
+        if not _is_valid_domain(domain):
+            return []
+
+        relationships = []
+
+        # Query all record types
+        for record_type in ['A', 'AAAA', 'CNAME', 'MX', 'NS', 'SOA', 'TXT', 'SRV', 'CAA']:
+            try:
+                relationships.extend(self._query_record(domain, record_type))
+            except resolver.NoAnswer:
+                # This is not an error, just a confirmation that the record doesn't exist.
+                self.logger.logger.debug(f"No {record_type} record found for {domain}")
+            except Exception as e:
+                self.failed_requests += 1
+                self.logger.logger.debug(f"{record_type} record query failed for {domain}: {e}")
+                # Optionally, you might want to re-raise other, more serious exceptions.
+
+        return relationships
+
+    def query_ip(self, ip: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]:
+        """
+        Query reverse DNS for the IP address.
+
+        Args:
+            ip: IP address to investigate
+
+        Returns:
+            List of relationships discovered from reverse DNS
+        """
+        if not _is_valid_ip(ip):
+            return []
+
+        relationships = []
+
+        try:
+            # Perform reverse DNS lookup
+            self.total_requests += 1
+            reverse_name = reversename.from_address(ip)
+            response = self.resolver.resolve(reverse_name, 'PTR')
+            self.successful_requests += 1
+
+            for ptr_record in response:
+                hostname = str(ptr_record).rstrip('.')
+
+                if _is_valid_domain(hostname):
+                    raw_data = {
+                        'query_type': 'PTR',
+                        'ip_address': ip,
+                        'hostname': hostname,
+                        'ttl': response.ttl
+                    }
+
+                    relationships.append((
+                        ip,
+                        hostname,
+                        'ptr_record',
+                        0.8,
+                        raw_data
+                    ))
+
+                    self.log_relationship_discovery(
+                        source_node=ip,
+                        target_node=hostname,
+                        relationship_type='ptr_record',
+                        confidence_score=0.8,
+                        raw_data=raw_data,
+                        discovery_method="reverse_dns_lookup"
+                    )
+
+        except resolver.NXDOMAIN:
+            self.failed_requests += 1
+            self.logger.logger.debug(f"Reverse DNS lookup failed for {ip}: NXDOMAIN")
+        except Exception as e:
+            self.failed_requests += 1
+            self.logger.logger.debug(f"Reverse DNS lookup failed for {ip}: {e}")
+            # Re-raise the exception so the scanner can handle the failure
+            raise e
+
+        return relationships
+
+    def _query_record(self, domain: str, record_type: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]:
+        """
+        Query a specific type of DNS record for the domain.
+        """
+        relationships = []
+        try:
+            self.total_requests += 1
+            response = self.resolver.resolve(domain, record_type)
+            self.successful_requests += 1
+
+            for record in response:
+                target = ""
+                if record_type in ['A', 'AAAA']:
+                    target = str(record)
+                elif record_type in ['CNAME', 'NS', 'PTR']:
+                    target = str(record.target).rstrip('.')
+                elif record_type == 'MX':
+                    target = str(record.exchange).rstrip('.')
+                elif record_type == 'SOA':
+                    target = str(record.mname).rstrip('.')
+                elif record_type in ['TXT']:
+                    # TXT records are treated as metadata, not relationships.
+                    continue
+                elif record_type == 'SRV':
+                    target = str(record.target).rstrip('.')
+                elif record_type == 'CAA':
+                    target = f"{record.flags} {record.tag.decode('utf-8')} \"{record.value.decode('utf-8')}\""
+                else:
+                    target = str(record)
+
+                if target:
+                    raw_data = {
+                        'query_type': record_type,
+                        'domain': domain,
+                        'value': target,
+                        'ttl': response.ttl
+                    }
+                    relationship_type = f"{record_type.lower()}_record"
+                    confidence = 0.8  # Default confidence for DNS records
+
+                    relationships.append((
+                        domain,
+                        target,
+                        relationship_type,
+                        confidence,
+                        raw_data
+                    ))
+
+                    self.log_relationship_discovery(
+                        source_node=domain,
+                        target_node=target,
+                        relationship_type=relationship_type,
+                        confidence_score=confidence,
+                        raw_data=raw_data,
+                        discovery_method=f"dns_{record_type.lower()}_record"
+                    )
+
+        except Exception as e:
+            self.failed_requests += 1
+            self.logger.logger.debug(f"{record_type} record query failed for {domain}: {e}")
+            # Re-raise the exception so the scanner can handle it
+            raise e
+
+        return relationships

providers/shodan_provider.py

@@ -0,0 +1,307 @@
+# dnsrecon/providers/shodan_provider.py
+
+import json
+from typing import List, Dict, Any, Tuple
+from .base_provider import BaseProvider
+from utils.helpers import _is_valid_ip, _is_valid_domain
+
+
+class ShodanProvider(BaseProvider):
+    """
+    Provider for querying Shodan API for IP address and hostname information.
+    Now uses session-specific API keys.
+    """
+
+    def __init__(self, name=None, session_config=None):
+        """Initialize Shodan provider with session-specific configuration."""
+        super().__init__(
+            name="shodan",
+            rate_limit=60,
+            timeout=30,
+            session_config=session_config
+        )
+        self.base_url = "https://api.shodan.io"
+        self.api_key = self.config.get_api_key('shodan')
+
+    def is_available(self) -> bool:
+        """Check if Shodan provider is available (has valid API key in this session)."""
+        return self.api_key is not None and len(self.api_key.strip()) > 0
+
+    def get_name(self) -> str:
+        """Return the provider name."""
+        return "shodan"
+
+    def get_display_name(self) -> str:
+        """Return the provider display name for the UI."""
+        return "shodan"
+
+    def requires_api_key(self) -> bool:
+        """Return True if the provider requires an API key."""
+        return True
+
+    def get_eligibility(self) -> Dict[str, bool]:
+        """Return a dictionary indicating if the provider can query domains and/or IPs."""
+        return {'domains': True, 'ips': True}
+
+    def query_domain(self, domain: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]:
+        """
+        Query Shodan for information about a domain.
+        Uses Shodan's hostname search to find associated IPs.
+
+        Args:
+            domain: Domain to investigate
+
+        Returns:
+            List of relationships discovered from Shodan data
+        """
+        if not _is_valid_domain(domain) or not self.is_available():
+            return []
+
+        relationships = []
+
+        try:
+            # Search for hostname in Shodan
+            search_query = f"hostname:{domain}"
+            url = f"{self.base_url}/shodan/host/search"
+            params = {
+                'key': self.api_key,
+                'query': search_query,
+                'minify': True  # Get minimal data to reduce bandwidth
+            }
+
+            response = self.make_request(url, method="GET", params=params, target_indicator=domain)
+
+            if not response or response.status_code != 200:
+                return []
+
+            data = response.json()
+
+            if 'matches' not in data:
+                return []
+
+            # Process search results
+            for match in data['matches']:
+                ip_address = match.get('ip_str')
+                hostnames = match.get('hostnames', [])
+
+                if ip_address and domain in hostnames:
+                    raw_data = {
+                        'ip_address': ip_address,
+                        'hostnames': hostnames,
+                        'country': match.get('location', {}).get('country_name', ''),
+                        'city': match.get('location', {}).get('city', ''),
+                        'isp': match.get('isp', ''),
+                        'org': match.get('org', ''),
+                        'ports': match.get('ports', []),
+                        'last_update': match.get('last_update', '')
+                    }
+
+                    relationships.append((
+                        domain,
+                        ip_address,
+                        'a_record',  # Domain resolves to IP
+                        0.8,
+                        raw_data
+                    ))
+
+                    self.log_relationship_discovery(
+                        source_node=domain,
+                        target_node=ip_address,
+                        relationship_type='a_record',
+                        confidence_score=0.8,
+                        raw_data=raw_data,
+                        discovery_method="shodan_hostname_search"
+                    )
+
+                    # Also create relationships to other hostnames on the same IP
+                    for hostname in hostnames:
+                        if hostname != domain and _is_valid_domain(hostname):
+                            hostname_raw_data = {
+                                'shared_ip': ip_address,
+                                'all_hostnames': hostnames,
+                                'discovery_context': 'shared_hosting'
+                            }
+
+                            relationships.append((
+                                domain,
+                                hostname,
+                                'passive_dns',  # Shared hosting relationship
+                                0.6,  # Lower confidence for shared hosting
+                                hostname_raw_data
+                            ))
+
+                            self.log_relationship_discovery(
+                                source_node=domain,
+                                target_node=hostname,
+                                relationship_type='passive_dns',
+                                confidence_score=0.6,
+                                raw_data=hostname_raw_data,
+                                discovery_method="shodan_shared_hosting"
+                            )
+
+        except json.JSONDecodeError as e:
+            self.logger.logger.error(f"Failed to parse JSON response from Shodan: {e}")
+
+        return relationships
+
+    def query_ip(self, ip: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]:
+        """
+        Query Shodan for information about an IP address.
+
+        Args:
+            ip: IP address to investigate
+
+        Returns:
+            List of relationships discovered from Shodan IP data
+        """
+        if not _is_valid_ip(ip) or not self.is_available():
+            return []
+
+        relationships = []
+
+        try:
+            # Query Shodan host information
+            url = f"{self.base_url}/shodan/host/{ip}"
+            params = {'key': self.api_key}
+
+            response = self.make_request(url, method="GET", params=params, target_indicator=ip)
+
+            if not response or response.status_code != 200:
+                return []
+
+            data = response.json()
+
+            # Extract hostname relationships
+            hostnames = data.get('hostnames', [])
+            for hostname in hostnames:
+                if _is_valid_domain(hostname):
+                    raw_data = {
+                        'ip_address': ip,
+                        'hostname': hostname,
+                        'country': data.get('country_name', ''),
+                        'city': data.get('city', ''),
+                        'isp': data.get('isp', ''),
+                        'org': data.get('org', ''),
+                        'asn': data.get('asn', ''),
+                        'ports': data.get('ports', []),
+                        'last_update': data.get('last_update', ''),
+                        'os': data.get('os', '')
+                    }
+
+                    relationships.append((
+                        ip,
+                        hostname,
+                        'a_record',  # IP resolves to hostname
+                        0.8,
+                        raw_data
+                    ))
+
+                    self.log_relationship_discovery(
+                        source_node=ip,
+                        target_node=hostname,
+                        relationship_type='a_record',
+                        confidence_score=0.8,
+                        raw_data=raw_data,
+                        discovery_method="shodan_host_lookup"
+                    )
+
+            # Extract ASN relationship if available
+            asn = data.get('asn')
+            if asn:
+                # Ensure the ASN starts with "AS"
+                if isinstance(asn, str) and asn.startswith('AS'):
+                    asn_name = asn
+                    asn_number = asn[2:]
+                else:
+                    asn_name = f"AS{asn}"
+                    asn_number = str(asn)
+
+                asn_raw_data = {
+                    'ip_address': ip,
+                    'asn': asn_number,
+                    'isp': data.get('isp', ''),
+                    'org': data.get('org', '')
+                }
+
+                relationships.append((
+                    ip,
+                    asn_name,
+                    'asn_membership',
+                    0.7,
+                    asn_raw_data
+                ))
+
+                self.log_relationship_discovery(
+                    source_node=ip,
+                    target_node=asn_name,
+                    relationship_type='asn_membership',
+                    confidence_score=0.7,
+                    raw_data=asn_raw_data,
+                    discovery_method="shodan_asn_lookup"
+                )
+
+        except json.JSONDecodeError as e:
+            self.logger.logger.error(f"Failed to parse JSON response from Shodan: {e}")
+
+        return relationships
+
+    def search_by_organization(self, org_name: str) -> List[Dict[str, Any]]:
+        """
+        Search Shodan for hosts belonging to a specific organization.
+
+        Args:
+            org_name: Organization name to search for
+
+        Returns:
+            List of host information dictionaries
+        """
+        if not self.is_available():
+            return []
+
+        try:
+            search_query = f"org:\"{org_name}\""
+            url = f"{self.base_url}/shodan/host/search"
+            params = {
+                'key': self.api_key,
+                'query': search_query,
+                'minify': True
+            }
+
+            response = self.make_request(url, method="GET", params=params, target_indicator=org_name)
+
+            if response and response.status_code == 200:
+                data = response.json()
+                return data.get('matches', [])
+
+        except Exception as e:
+            self.logger.logger.error(f"Error searching Shodan by organization {org_name}: {e}")
+
+        return []
+
+    def get_host_services(self, ip: str) -> List[Dict[str, Any]]:
+        """
+        Get service information for a specific IP address.
+
+        Args:
+            ip: IP address to query
+
+        Returns:
+            List of service information dictionaries
+        """
+        if not _is_valid_ip(ip) or not self.is_available():
+            return []
+
+        try:
+            url = f"{self.base_url}/shodan/host/{ip}"
+            params = {'key': self.api_key}
+
+            response = self.make_request(url, method="GET", params=params, target_indicator=ip)
+
+            if response and response.status_code == 200:
+                data = response.json()
+                return data.get('data', [])  # Service banners
+
+        except Exception as e:
+            self.logger.logger.error(f"Error getting Shodan services for IP {ip}: {e}")
+
+        return []

requirements.txt

@@ -1,4 +1,11 @@
+Flask>=2.3.3
+networkx>=3.1
 requests>=2.31.0
-flask>=2.3.0
-dnspython>=2.4.0
-click>=8.1.0
+python-dateutil>=2.8.2
+Werkzeug>=2.3.7
+urllib3>=2.0.0
+dnspython>=2.4.2
+gunicorn
+redis
+python-dotenv
+psycopg2-binary

src/__init__.py

@@ -1,20 +0,0 @@
-# File: src/__init__.py
-"""DNS Reconnaissance Tool Package."""
-
-__version__ = "1.0.0"
-__author__ = "DNS Recon Tool"
-__email__ = ""
-__description__ = "A comprehensive DNS reconnaissance tool for investigators"
-
-from .main import main
-from .config import Config
-from .reconnaissance import ReconnaissanceEngine
-from .data_structures import ReconData
-
-__all__ = [
-    'main',
-    'Config', 
-    'ReconnaissanceEngine',
-    'ReconData'
-]
-

src/certificate_checker.py

@@ -1,356 +0,0 @@
-# File: src/certificate_checker.py
-"""Certificate transparency log checker using crt.sh with forensic operation tracking."""
-
-import requests
-import json
-import time
-import logging
-import socket
-import uuid
-from datetime import datetime
-from typing import List, Optional, Set
-from .data_structures import Certificate
-from .config import Config
-
-# Module logger
-logger = logging.getLogger(__name__)
-
-class CertificateChecker:
-    """Check certificates using crt.sh with simple query caching and forensic tracking."""
-    
-    CRT_SH_URL = "https://crt.sh/"
-    
-    def __init__(self, config: Config):
-        self.config = config
-        self.last_request = 0
-        self.query_count = 0
-        self.connection_failures = 0
-        self.max_connection_failures = 3
-        
-        # Simple HTTP request cache to avoid duplicate queries
-        self._http_cache = {}  # query_string -> List[Certificate]
-        
-        logger.info("🔐 Certificate checker initialized with HTTP request caching")
-        self._test_connectivity()
-    
-    def _test_connectivity(self):
-        """Test if we can reach crt.sh."""
-        try:
-            logger.info("Testing connectivity to crt.sh...")
-            
-            try:
-                socket.gethostbyname('crt.sh')
-                logger.debug("DNS resolution for crt.sh successful")
-            except socket.gaierror as e:
-                logger.warning(f"DNS resolution failed for crt.sh: {e}")
-                return False
-            
-            response = requests.get(
-                self.CRT_SH_URL,
-                params={'q': 'example.com', 'output': 'json'},
-                timeout=10,
-                headers={'User-Agent': 'DNS-Recon-Tool/1.0'}
-            )
-            
-            if response.status_code in [200, 404]:
-                logger.info("crt.sh connectivity test successful")
-                return True
-            else:
-                logger.warning(f"crt.sh returned status {response.status_code}")
-                return False
-                
-        except requests.exceptions.ConnectionError as e:
-            logger.warning(f"Cannot reach crt.sh: {e}")
-            return False
-        except requests.exceptions.Timeout:
-            logger.warning("crt.sh connectivity test timed out")
-            return False
-        except Exception as e:
-            logger.warning(f"Unexpected error testing crt.sh connectivity: {e}")
-            return False
-    
-    def _rate_limit(self):
-        """Apply rate limiting for crt.sh."""
-        now = time.time()
-        time_since_last = now - self.last_request
-        min_interval = 1.0 / self.config.CRT_SH_RATE_LIMIT
-        
-        if time_since_last < min_interval:
-            sleep_time = min_interval - time_since_last
-            logger.debug(f"crt.sh rate limiting: sleeping for {sleep_time:.2f}s")
-            time.sleep(sleep_time)
-        
-        self.last_request = time.time()
-        self.query_count += 1
-    
-    def get_certificates(self, domain: str, operation_id: Optional[str] = None) -> List[Certificate]:
-        """Get certificates for a domain with forensic tracking."""
-        # Generate operation ID if not provided
-        if operation_id is None:
-            operation_id = str(uuid.uuid4())
-            
-        logger.debug(f"🔐 Getting certificates for domain: {domain} (operation: {operation_id})")
-        
-        if self.connection_failures >= self.max_connection_failures:
-            logger.warning(f"Skipping certificate lookup for {domain} due to repeated connection failures")
-            return []
-        
-        certificates = []
-        
-        # Query for the domain itself
-        domain_certs = self._query_crt_sh(domain, operation_id)
-        certificates.extend(domain_certs)
-        
-        # Query for wildcard certificates  
-        wildcard_certs = self._query_crt_sh(f"%.{domain}", operation_id)
-        certificates.extend(wildcard_certs)
-        
-        # Remove duplicates based on certificate ID
-        unique_certs = {cert.id: cert for cert in certificates}
-        final_certs = list(unique_certs.values())
-        
-        if final_certs:
-            logger.info(f"✅ Found {len(final_certs)} unique certificates for {domain}")
-        else:
-            logger.debug(f"ℹ️ No certificates found for {domain}")
-        
-        return final_certs
-    
-    def _query_crt_sh(self, query: str, operation_id: str) -> List[Certificate]:
-        """Query crt.sh API with HTTP caching and forensic tracking."""
-        
-        # Check HTTP cache first
-        cache_key = f"{query}:{operation_id}"  # Include operation_id in cache key
-        if query in self._http_cache:
-            logger.debug(f"Using cached HTTP result for crt.sh query: {query}")
-            # Need to create new Certificate objects with current operation_id
-            cached_certs = self._http_cache[query]
-            return [
-                Certificate(
-                    id=cert.id,
-                    issuer=cert.issuer,
-                    subject=cert.subject,
-                    not_before=cert.not_before,
-                    not_after=cert.not_after,
-                    is_wildcard=cert.is_wildcard,
-                    operation_id=operation_id  # Use current operation_id
-                ) for cert in cached_certs
-            ]
-        
-        # Not cached, make the HTTP request
-        certificates = self._make_http_request(query, operation_id)
-        
-        # Cache the HTTP result (without operation_id)
-        if certificates:
-            self._http_cache[query] = [
-                Certificate(
-                    id=cert.id,
-                    issuer=cert.issuer,
-                    subject=cert.subject,
-                    not_before=cert.not_before,
-                    not_after=cert.not_after,
-                    is_wildcard=cert.is_wildcard,
-                    operation_id=""  # Cache without operation_id
-                ) for cert in certificates
-            ]
-        
-        return certificates
-    
-    def _make_http_request(self, query: str, operation_id: str) -> List[Certificate]:
-        """Make actual HTTP request to crt.sh API with retry logic and forensic tracking."""
-        certificates = []
-        self._rate_limit()
-        
-        logger.debug(f"🌐 Making HTTP request to crt.sh for: {query} (operation: {operation_id})")
-        
-        max_retries = 2
-        backoff_delays = [1, 3]
-        
-        for attempt in range(max_retries):
-            try:
-                params = {
-                    'q': query,
-                    'output': 'json'
-                }
-                
-                response = requests.get(
-                    self.CRT_SH_URL,
-                    params=params,
-                    timeout=self.config.HTTP_TIMEOUT,
-                    headers={'User-Agent': 'DNS-Recon-Tool/1.0'}
-                )
-                
-                logger.debug(f"📡 crt.sh API response for {query}: {response.status_code}")
-                
-                if response.status_code == 200:
-                    try:
-                        data = response.json()
-                        logger.debug(f"crt.sh returned {len(data)} certificate entries for {query}")
-                        
-                        for cert_data in data:
-                            try:
-                                not_before = self._parse_date(cert_data.get('not_before'))
-                                not_after = self._parse_date(cert_data.get('not_after'))
-                                
-                                if not_before and not_after:
-                                    # Create Certificate with forensic metadata
-                                    certificate = Certificate(
-                                        id=cert_data.get('id'),
-                                        issuer=cert_data.get('issuer_name', ''),
-                                        subject=cert_data.get('name_value', ''),
-                                        not_before=not_before,
-                                        not_after=not_after,
-                                        is_wildcard='*.' in cert_data.get('name_value', ''),
-                                        operation_id=operation_id  # Forensic tracking
-                                    )
-                                    certificates.append(certificate)
-                                    logger.debug(f"📋 Parsed certificate ID {certificate.id} for {query}")
-                                else:
-                                    logger.debug(f"Skipped certificate with invalid dates: {cert_data.get('id')}")
-                                    
-                            except (ValueError, TypeError, KeyError) as e:
-                                logger.debug(f"Error parsing certificate data: {e}")
-                                continue
-                        
-                        self.connection_failures = 0
-                        logger.info(f"✅ Successfully processed {len(certificates)} certificates from crt.sh for {query}")
-                        return certificates
-                    
-                    except json.JSONDecodeError as e:
-                        logger.warning(f"Invalid JSON response from crt.sh for {query}: {e}")
-                        if attempt < max_retries - 1:
-                            time.sleep(backoff_delays[attempt])
-                            continue
-                        return certificates
-                
-                elif response.status_code == 404:
-                    logger.debug(f"ℹ️ No certificates found for {query} (404)")
-                    self.connection_failures = 0
-                    return certificates
-                
-                elif response.status_code == 429:
-                    logger.warning(f"⚠️ crt.sh rate limit exceeded for {query}")
-                    if attempt < max_retries - 1:
-                        time.sleep(5)
-                        continue
-                    return certificates
-                    
-                else:
-                    logger.warning(f"⚠️ crt.sh HTTP error for {query}: {response.status_code}")
-                    if attempt < max_retries - 1:
-                        time.sleep(backoff_delays[attempt])
-                        continue
-                    return certificates
-                            
-            except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as e:
-                error_type = "Connection Error" if isinstance(e, requests.exceptions.ConnectionError) else "Timeout"
-                logger.warning(f"⚠️ crt.sh {error_type} for {query} (attempt {attempt+1}/{max_retries}): {e}")
-                
-                if isinstance(e, requests.exceptions.ConnectionError):
-                    self.connection_failures += 1
-                
-                if attempt < max_retries - 1:
-                    time.sleep(backoff_delays[attempt])
-                    continue
-                    
-            except requests.exceptions.RequestException as e:
-                logger.warning(f"🌐 crt.sh network error for {query} (attempt {attempt+1}/{max_retries}): {e}")
-                if attempt < max_retries - 1:
-                    time.sleep(backoff_delays[attempt])
-                    continue
-            except Exception as e:
-                logger.error(f"❌ Unexpected error querying crt.sh for {query}: {e}")
-                if attempt < max_retries - 1:
-                    time.sleep(backoff_delays[attempt])
-                    continue
-        
-        logger.warning(f"⚠️ All {max_retries} attempts failed for crt.sh query: {query}")
-        return certificates
-    
-    def _parse_date(self, date_str: str) -> Optional[datetime]:
-        """Parse date string with multiple format support."""
-        if not date_str:
-            return None
-        
-        date_formats = [
-            '%Y-%m-%dT%H:%M:%S',
-            '%Y-%m-%dT%H:%M:%SZ',
-            '%Y-%m-%d %H:%M:%S',
-            '%Y-%m-%dT%H:%M:%S.%f',
-            '%Y-%m-%dT%H:%M:%S.%fZ',
-        ]
-        
-        for fmt in date_formats:
-            try:
-                return datetime.strptime(date_str, fmt)
-            except ValueError:
-                continue
-        
-        try:
-            return datetime.fromisoformat(date_str.replace('Z', '+00:00'))
-        except ValueError:
-            pass
-        
-        logger.debug(f"Could not parse date: {date_str}")
-        return None
-    
-    def extract_subdomains_from_certificates(self, certificates: List[Certificate]) -> Set[str]:
-        """Extract subdomains from certificate subjects."""
-        subdomains = set()
-        
-        logger.debug(f"🌿 Extracting subdomains from {len(certificates)} certificates")
-        
-        for cert in certificates:
-            # Parse subject field for domain names
-            subject_lines = cert.subject.split('\n')
-            
-            for line in subject_lines:
-                line = line.strip()
-                
-                # Skip wildcard domains for recursion (they don't resolve directly)
-                if line.startswith('*.'):
-                    logger.debug(f"⭐ Skipping wildcard domain: {line}")
-                    continue
-                
-                if self._is_valid_domain(line):
-                    subdomains.add(line.lower())
-                    logger.debug(f"🌿 Found subdomain from certificate: {line}")
-        
-        if subdomains:
-            logger.info(f"🌿 Extracted {len(subdomains)} subdomains from certificates")
-        else:
-            logger.debug("ℹ️ No subdomains extracted from certificates")
-        
-        return subdomains
-    
-    def _is_valid_domain(self, domain: str) -> bool:
-        """Basic domain validation."""
-        if not domain or '.' not in domain:
-            return False
-        
-        domain = domain.lower().strip()
-        if domain.startswith('www.'):
-            domain = domain[4:]
-        
-        if len(domain) < 3 or len(domain) > 255:
-            return False
-        
-        # Must not be an IP address
-        try:
-            socket.inet_aton(domain)
-            return False
-        except socket.error:
-            pass
-        
-        # Check for reasonable domain structure
-        parts = domain.split('.')
-        if len(parts) < 2:
-            return False
-        
-        for part in parts:
-            if len(part) < 1 or len(part) > 63:
-                return False
-            if not part.replace('-', '').replace('_', '').isalnum():
-                return False
-        
-        return True

src/config.py

@@ -1,80 +0,0 @@
-# File: src/config.py
-"""Configuration settings for the reconnaissance tool."""
-
-import os
-import logging
-from dataclasses import dataclass
-from typing import List, Optional
-
-@dataclass
-class Config:
-    """Configuration class for the reconnaissance tool."""
-    
-    # DNS servers to query
-    DNS_SERVERS: List[str] = None
-    
-    # API keys
-    shodan_key: Optional[str] = None
-    virustotal_key: Optional[str] = None
-    
-    # Rate limiting (requests per second)
-    # DNS servers are generally quite robust, increased from 10 to 50/s
-    DNS_RATE_LIMIT: float = 50.0
-    CRT_SH_RATE_LIMIT: float = 2.0
-    SHODAN_RATE_LIMIT: float = 0.5  # Shodan is more restrictive
-    VIRUSTOTAL_RATE_LIMIT: float = 0.25  # VirusTotal is very restrictive
-    
-    # Recursive depth
-    max_depth: int = 2
-    
-    # Timeouts
-    DNS_TIMEOUT: int = 5
-    HTTP_TIMEOUT: int = 20
-    
-    # Logging level
-    log_level: str = "INFO"
-    
-    def __post_init__(self):
-        if self.DNS_SERVERS is None:
-            # Use multiple reliable DNS servers
-            self.DNS_SERVERS = [
-                '1.1.1.1',      # Cloudflare
-                '8.8.8.8',      # Google
-                '9.9.9.9'      # Quad9
-            ]
-    
-    @classmethod
-    def from_args(cls, shodan_key: Optional[str] = None, 
-                  virustotal_key: Optional[str] = None,
-                  max_depth: int = 2,
-                  log_level: str = "INFO") -> 'Config':
-        """Create config from command line arguments."""
-        return cls(
-            shodan_key=shodan_key,
-            virustotal_key=virustotal_key,
-            max_depth=max_depth,
-            log_level=log_level.upper()
-        )
-    
-    def setup_logging(self, cli_mode: bool = True):
-        """Set up logging configuration."""
-        log_format = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
-        
-        if cli_mode:
-            # For CLI, use a more readable format
-            log_format = '%(asctime)s [%(levelname)s] %(message)s'
-        
-        logging.basicConfig(
-            level=getattr(logging, self.log_level, logging.INFO),
-            format=log_format,
-            datefmt='%H:%M:%S'
-        )
-        
-        # Set specific loggers
-        logging.getLogger('urllib3').setLevel(logging.WARNING)  # Reduce HTTP noise
-        logging.getLogger('requests').setLevel(logging.WARNING)  # Reduce HTTP noise
-        
-        if self.log_level == "DEBUG":
-            logging.getLogger(__name__.split('.')[0]).setLevel(logging.DEBUG)
-        
-        return logging.getLogger(__name__)

src/data_structures.py

@@ -1,475 +0,0 @@
-# File: src/data_structures.py
-"""Enhanced data structures for forensic DNS reconnaissance with full provenance tracking."""
-
-from dataclasses import dataclass, field
-from typing import Dict, List, Set, Optional, Any, Tuple
-from datetime import datetime
-from enum import Enum
-import json
-import logging
-import uuid
-
-# Set up logging for this module
-logger = logging.getLogger(__name__)
-
-class OperationType(Enum):
-    """Types of discovery operations."""
-    INITIAL_TARGET = "initial_target"
-    TLD_EXPANSION = "tld_expansion"
-    DNS_A = "dns_a"
-    DNS_AAAA = "dns_aaaa"
-    DNS_MX = "dns_mx"
-    DNS_NS = "dns_ns"
-    DNS_TXT = "dns_txt"
-    DNS_CNAME = "dns_cname"
-    DNS_SOA = "dns_soa"
-    DNS_PTR = "dns_ptr"
-    DNS_SRV = "dns_srv"
-    DNS_CAA = "dns_caa"
-    DNS_DNSKEY = "dns_dnskey"
-    DNS_DS = "dns_ds"
-    DNS_RRSIG = "dns_rrsig"
-    DNS_NSEC = "dns_nsec"
-    DNS_NSEC3 = "dns_nsec3"
-    DNS_REVERSE = "dns_reverse"
-    CERTIFICATE_CHECK = "certificate_check"
-    SHODAN_LOOKUP = "shodan_lookup"
-    VIRUSTOTAL_IP = "virustotal_ip"
-    VIRUSTOTAL_DOMAIN = "virustotal_domain"
-
-class DiscoveryMethod(Enum):
-    """How a hostname was discovered."""
-    INITIAL_TARGET = "initial_target"
-    TLD_EXPANSION = "tld_expansion"
-    DNS_RECORD_VALUE = "dns_record_value"
-    CERTIFICATE_SUBJECT = "certificate_subject"
-    DNS_SUBDOMAIN_EXTRACTION = "dns_subdomain_extraction"
-
-@dataclass
-class DNSRecord:
-    """DNS record information with enhanced metadata."""
-    record_type: str
-    value: str
-    ttl: Optional[int] = None
-    discovered_at: datetime = field(default_factory=datetime.now)
-    operation_id: str = field(default_factory=lambda: str(uuid.uuid4()))
-    
-    def to_dict(self) -> dict:
-        return {
-            'record_type': self.record_type,
-            'value': self.value,
-            'ttl': self.ttl,
-            'discovered_at': self.discovered_at.isoformat() if self.discovered_at else None,
-            'operation_id': self.operation_id
-        }
-
-@dataclass
-class Certificate:
-    """Certificate information with enhanced metadata."""
-    id: int
-    issuer: str
-    subject: str
-    not_before: datetime
-    not_after: datetime
-    is_wildcard: bool = False
-    discovered_at: datetime = field(default_factory=datetime.now)
-    operation_id: str = field(default_factory=lambda: str(uuid.uuid4()))
-    is_valid_now: bool = field(default=True)  # Based on current timestamp vs cert validity
-    
-    def __post_init__(self):
-        """Calculate if certificate is currently valid."""
-        now = datetime.now()
-        self.is_valid_now = self.not_before <= now <= self.not_after
-    
-    def to_dict(self) -> dict:
-        return {
-            'id': self.id,
-            'issuer': self.issuer,
-            'subject': self.subject,
-            'not_before': self.not_before.isoformat() if self.not_before else None,
-            'not_after': self.not_after.isoformat() if self.not_after else None,
-            'is_wildcard': self.is_wildcard,
-            'discovered_at': self.discovered_at.isoformat() if self.discovered_at else None,
-            'operation_id': self.operation_id,
-            'is_valid_now': self.is_valid_now
-        }
-
-@dataclass
-class ShodanResult:
-    """Shodan scan result with metadata."""
-    ip: str
-    ports: List[int]
-    services: Dict[str, Any]
-    organization: Optional[str] = None
-    country: Optional[str] = None
-    discovered_at: datetime = field(default_factory=datetime.now)
-    operation_id: str = field(default_factory=lambda: str(uuid.uuid4()))
-    
-    def to_dict(self) -> dict:
-        return {
-            'ip': self.ip,
-            'ports': self.ports,
-            'services': self.services,
-            'organization': self.organization,
-            'country': self.country,
-            'discovered_at': self.discovered_at.isoformat() if self.discovered_at else None,
-            'operation_id': self.operation_id
-        }
-
-@dataclass
-class VirusTotalResult:
-    """VirusTotal scan result with metadata."""
-    resource: str  # IP or domain
-    positives: int
-    total: int
-    scan_date: datetime
-    permalink: str
-    discovered_at: datetime = field(default_factory=datetime.now)
-    operation_id: str = field(default_factory=lambda: str(uuid.uuid4()))
-    
-    def to_dict(self) -> dict:
-        return {
-            'resource': self.resource,
-            'positives': self.positives,
-            'total': self.total,
-            'scan_date': self.scan_date.isoformat() if self.scan_date else None,
-            'permalink': self.permalink,
-            'discovered_at': self.discovered_at.isoformat() if self.discovered_at else None,
-            'operation_id': self.operation_id
-        }
-
-@dataclass
-class DiscoveryOperation:
-    """A single discovery operation with complete metadata."""
-    operation_id: str = field(default_factory=lambda: str(uuid.uuid4()))
-    operation_type: OperationType = OperationType.INITIAL_TARGET
-    target: str = ""
-    timestamp: datetime = field(default_factory=datetime.now)
-    success: bool = True
-    error_message: Optional[str] = None
-    
-    # Results of the operation
-    dns_records: List[DNSRecord] = field(default_factory=list)
-    certificates: List[Certificate] = field(default_factory=list)
-    shodan_results: List[ShodanResult] = field(default_factory=list)
-    virustotal_results: List[VirusTotalResult] = field(default_factory=list)
-    discovered_hostnames: List[str] = field(default_factory=list)  # New hostnames found
-    discovered_ips: List[str] = field(default_factory=list)  # New IPs found
-    
-    # Operation-specific metadata
-    metadata: Dict[str, Any] = field(default_factory=dict)
-    
-    def to_dict(self) -> dict:
-        return {
-            'operation_id': self.operation_id,
-            'operation_type': self.operation_type.value,
-            'target': self.target,
-            'timestamp': self.timestamp.isoformat() if self.timestamp else None,
-            'success': self.success,
-            'error_message': self.error_message,
-            'dns_records': [record.to_dict() for record in self.dns_records],
-            'certificates': [cert.to_dict() for cert in self.certificates],
-            'shodan_results': [result.to_dict() for result in self.shodan_results],
-            'virustotal_results': [result.to_dict() for result in self.virustotal_results],
-            'discovered_hostnames': self.discovered_hostnames,
-            'discovered_ips': self.discovered_ips,
-            'metadata': self.metadata
-        }
-
-@dataclass
-class DiscoveryEdge:
-    """Represents a discovery relationship between two hostnames."""
-    source_hostname: str  # The hostname that led to the discovery
-    target_hostname: str  # The hostname that was discovered
-    discovery_method: DiscoveryMethod
-    operation_id: str
-    timestamp: datetime
-    metadata: Dict[str, Any] = field(default_factory=dict)  # Additional context
-    
-    def to_dict(self) -> dict:
-        return {
-            'source_hostname': self.source_hostname,
-            'target_hostname': self.target_hostname,
-            'discovery_method': self.discovery_method.value,
-            'operation_id': self.operation_id,
-            'timestamp': self.timestamp.isoformat() if self.timestamp else None,
-            'metadata': self.metadata
-        }
-
-@dataclass
-class DiscoveryNode:
-    """A discovered hostname with complete provenance and current state."""
-    hostname: str
-    depth: int = 0
-    first_seen: datetime = field(default_factory=datetime.now)
-    last_updated: datetime = field(default_factory=datetime.now)
-    
-    # Current validity state
-    dns_exists: Optional[bool] = None  # Does this hostname resolve?
-    last_dns_check: Optional[datetime] = None
-    resolved_ips: Set[str] = field(default_factory=set)
-    
-    # Discovery provenance - can have multiple discovery paths
-    discovery_methods: Set[DiscoveryMethod] = field(default_factory=set)
-    discovered_by_operations: Set[str] = field(default_factory=set)  # operation_ids
-    
-    # Associated data (current state)
-    dns_records_by_type: Dict[str, List[DNSRecord]] = field(default_factory=dict)
-    certificates: List[Certificate] = field(default_factory=list)
-    shodan_results: List[ShodanResult] = field(default_factory=list)
-    virustotal_results: List[VirusTotalResult] = field(default_factory=list)
-    reverse_dns: Optional[str] = None
-    
-    def add_dns_record(self, record: DNSRecord) -> None:
-        """Add a DNS record to this node."""
-        record_type = record.record_type
-        if record_type not in self.dns_records_by_type:
-            self.dns_records_by_type[record_type] = []
-        self.dns_records_by_type[record_type].append(record)
-        self.last_updated = datetime.now()
-        
-        # Update resolved IPs if this is an A or AAAA record
-        if record_type in ['A', 'AAAA']:
-            self.resolved_ips.add(record.value)
-            self.dns_exists = True
-            self.last_dns_check = record.discovered_at
-    
-    def add_certificate(self, certificate: Certificate) -> None:
-        """Add a certificate to this node."""
-        self.certificates.append(certificate)
-        self.last_updated = datetime.now()
-    
-    def get_all_dns_records(self) -> List[DNSRecord]:
-        """Get all DNS records for this node."""
-        all_records = []
-        for records in self.dns_records_by_type.values():
-            all_records.extend(records)
-        return all_records
-    
-    def get_current_certificates(self) -> List[Certificate]:
-        """Get currently valid certificates."""
-        return [cert for cert in self.certificates if cert.is_valid_now]
-    
-    def get_expired_certificates(self) -> List[Certificate]:
-        """Get expired certificates."""
-        return [cert for cert in self.certificates if not cert.is_valid_now]
-    
-    def to_dict(self) -> dict:
-        return {
-            'hostname': self.hostname,
-            'depth': self.depth,
-            'first_seen': self.first_seen.isoformat() if self.first_seen else None,
-            'last_updated': self.last_updated.isoformat() if self.last_updated else None,
-            'dns_exists': self.dns_exists,
-            'last_dns_check': self.last_dns_check.isoformat() if self.last_dns_check else None,
-            'resolved_ips': sorted(list(self.resolved_ips)),
-            'discovery_methods': [method.value for method in self.discovery_methods],
-            'discovered_by_operations': sorted(list(self.discovered_by_operations)),
-            'dns_records_by_type': {
-                record_type: [record.to_dict() for record in records]
-                for record_type, records in self.dns_records_by_type.items()
-            },
-            'certificates': [cert.to_dict() for cert in self.certificates],
-            'current_certificates': [cert.to_dict() for cert in self.get_current_certificates()],
-            'expired_certificates': [cert.to_dict() for cert in self.get_expired_certificates()],
-            'shodan_results': [result.to_dict() for result in self.shodan_results],
-            'virustotal_results': [result.to_dict() for result in self.virustotal_results],
-            'reverse_dns': self.reverse_dns
-        }
-
-@dataclass
-class ForensicReconData:
-    """Enhanced reconnaissance data with full forensic tracking and graph structure."""
-    
-    # Core graph structure
-    nodes: Dict[str, DiscoveryNode] = field(default_factory=dict)  # hostname -> node
-    edges: List[DiscoveryEdge] = field(default_factory=list)
-    operations: Dict[str, DiscoveryOperation] = field(default_factory=dict)  # operation_id -> operation
-    
-    # Quick lookup indexes
-    ip_addresses: Set[str] = field(default_factory=set)
-    operation_timeline: List[str] = field(default_factory=list)  # ordered operation_ids
-    
-    # Metadata
-    start_time: datetime = field(default_factory=datetime.now)
-    end_time: Optional[datetime] = None
-    scan_config: Dict[str, Any] = field(default_factory=dict)
-    
-    def add_node(self, hostname: str, depth: int = 0, 
-                 discovery_method: DiscoveryMethod = DiscoveryMethod.INITIAL_TARGET,
-                 operation_id: Optional[str] = None) -> DiscoveryNode:
-        """Add or get a discovery node."""
-        hostname = hostname.lower()
-        
-        if hostname not in self.nodes:
-            node = DiscoveryNode(hostname=hostname, depth=depth)
-            self.nodes[hostname] = node
-            logger.debug(f"Created new node: {hostname} at depth {depth}")
-        else:
-            node = self.nodes[hostname]
-            # Update depth if this is a shallower discovery
-            if depth < node.depth:
-                node.depth = depth
-                logger.debug(f"Updated node {hostname} depth: {node.depth} -> {depth}")
-        
-        # Track discovery method and operation
-        node.discovery_methods.add(discovery_method)
-        if operation_id:
-            node.discovered_by_operations.add(operation_id)
-        
-        return node
-    
-    def add_edge(self, source: str, target: str, discovery_method: DiscoveryMethod,
-                 operation_id: str, metadata: Dict[str, Any] = None) -> None:
-        """Add a discovery edge."""
-        edge = DiscoveryEdge(
-            source_hostname=source.lower(),
-            target_hostname=target.lower(),
-            discovery_method=discovery_method,
-            operation_id=operation_id,
-            timestamp=datetime.now(),
-            metadata=metadata or {}
-        )
-        self.edges.append(edge)
-        logger.debug(f"Added edge: {source} -> {target} via {discovery_method.value}")
-    
-    def add_operation(self, operation: DiscoveryOperation) -> None:
-        """Add an operation to the timeline."""
-        self.operations[operation.operation_id] = operation
-        self.operation_timeline.append(operation.operation_id)
-        logger.debug(f"Added operation: {operation.operation_type.value} on {operation.target}")
-    
-    def get_node(self, hostname: str) -> Optional[DiscoveryNode]:
-        """Get a node by hostname."""
-        return self.nodes.get(hostname.lower())
-    
-    def get_children(self, hostname: str) -> List[str]:
-        """Get all hostnames discovered from this hostname."""
-        hostname = hostname.lower()
-        children = []
-        for edge in self.edges:
-            if edge.source_hostname == hostname:
-                children.append(edge.target_hostname)
-        return children
-    
-    def get_parents(self, hostname: str) -> List[str]:
-        """Get all hostnames that led to discovering this hostname."""
-        hostname = hostname.lower()
-        parents = []
-        for edge in self.edges:
-            if edge.target_hostname == hostname:
-                parents.append(edge.source_hostname)
-        return parents
-    
-    def get_discovery_path(self, hostname: str) -> List[Tuple[str, str, DiscoveryMethod]]:
-        """Get the discovery path(s) to a hostname."""
-        hostname = hostname.lower()
-        paths = []
-        
-        def trace_path(current: str, visited: Set[str], current_path: List):
-            if current in visited:
-                return  # Avoid cycles
-            
-            visited.add(current)
-            parents = self.get_parents(current)
-            
-            if not parents:
-                # This is a root node, we have a complete path
-                paths.append(current_path.copy())
-            else:
-                for parent in parents:
-                    # Find the edge that connects parent to current
-                    for edge in self.edges:
-                        if edge.source_hostname == parent and edge.target_hostname == current:
-                            new_path = [(parent, current, edge.discovery_method)] + current_path
-                            trace_path(parent, visited.copy(), new_path)
-        
-        trace_path(hostname, set(), [])
-        return paths
-    
-    def get_stats(self) -> Dict[str, int]:
-        """Get current statistics."""
-        total_dns_records = sum(len(node.get_all_dns_records()) for node in self.nodes.values())
-        total_certificates = sum(len(node.certificates) for node in self.nodes.values())
-        current_certificates = sum(len(node.get_current_certificates()) for node in self.nodes.values())
-        expired_certificates = sum(len(node.get_expired_certificates()) for node in self.nodes.values())
-        total_shodan = sum(len(node.shodan_results) for node in self.nodes.values())
-        total_virustotal = sum(len(node.virustotal_results) for node in self.nodes.values())
-        
-        return {
-            'hostnames': len(self.nodes),
-            'ip_addresses': len(self.ip_addresses),
-            'discovery_edges': len(self.edges),
-            'operations_performed': len(self.operations),
-            'dns_records': total_dns_records,
-            'certificates_total': total_certificates,
-            'certificates_current': current_certificates,
-            'certificates_expired': expired_certificates,
-            'shodan_results': total_shodan,
-            'virustotal_results': total_virustotal
-        }
-    
-    def to_dict(self) -> dict:
-        """Export data as a serializable dictionary."""
-        logger.info(f"Serializing ForensicReconData with {len(self.nodes)} nodes, {len(self.edges)} edges, {len(self.operations)} operations")
-        
-        return {
-            'nodes': {hostname: node.to_dict() for hostname, node in self.nodes.items()},
-            'edges': [edge.to_dict() for edge in self.edges],
-            'operations': {op_id: op.to_dict() for op_id, op in self.operations.items()},
-            'operation_timeline': self.operation_timeline,
-            'ip_addresses': sorted(list(self.ip_addresses)),
-            'metadata': {
-                'start_time': self.start_time.isoformat() if self.start_time else None,
-                'end_time': self.end_time.isoformat() if self.end_time else None,
-                'scan_config': self.scan_config,
-                'stats': self.get_stats()
-            },
-            'graph_analysis': self._generate_graph_analysis()
-        }
-    
-    def _generate_graph_analysis(self) -> Dict[str, Any]:
-        """Generate graph analysis metadata."""
-        # Depth distribution
-        depth_distribution = {}
-        for node in self.nodes.values():
-            depth = node.depth
-            depth_distribution[depth] = depth_distribution.get(depth, 0) + 1
-        
-        # Root nodes (no parents)
-        root_nodes = [hostname for hostname in self.nodes.keys() 
-                     if not self.get_parents(hostname)]
-        
-        # Leaf nodes (no children)
-        leaf_nodes = [hostname for hostname in self.nodes.keys() 
-                     if not self.get_children(hostname)]
-        
-        # Discovery method distribution
-        method_distribution = {}
-        for edge in self.edges:
-            method = edge.discovery_method.value
-            method_distribution[method] = method_distribution.get(method, 0) + 1
-        
-        return {
-            'depth_distribution': depth_distribution,
-            'max_depth': max(depth_distribution.keys()) if depth_distribution else 0,
-            'root_nodes': root_nodes,
-            'leaf_nodes': leaf_nodes,
-            'discovery_method_distribution': method_distribution,
-            'total_discovery_paths': len(self.edges)
-        }
-    
-    def to_json(self) -> str:
-        """Export data as JSON."""
-        try:
-            return json.dumps(self.to_dict(), indent=2, ensure_ascii=False)
-        except Exception as e:
-            logger.error(f"Failed to serialize to JSON: {e}")
-            return json.dumps({
-                'error': str(e),
-                'stats': self.get_stats(),
-                'timestamp': datetime.now().isoformat()
-            }, indent=2)
-
-# Backward compatibility aliases (for gradual migration)
-ReconData = ForensicReconData

src/dns_resolver.py

@@ -1,368 +0,0 @@
-# File: src/dns_resolver.py
-"""DNS resolution functionality with enhanced TLD testing and forensic operation tracking."""
-
-import dns.resolver
-import dns.reversename
-import dns.query
-import dns.zone
-from typing import List, Dict, Optional, Set
-import socket
-import time
-import logging
-import uuid
-from .data_structures import DNSRecord, ReconData
-from .config import Config
-
-# Module logger
-logger = logging.getLogger(__name__)
-
-class DNSResolver:
-    """DNS resolution and record lookup with optimized TLD testing and forensic tracking."""
-    
-    # All DNS record types to query
-    RECORD_TYPES = [
-        'A', 'AAAA', 'MX', 'NS', 'TXT', 'CNAME', 'SOA', 'PTR', 
-        'SRV', 'CAA', 'DNSKEY', 'DS', 'RRSIG', 'NSEC', 'NSEC3'
-    ]
-    
-    def __init__(self, config: Config):
-        self.config = config
-        self.last_request = 0
-        self.query_count = 0
-        
-        logger.info(f"🌐 DNS resolver initialized with {len(config.DNS_SERVERS)} servers: {config.DNS_SERVERS}")
-        logger.info(f"⚡ DNS rate limit: {config.DNS_RATE_LIMIT}/s, timeout: {config.DNS_TIMEOUT}s")
-    
-    def _rate_limit(self):
-        """Apply rate limiting - more graceful for DNS servers."""
-        now = time.time()
-        time_since_last = now - self.last_request
-        min_interval = 1.0 / self.config.DNS_RATE_LIMIT
-        
-        if time_since_last < min_interval:
-            sleep_time = min_interval - time_since_last
-            # Only log if sleep is significant to reduce spam
-            if sleep_time > 0.1:
-                logger.debug(f"⏸️ DNS rate limiting: sleeping for {sleep_time:.2f}s")
-            time.sleep(sleep_time)
-        
-        self.last_request = time.time()
-        self.query_count += 1
-    
-    def resolve_hostname_fast(self, hostname: str) -> List[str]:
-        """Fast hostname resolution optimized for TLD testing."""
-        ips = []
-        
-        logger.debug(f"🚀 Fast resolving hostname: {hostname}")
-        
-        # Use only the first DNS server and shorter timeout for TLD testing
-        resolver = dns.resolver.Resolver()
-        resolver.nameservers = [self.config.DNS_SERVERS[0]]  # Use primary DNS only
-        resolver.timeout = 2  # Shorter timeout for TLD testing
-        resolver.lifetime = 2  # Total query time limit
-        
-        try:
-            # Try A records only for speed (most common)
-            answers = resolver.resolve(hostname, 'A')
-            for answer in answers:
-                ips.append(str(answer))
-                logger.debug(f"⚡ Fast A record for {hostname}: {answer}")
-        except dns.resolver.NXDOMAIN:
-            logger.debug(f"❌ NXDOMAIN for {hostname}")
-        except dns.resolver.NoAnswer:
-            logger.debug(f"⚠️ No A record for {hostname}")
-        except dns.resolver.Timeout:
-            logger.debug(f"⏱️ Timeout for {hostname}")
-        except Exception as e:
-            logger.debug(f"⚠️ Error fast resolving {hostname}: {e}")
-        
-        if ips:
-            logger.debug(f"⚡ Fast resolved {hostname} to {len(ips)} IPs: {ips}")
-        
-        return ips
-    
-    def resolve_hostname(self, hostname: str, operation_id: Optional[str] = None) -> List[str]:
-        """Resolve hostname to IP addresses (full resolution with retries)."""
-        ips = []
-        
-        logger.debug(f"🔍 Resolving hostname: {hostname}")
-        
-        for dns_server in self.config.DNS_SERVERS:
-            self._rate_limit()
-            resolver = dns.resolver.Resolver()
-            resolver.nameservers = [dns_server]
-            resolver.timeout = self.config.DNS_TIMEOUT
-            
-            try:
-                # Try A records
-                answers = resolver.resolve(hostname, 'A')
-                for answer in answers:
-                    ips.append(str(answer))
-                    logger.debug(f"✅ A record for {hostname}: {answer}")
-            except dns.resolver.NXDOMAIN:
-                logger.debug(f"❌ NXDOMAIN for {hostname} A record on {dns_server}")
-            except dns.resolver.NoAnswer:
-                logger.debug(f"⚠️ No A record for {hostname} on {dns_server}")
-            except Exception as e:
-                logger.debug(f"⚠️ Error resolving A record for {hostname} on {dns_server}: {e}")
-            
-            try:
-                # Try AAAA records (IPv6)
-                answers = resolver.resolve(hostname, 'AAAA')
-                for answer in answers:
-                    ips.append(str(answer))
-                    logger.debug(f"✅ AAAA record for {hostname}: {answer}")
-            except dns.resolver.NXDOMAIN:
-                logger.debug(f"❌ NXDOMAIN for {hostname} AAAA record on {dns_server}")
-            except dns.resolver.NoAnswer:
-                logger.debug(f"⚠️ No AAAA record for {hostname} on {dns_server}")
-            except Exception as e:
-                logger.debug(f"⚠️ Error resolving AAAA record for {hostname} on {dns_server}: {e}")
-        
-        unique_ips = list(set(ips))
-        if unique_ips:
-            logger.info(f"✅ Resolved {hostname} to {len(unique_ips)} unique IPs: {unique_ips}")
-        else:
-            logger.debug(f"❌ No IPs found for {hostname}")
-            
-        return unique_ips
-    
-    def get_all_dns_records(self, hostname: str, operation_id: Optional[str] = None) -> List[DNSRecord]:
-        """Get all DNS records for a hostname with forensic tracking."""
-        records = []
-        successful_queries = 0
-        
-        # Generate operation ID if not provided
-        if operation_id is None:
-            operation_id = str(uuid.uuid4())
-        
-        logger.debug(f"📋 Getting all DNS records for: {hostname} (operation: {operation_id})")
-        
-        for record_type in self.RECORD_TYPES:
-            type_found = False
-            
-            for dns_server in self.config.DNS_SERVERS:
-                self._rate_limit()
-                resolver = dns.resolver.Resolver()
-                resolver.nameservers = [dns_server]
-                resolver.timeout = self.config.DNS_TIMEOUT
-                
-                try:
-                    answers = resolver.resolve(hostname, record_type)
-                    for answer in answers:
-                        # Create DNSRecord with forensic metadata
-                        record = DNSRecord(
-                            record_type=record_type,
-                            value=str(answer),
-                            ttl=answers.ttl,
-                            operation_id=operation_id  # Forensic tracking
-                        )
-                        records.append(record)
-                        
-                        if not type_found:
-                            logger.debug(f"✅ Found {record_type} record for {hostname}: {answer}")
-                            type_found = True
-                    
-                    if not type_found:
-                        successful_queries += 1
-                    break  # Found records, no need to query other DNS servers for this type
-                    
-                except dns.resolver.NXDOMAIN:
-                    logger.debug(f"❌ NXDOMAIN for {hostname} {record_type} on {dns_server}")
-                    break  # Domain doesn't exist, no point checking other servers
-                except dns.resolver.NoAnswer:
-                    logger.debug(f"⚠️ No {record_type} record for {hostname} on {dns_server}")
-                    continue  # Try next DNS server
-                except dns.resolver.Timeout:
-                    logger.debug(f"⏱️ Timeout for {hostname} {record_type} on {dns_server}")
-                    continue  # Try next DNS server
-                except Exception as e:
-                    logger.debug(f"⚠️ Error querying {record_type} for {hostname} on {dns_server}: {e}")
-                    continue  # Try next DNS server
-        
-        logger.info(f"📋 Found {len(records)} DNS records for {hostname} across {len(set(r.record_type for r in records))} record types")
-        
-        # Log query statistics every 100 queries
-        if self.query_count % 100 == 0:
-            logger.info(f"📊 DNS query statistics: {self.query_count} total queries performed")
-            
-        return records
-    
-    def query_specific_record_type(self, hostname: str, record_type: str, operation_id: Optional[str] = None) -> List[DNSRecord]:
-        """Query a specific DNS record type with forensic tracking."""
-        records = []
-        
-        # Generate operation ID if not provided
-        if operation_id is None:
-            operation_id = str(uuid.uuid4())
-        
-        logger.debug(f"🎯 Querying {record_type} records for {hostname} (operation: {operation_id})")
-        
-        for dns_server in self.config.DNS_SERVERS:
-            self._rate_limit()
-            resolver = dns.resolver.Resolver()
-            resolver.nameservers = [dns_server]
-            resolver.timeout = self.config.DNS_TIMEOUT
-            
-            try:
-                answers = resolver.resolve(hostname, record_type)
-                for answer in answers:
-                    # Create DNSRecord with forensic metadata
-                    record = DNSRecord(
-                        record_type=record_type,
-                        value=str(answer),
-                        ttl=answers.ttl,
-                        operation_id=operation_id  # Forensic tracking
-                    )
-                    records.append(record)
-                    logger.debug(f"✅ {record_type} record for {hostname}: {answer}")
-                
-                break  # Found records, no need to query other DNS servers
-                
-            except dns.resolver.NXDOMAIN:
-                logger.debug(f"❌ NXDOMAIN for {hostname} {record_type} on {dns_server}")
-                break  # Domain doesn't exist, no point checking other servers
-            except dns.resolver.NoAnswer:
-                logger.debug(f"⚠️ No {record_type} record for {hostname} on {dns_server}")
-                continue  # Try next DNS server
-            except dns.resolver.Timeout:
-                logger.debug(f"⏱️ Timeout for {hostname} {record_type} on {dns_server}")
-                continue  # Try next DNS server
-            except Exception as e:
-                logger.debug(f"⚠️ Error querying {record_type} for {hostname} on {dns_server}: {e}")
-                continue  # Try next DNS server
-        
-        logger.debug(f"🎯 Found {len(records)} {record_type} records for {hostname}")
-        return records
-    
-    def reverse_dns_lookup(self, ip: str, operation_id: Optional[str] = None) -> Optional[str]:
-        """Perform reverse DNS lookup with forensic tracking."""
-        logger.debug(f"🔍 Reverse DNS lookup for: {ip} (operation: {operation_id or 'auto'})")
-        
-        try:
-            self._rate_limit()
-            hostname = socket.gethostbyaddr(ip)[0]
-            logger.info(f"✅ Reverse DNS for {ip}: {hostname}")
-            return hostname
-        except socket.herror:
-            logger.debug(f"❌ No reverse DNS for {ip}")
-            return None
-        except Exception as e:
-            logger.debug(f"⚠️ Error in reverse DNS for {ip}: {e}")
-            return None
-    
-    def extract_subdomains_from_dns(self, records: List[DNSRecord]) -> Set[str]:
-        """Extract potential subdomains from DNS records."""
-        subdomains = set()
-        
-        logger.debug(f"🌿 Extracting subdomains from {len(records)} DNS records")
-        
-        for record in records:
-            value = record.value.lower()
-            
-            # Extract from different record types
-            try:
-                if record.record_type == 'MX':
-                    # MX record format: "priority hostname"
-                    parts = value.split()
-                    if len(parts) >= 2:
-                        hostname = parts[-1].rstrip('.')  # Take the last part (hostname)
-                        if self._is_valid_hostname(hostname):
-                            subdomains.add(hostname)
-                            logger.debug(f"🌿 Found subdomain from MX: {hostname}")
-                
-                elif record.record_type in ['CNAME', 'NS']:
-                    # Direct hostname records
-                    hostname = value.rstrip('.')
-                    if self._is_valid_hostname(hostname):
-                        subdomains.add(hostname)
-                        logger.debug(f"🌿 Found subdomain from {record.record_type}: {hostname}")
-                
-                elif record.record_type == 'TXT':
-                    # Search for domain-like strings in TXT records
-                    # Common patterns: include:example.com, v=spf1 include:_spf.google.com
-                    words = value.replace(',', ' ').replace(';', ' ').split()
-                    for word in words:
-                        # Look for include: patterns
-                        if word.startswith('include:'):
-                            hostname = word[8:].rstrip('.')
-                            if self._is_valid_hostname(hostname):
-                                subdomains.add(hostname)
-                                logger.debug(f"🌿 Found subdomain from TXT include: {hostname}")
-                        
-                        # Look for other domain patterns
-                        elif '.' in word and not word.startswith('http'):
-                            clean_word = word.strip('",\'()[]{}').rstrip('.')
-                            if self._is_valid_hostname(clean_word):
-                                subdomains.add(clean_word)
-                                logger.debug(f"🌿 Found subdomain from TXT: {clean_word}")
-                
-                elif record.record_type == 'SRV':
-                    # SRV record format: "priority weight port target"
-                    parts = value.split()
-                    if len(parts) >= 4:
-                        hostname = parts[-1].rstrip('.')  # Target hostname
-                        if self._is_valid_hostname(hostname):
-                            subdomains.add(hostname)
-                            logger.debug(f"🌿 Found subdomain from SRV: {hostname}")
-                        
-            except Exception as e:
-                logger.debug(f"⚠️ Error extracting subdomain from {record.record_type} record '{value}': {e}")
-                continue
-        
-        if subdomains:
-            logger.info(f"🌿 Extracted {len(subdomains)} potential subdomains")
-        else:
-            logger.debug("❌ No subdomains extracted from DNS records")
-        
-        return subdomains
-    
-    def _is_valid_hostname(self, hostname: str) -> bool:
-        """Basic hostname validation."""
-        if not hostname or len(hostname) > 255:
-            return False
-        
-        # Must contain at least one dot
-        if '.' not in hostname:
-            return False
-        
-        # Must not be an IP address
-        if self._looks_like_ip(hostname):
-            return False
-        
-        # Basic character check - allow international domains
-        # Remove overly restrictive character filtering
-        if not hostname.replace('-', '').replace('.', '').replace('_', '').isalnum():
-            # Allow some special cases for internationalized domains
-            try:
-                hostname.encode('ascii')
-            except UnicodeEncodeError:
-                return False  # Skip non-ASCII for now
-        
-        # Must have reasonable length parts
-        parts = hostname.split('.')
-        if len(parts) < 2:
-            return False
-        
-        # Each part should be reasonable length
-        for part in parts:
-            if len(part) < 1 or len(part) > 63:
-                return False
-        
-        return True
-    
-    def _looks_like_ip(self, text: str) -> bool:
-        """Check if text looks like an IP address."""
-        try:
-            socket.inet_aton(text)
-            return True
-        except socket.error:
-            pass
-        
-        try:
-            socket.inet_pton(socket.AF_INET6, text)
-            return True
-        except socket.error:
-            pass
-        
-        return False

src/main.py

@@ -1,220 +0,0 @@
-# File: src/main.py
-"""Main CLI interface for the forensic reconnaissance tool with two-mode operation."""
-
-import click
-import json
-import sys
-import logging
-from pathlib import Path
-from .config import Config
-from .reconnaissance import ForensicReconnaissanceEngine
-from .report_generator import ForensicReportGenerator
-from .web_app import create_app
-
-# Module logger
-logger = logging.getLogger(__name__)
-
-@click.command()
-@click.argument('target', required=False)
-@click.option('--web', is_flag=True, help='Start web interface instead of CLI')
-@click.option('--shodan-key', help='Shodan API key')
-@click.option('--virustotal-key', help='VirusTotal API key')
-@click.option('--max-depth', default=2, help='Maximum recursion depth for full domain mode (default: 2)')
-@click.option('--output', '-o', help='Output file prefix (will create .json and .txt files)')
-@click.option('--json-only', is_flag=True, help='Only output JSON')
-@click.option('--text-only', is_flag=True, help='Only output text report')
-@click.option('--port', default=5000, help='Port for web interface (default: 5000)')
-@click.option('--verbose', '-v', is_flag=True, help='Enable verbose logging (DEBUG level)')
-@click.option('--quiet', '-q', is_flag=True, help='Quiet mode (WARNING level only)')
-def main(target, web, shodan_key, virustotal_key, max_depth, output, json_only, text_only, port, verbose, quiet):
-    """Forensic DNS Reconnaissance Tool - Two-Mode Operation
-    
-    MODE 1 - Hostname-only (e.g., 'cc24'):
-      Expands hostname to all TLDs (cc24.com, cc24.net, etc.)
-      No recursive enumeration to avoid third-party infrastructure noise
-      Perfect for discovering domains using a specific hostname
-    
-    MODE 2 - Full domain (e.g., 'cc24.com'):
-      Full recursive reconnaissance with subdomain discovery
-      Maps complete infrastructure of the specified domain
-      Uses max-depth for recursive enumeration
-      Creates forensic-grade evidence chain with operation tracking
-    
-    Examples:
-        recon cc24                           # Mode 1: Find all cc24.* domains (no recursion)
-        recon cc24.com                       # Mode 2: Map cc24.com infrastructure (with forensic tracking)
-        recon cc24.com --max-depth 3         # Mode 2: Deeper recursive enumeration
-        recon cc24 -v                        # Mode 1: Verbose TLD expansion
-        recon --web                          # Start web interface with forensic visualization
-    """
-    
-    # Determine log level
-    if verbose:
-        log_level = "DEBUG"
-    elif quiet:
-        log_level = "WARNING"
-    else:
-        log_level = "INFO"
-    
-    # Create configuration and setup logging
-    config = Config.from_args(shodan_key, virustotal_key, max_depth, log_level)
-    config.setup_logging(cli_mode=True)
-    
-    if web:
-        # Start web interface
-        logger.info("🌐 Starting forensic web interface...")
-        app = create_app(config)
-        logger.info(f"🚀 Forensic web interface starting on http://0.0.0.0:{port}")
-        app.run(host='0.0.0.0', port=port, debug=False)
-        return
-    
-    if not target:
-        click.echo("Error: TARGET is required for CLI mode. Use --web for web interface.")
-        sys.exit(1)
-    
-    # Initialize forensic reconnaissance engine
-    logger.info("🔬 Initializing forensic reconnaissance engine...")
-    engine = ForensicReconnaissanceEngine(config)
-    
-    # Set up progress callback for CLI
-    def progress_callback(message, percentage=None):
-        if percentage is not None:
-            click.echo(f"[{percentage:3d}%] {message}")
-        else:
-            click.echo(f"       {message}")
-    
-    engine.set_progress_callback(progress_callback)
-    
-    # Display startup information
-    click.echo("=" * 80)
-    click.echo("FORENSIC DNS RECONNAISSANCE TOOL")
-    click.echo("=" * 80)
-    click.echo(f"Target: {target}")
-    
-    # Show operation mode
-    if '.' in target:
-        click.echo(f"Mode: Full domain reconnaissance (recursive depth: {max_depth})")
-        click.echo("   → Will map complete infrastructure with forensic tracking")
-        click.echo("   → Creates evidence chain for each discovery operation")
-    else:
-        click.echo(f"Mode: Hostname-only reconnaissance (TLD expansion)")
-        click.echo("   → Will find all domains using this hostname (no recursion)")
-        click.echo("   → Limited forensic tracking for efficiency")
-    
-    click.echo(f"DNS servers: {', '.join(config.DNS_SERVERS[:3])}{'...' if len(config.DNS_SERVERS) > 3 else ''}")
-    click.echo(f"DNS rate limit: {config.DNS_RATE_LIMIT}/s")
-    
-    if shodan_key:
-        click.echo("🕵️ Shodan integration enabled")
-    else:
-        click.echo("Shodan integration disabled (no API key)")
-        
-    if virustotal_key:
-        click.echo("🛡️ VirusTotal integration enabled")
-    else:
-        click.echo("VirusTotal integration disabled (no API key)")
-    
-    click.echo("")
-    
-    # Run forensic reconnaissance
-    try:
-        logger.info(f"🎯 Starting forensic reconnaissance for target: {target}")
-        data = engine.run_reconnaissance(target)
-        
-        # Display final statistics
-        stats = data.get_stats()
-        graph_analysis = data._generate_graph_analysis()
-        
-        click.echo("")
-        click.echo("=" * 80)
-        click.echo("FORENSIC RECONNAISSANCE COMPLETE")
-        click.echo("=" * 80)
-        click.echo(f"Hostnames discovered: {stats['hostnames']}")
-        click.echo(f"IP addresses found: {stats['ip_addresses']}")
-        click.echo(f"Discovery relationships: {stats['discovery_edges']}")
-        click.echo(f"Operations performed: {stats['operations_performed']}")
-        click.echo(f"DNS records collected: {stats['dns_records']}")
-        click.echo(f"Certificates found: {stats['certificates_total']} ({stats['certificates_current']} valid, {stats['certificates_expired']} expired)")
-        click.echo(f"Shodan results: {stats['shodan_results']}")
-        click.echo(f"VirusTotal results: {stats['virustotal_results']}")
-        click.echo(f"Maximum discovery depth: {graph_analysis['max_depth']}")
-        
-        # Calculate and display timing
-        if data.end_time and data.start_time:
-            duration = data.end_time - data.start_time
-            click.echo(f"Total time: {duration}")
-        
-        click.echo("")
-        
-        # Generate forensic reports
-        logger.info("📄 Generating forensic reports...")
-        report_gen = ForensicReportGenerator(data)
-        
-        if output:
-            # Save to files
-            saved_files = []
-            
-            if not text_only:
-                json_file = f"{output}.json"
-                try:
-                    json_content = data.to_json()
-                    with open(json_file, 'w', encoding='utf-8') as f:
-                        f.write(json_content)
-                    saved_files.append(json_file)
-                    logger.info(f"📄 Forensic JSON report saved: {json_file}")
-                except Exception as e:
-                    logger.error(f"❌ Failed to save JSON report: {e}")
-            
-            if not json_only:
-                text_file = f"{output}.txt"
-                try:
-                    with open(text_file, 'w', encoding='utf-8') as f:
-                        f.write(report_gen.generate_text_report())
-                    saved_files.append(text_file)
-                    logger.info(f"📄 Forensic text report saved: {text_file}")
-                except Exception as e:
-                    logger.error(f"❌ Failed to save text report: {e}")
-            
-            if saved_files:
-                click.echo(f"📁 Forensic reports saved:")
-                for file in saved_files:
-                    click.echo(f"   {file}")
-        
-        else:
-            # Output to stdout
-            if json_only:
-                try:
-                    click.echo(data.to_json())
-                except Exception as e:
-                    logger.error(f"❌ Failed to generate JSON output: {e}")
-                    click.echo(f"Error generating JSON: {e}")
-            elif text_only:
-                try:
-                    click.echo(report_gen.generate_text_report())
-                except Exception as e:
-                    logger.error(f"❌ Failed to generate text report: {e}")
-                    click.echo(f"Error generating text report: {e}")
-            else:
-                # Default: show text report
-                try:
-                    click.echo(report_gen.generate_text_report())
-                    click.echo(f"\n📊 To get JSON output with full forensic data, use: --json-only")
-                    click.echo(f"💾 To save reports, use: --output filename")
-                    click.echo(f"🌐 For interactive visualization, use: --web")
-                except Exception as e:
-                    logger.error(f"❌ Failed to generate report: {e}")
-                    click.echo(f"Error generating report: {e}")
-    
-    except KeyboardInterrupt:
-        logger.warning("⚠️ Forensic reconnaissance interrupted by user")
-        click.echo("\n🛑 Reconnaissance interrupted by user.")
-        sys.exit(1)
-    except Exception as e:
-        logger.error(f"❌ Error during forensic reconnaissance: {e}", exc_info=True)
-        click.echo(f"❌ Error during reconnaissance: {e}")
-        if verbose:
-            raise  # Re-raise in verbose mode to show full traceback
-        sys.exit(1)
-
-if __name__ == '__main__':
-    main()

src/reconnaissance.py

@@ -1,670 +0,0 @@
-# File: src/reconnaissance.py
-"""Enhanced reconnaissance logic with forensic-grade operation tracking and provenance."""
-
-import threading
-import concurrent.futures
-import logging
-from datetime import datetime
-from typing import Set, List, Optional, Tuple, Dict, Any
-from .data_structures import (
-    ForensicReconData, DiscoveryNode, DiscoveryOperation, DiscoveryEdge,
-    OperationType, DiscoveryMethod, DNSRecord, Certificate
-)
-from .config import Config
-from .dns_resolver import DNSResolver
-from .certificate_checker import CertificateChecker
-from .shodan_client import ShodanClient
-from .virustotal_client import VirusTotalClient
-from .tld_fetcher import TLDFetcher
-
-# Set up logging for this module
-logger = logging.getLogger(__name__)
-
-class ForensicReconnaissanceEngine:
-    """Enhanced reconnaissance engine with complete forensic tracking and provenance."""
-    
-    def __init__(self, config: Config):
-        self.config = config
-        
-        # Initialize clients
-        self.dns_resolver = DNSResolver(config)
-        self.cert_checker = CertificateChecker(config)
-        self.tld_fetcher = TLDFetcher()
-        
-        # Optional clients
-        self.shodan_client = None
-        if config.shodan_key:
-            self.shodan_client = ShodanClient(config.shodan_key, config)
-            logger.info("🕵️ Shodan client initialized")
-        
-        self.virustotal_client = None
-        if config.virustotal_key:
-            self.virustotal_client = VirusTotalClient(config.virustotal_key, config)
-            logger.info("🛡️ VirusTotal client initialized")
-        
-        # Progress tracking
-        self.progress_callback = None
-        self._lock = threading.Lock()
-        
-        # Track operation mode
-        self.is_hostname_only_mode = False
-        
-        # Operation tracking
-        self.pending_operations = []
-        self.completed_operations = []
-    
-    def set_progress_callback(self, callback):
-        """Set callback for progress updates."""
-        self.progress_callback = callback
-    
-    def set_shared_data(self, shared_data: ForensicReconData):
-        """Set shared data object for live updates."""
-        self.data = shared_data
-        logger.info("📊 Using shared forensic data object for live updates")
-    
-    def _update_progress(self, message: str, percentage: int = None):
-        """Update progress if callback is set."""
-        logger.info(f"Progress: {message} ({percentage}%)" if percentage else f"Progress: {message}")
-        if self.progress_callback:
-            self.progress_callback(message, percentage)
-    
-    def run_reconnaissance(self, target: str) -> ForensicReconData:
-        """Run forensic reconnaissance with complete operation tracking."""
-        # Initialize data structure
-        if not hasattr(self, 'data') or self.data is None:
-            self.data = ForensicReconData()
-            logger.info("🔬 Created new forensic data structure")
-        
-        self.data.start_time = datetime.now()
-        self.data.scan_config = {
-            'target': target,
-            'max_depth': self.config.max_depth,
-            'dns_servers': self.config.DNS_SERVERS,
-            'shodan_enabled': self.shodan_client is not None,
-            'virustotal_enabled': self.virustotal_client is not None
-        }
-        
-        logger.info(f"🎯 Starting forensic reconnaissance for target: {target}")
-        
-        try:
-            # Determine operation mode and create initial operation
-            if '.' in target:
-                self.is_hostname_only_mode = False
-                logger.info(f"🔍 Full domain mode: {target}")
-                self._create_initial_operation(target, OperationType.INITIAL_TARGET)
-            else:
-                self.is_hostname_only_mode = True
-                logger.info(f"🔍 Hostname expansion mode: {target}")
-                self._perform_tld_expansion(target)
-            
-            # Process all discovery operations
-            self._process_discovery_queue()
-            
-            # Perform external lookups for full domain mode
-            if not self.is_hostname_only_mode:
-                self._perform_external_lookups()
-            
-            # Finalize
-            self.data.end_time = datetime.now()
-            duration = self.data.end_time - self.data.start_time
-            
-            stats = self.data.get_stats()
-            logger.info(f"🏁 Forensic reconnaissance complete in {duration}")
-            logger.info(f"📊 Final stats: {stats}")
-            
-            self._update_progress("Forensic reconnaissance complete", 100)
-            
-        except Exception as e:
-            logger.error(f"❌ Error during forensic reconnaissance: {e}", exc_info=True)
-            raise
-        
-        return self.data
-    
-    def _create_initial_operation(self, target: str, operation_type: OperationType):
-        """Create initial operation for the target."""
-        operation = DiscoveryOperation(
-            operation_type=operation_type,
-            target=target,
-            discovered_hostnames=[target]
-        )
-        
-        self.data.add_operation(operation)
-        
-        # Create initial node
-        node = self.data.add_node(
-            hostname=target,
-            depth=0,
-            discovery_method=DiscoveryMethod.INITIAL_TARGET,
-            operation_id=operation.operation_id
-        )
-        
-        # Queue initial DNS operations
-        self._queue_dns_operations_for_hostname(target, 0)
-        
-        logger.info(f"🔍 Created initial operation for {target}")
-    
-    def _perform_tld_expansion(self, hostname: str):
-        """Perform TLD expansion with forensic tracking."""
-        logger.info(f"🌐 Starting TLD expansion for: {hostname}")
-        
-        # Create TLD expansion operation
-        operation = DiscoveryOperation(
-            operation_type=OperationType.TLD_EXPANSION,
-            target=hostname,
-            metadata={'expansion_type': 'smart_prioritized'}
-        )
-        
-        self._update_progress(f"Expanding {hostname} to all TLDs", 5)
-        
-        # Get prioritized TLD lists
-        priority_tlds, normal_tlds, deprioritized_tlds = self.tld_fetcher.get_prioritized_tlds()
-        
-        valid_domains = set()
-        
-        # Phase 1: Priority TLDs
-        logger.info("🎯 Phase 1: Checking priority TLDs...")
-        priority_results = self._check_tlds_parallel(hostname, priority_tlds, operation.operation_id)
-        valid_domains.update(priority_results)
-        
-        # Phase 2: Normal TLDs (if needed)
-        if len(valid_domains) < 5:
-            logger.info("🔍 Phase 2: Checking normal TLDs...")
-            normal_results = self._check_tlds_parallel(hostname, normal_tlds, operation.operation_id)
-            valid_domains.update(normal_results)
-        
-        # Phase 3: Deprioritized TLDs (if really needed)
-        if len(valid_domains) < 2:
-            logger.info("🔎 Phase 3: Checking deprioritized TLDs...")
-            depri_results = self._check_tlds_parallel(hostname, deprioritized_tlds, operation.operation_id)
-            valid_domains.update(depri_results)
-        
-        # Update operation with results
-        operation.discovered_hostnames = list(valid_domains)
-        operation.success = len(valid_domains) > 0
-        self.data.add_operation(operation)
-        
-        # Create nodes for all discovered domains
-        for domain in valid_domains:
-            node = self.data.add_node(
-                hostname=domain,
-                depth=0,
-                discovery_method=DiscoveryMethod.TLD_EXPANSION,
-                operation_id=operation.operation_id
-            )
-            
-            # Add discovery edge (synthetic source for TLD expansion)
-            self.data.add_edge(
-                source=f"tld_expansion:{hostname}",
-                target=domain,
-                discovery_method=DiscoveryMethod.TLD_EXPANSION,
-                operation_id=operation.operation_id,
-                metadata={'original_hostname': hostname}
-            )
-        
-        logger.info(f"✅ TLD expansion complete: {len(valid_domains)} domains found")
-        
-        # Queue lightweight operations for hostname-only mode
-        for domain in valid_domains:
-            self._queue_lightweight_operations(domain)
-    
-    def _check_tlds_parallel(self, hostname: str, tlds: List[str], operation_id: str) -> Set[str]:
-        """Check TLDs in parallel with operation tracking."""
-        valid_domains = set()
-        max_workers = min(20, len(tlds))
-        
-        with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
-            future_to_tld = {
-                executor.submit(self._check_single_tld_forensic, hostname, tld, operation_id): tld 
-                for tld in tlds
-            }
-            
-            for future in concurrent.futures.as_completed(future_to_tld):
-                tld = future_to_tld[future]
-                try:
-                    result = future.result(timeout=10)
-                    if result:
-                        domain, ips = result
-                        valid_domains.add(domain)
-                        
-                        # Track discovered IPs
-                        for ip in ips:
-                            self.data.ip_addresses.add(ip)
-                        
-                except Exception as e:
-                    logger.debug(f"Error checking {hostname}.{tld}: {e}")
-        
-        return valid_domains
-    
-    def _check_single_tld_forensic(self, hostname: str, tld: str, operation_id: str) -> Optional[Tuple[str, List[str]]]:
-        """Check single TLD with forensic tracking."""
-        full_hostname = f"{hostname}.{tld}"
-        
-        # Quick DNS resolution
-        ips = self.dns_resolver.resolve_hostname_fast(full_hostname)
-        
-        if ips:
-            # Create DNS operation record
-            dns_operation = DiscoveryOperation(
-                operation_type=OperationType.DNS_A,
-                target=full_hostname,
-                discovered_ips=ips,
-                metadata={'tld_expansion': True, 'parent_operation': operation_id}
-            )
-            
-            # Add DNS records
-            for ip in ips:
-                record = DNSRecord(
-                    record_type='A',
-                    value=ip,
-                    operation_id=dns_operation.operation_id
-                )
-                dns_operation.dns_records.append(record)
-            
-            self.data.add_operation(dns_operation)
-            return (full_hostname, ips)
-        
-        return None
-    
-    def _queue_dns_operations_for_hostname(self, hostname: str, depth: int):
-        """Queue comprehensive DNS operations for a hostname."""
-        # Map DNS record types to operation types
-        dns_operations = [
-            (OperationType.DNS_A, 'A'),
-            (OperationType.DNS_AAAA, 'AAAA'),
-            (OperationType.DNS_MX, 'MX'),
-            (OperationType.DNS_NS, 'NS'),
-            (OperationType.DNS_TXT, 'TXT'),
-            (OperationType.DNS_CNAME, 'CNAME'),
-            (OperationType.DNS_SOA, 'SOA'),
-            (OperationType.DNS_SRV, 'SRV'),
-            (OperationType.DNS_CAA, 'CAA'),
-        ]
-        
-        for operation_type, record_type in dns_operations:
-            self.pending_operations.append({
-                'type': 'dns_query',
-                'operation_type': operation_type,
-                'record_type': record_type,
-                'hostname': hostname,
-                'depth': depth
-            })
-        
-        # Queue certificate operation
-        self.pending_operations.append({
-            'type': 'certificate_check',
-            'operation_type': OperationType.CERTIFICATE_CHECK,
-            'hostname': hostname,
-            'depth': depth
-        })
-        
-        logger.debug(f"📋 Queued {len(dns_operations) + 1} operations for {hostname}")
-    
-    def _queue_lightweight_operations(self, hostname: str):
-        """Queue lightweight operations for hostname-only mode."""
-        # Only essential DNS operations
-        essential_operations = [
-            (OperationType.DNS_A, 'A'),
-            (OperationType.DNS_AAAA, 'AAAA'),
-            (OperationType.DNS_MX, 'MX'),
-            (OperationType.DNS_TXT, 'TXT'),
-        ]
-        
-        for operation_type, record_type in essential_operations:
-            self.pending_operations.append({
-                'type': 'dns_query',
-                'operation_type': operation_type,
-                'record_type': record_type,
-                'hostname': hostname,
-                'depth': 0
-            })
-        
-        logger.debug(f"📋 Queued {len(essential_operations)} lightweight operations for {hostname}")
-    
-    def _process_discovery_queue(self):
-        """Process all queued discovery operations."""
-        operation_count = 0
-        total_operations = len(self.pending_operations)
-        
-        logger.info(f"⚙️ Processing {total_operations} operations")
-        
-        while self.pending_operations:
-            operation_spec = self.pending_operations.pop(0)
-            operation_count += 1
-            
-            # Calculate progress
-            progress = 20 + (operation_count * 60 // max(total_operations, 1))
-            
-            try:
-                if operation_spec['type'] == 'dns_query':
-                    self._execute_dns_operation(operation_spec)
-                elif operation_spec['type'] == 'certificate_check':
-                    self._execute_certificate_operation(operation_spec)
-                
-                # Update progress periodically
-                if operation_count % 10 == 0:
-                    self._update_progress(f"Processed {operation_count}/{total_operations} operations", progress)
-                    
-            except Exception as e:
-                logger.error(f"❌ Error processing operation {operation_spec}: {e}")
-                continue
-        
-        logger.info(f"✅ Completed processing {operation_count} operations")
-    
-    def _execute_dns_operation(self, operation_spec: Dict[str, Any]):
-        """Execute a single DNS operation with full tracking."""
-        hostname = operation_spec['hostname']
-        record_type = operation_spec['record_type']
-        operation_type = operation_spec['operation_type']
-        depth = operation_spec['depth']
-        
-        logger.debug(f"🔍 DNS {record_type} query for {hostname}")
-        
-        # Create operation
-        operation = DiscoveryOperation(
-            operation_type=operation_type,
-            target=hostname,
-            metadata={'record_type': record_type, 'depth': depth}
-        )
-        
-        try:
-            # Perform DNS query using the updated method with operation_id
-            records = self.dns_resolver.query_specific_record_type(hostname, record_type, operation.operation_id)
-            operation.dns_records = records
-            operation.success = len(records) > 0
-            
-            # Process results
-            node = self.data.get_node(hostname)
-            if not node:
-                node = self.data.add_node(hostname, depth)
-            
-            new_hostnames = set()
-            new_ips = set()
-            
-            for record in records:
-                node.add_dns_record(record)
-                
-                # Extract IPs and hostnames
-                if record.record_type in ['A', 'AAAA']:
-                    new_ips.add(record.value)
-                    self.data.ip_addresses.add(record.value)
-                elif record.record_type in ['MX', 'NS', 'CNAME']:
-                    # Extract hostname from MX (format: "priority hostname")
-                    if record.record_type == 'MX':
-                        parts = record.value.split()
-                        if len(parts) >= 2:
-                            extracted_hostname = parts[-1].rstrip('.')
-                        else:
-                            continue
-                    else:
-                        extracted_hostname = record.value.rstrip('.')
-                    
-                    if self._is_valid_hostname(extracted_hostname) and extracted_hostname != hostname:
-                        new_hostnames.add(extracted_hostname)
-            
-            # Update operation results
-            operation.discovered_hostnames = list(new_hostnames)
-            operation.discovered_ips = list(new_ips)
-            
-            # Add discovery edges and queue new operations
-            for new_hostname in new_hostnames:
-                # Add node
-                new_node = self.data.add_node(
-                    hostname=new_hostname,
-                    depth=depth + 1,
-                    discovery_method=DiscoveryMethod.DNS_RECORD_VALUE,
-                    operation_id=operation.operation_id
-                )
-                
-                # Add edge
-                self.data.add_edge(
-                    source=hostname,
-                    target=new_hostname,
-                    discovery_method=DiscoveryMethod.DNS_RECORD_VALUE,
-                    operation_id=operation.operation_id,
-                    metadata={'record_type': record_type}
-                )
-                
-                # Queue operations for new hostname (if not in hostname-only mode and within depth)
-                if not self.is_hostname_only_mode and depth + 1 <= self.config.max_depth:
-                    self._queue_dns_operations_for_hostname(new_hostname, depth + 1)
-            
-            logger.debug(f"✅ DNS {record_type} for {hostname}: {len(records)} records, {len(new_hostnames)} new hostnames")
-            
-        except Exception as e:
-            operation.success = False
-            operation.error_message = str(e)
-            logger.debug(f"❌ DNS {record_type} query failed for {hostname}: {e}")
-        
-        self.data.add_operation(operation)
-    
-    def _execute_certificate_operation(self, operation_spec: Dict[str, Any]):
-        """Execute certificate check operation."""
-        hostname = operation_spec['hostname']
-        depth = operation_spec['depth']
-        
-        # Skip certificates in hostname-only mode
-        if self.is_hostname_only_mode:
-            logger.debug(f"⭐ Skipping certificate check for {hostname} (hostname-only mode)")
-            return
-        
-        logger.debug(f"🔍 Certificate check for {hostname}")
-        
-        operation = DiscoveryOperation(
-            operation_type=OperationType.CERTIFICATE_CHECK,
-            target=hostname,
-            metadata={'depth': depth}
-        )
-        
-        try:
-            # Use updated method with operation_id
-            certificates = self.cert_checker.get_certificates(hostname, operation.operation_id)
-            operation.certificates = certificates
-            operation.success = len(certificates) > 0
-            
-            # Process certificates
-            node = self.data.get_node(hostname)
-            if node:
-                new_hostnames = set()
-                
-                for cert in certificates:
-                    node.add_certificate(cert)
-                    
-                    # Extract hostnames from certificate subjects
-                    subject_hostnames = self.cert_checker.extract_subdomains_from_certificates([cert])
-                    for subject_hostname in subject_hostnames:
-                        if subject_hostname != hostname and self._is_valid_hostname(subject_hostname):
-                            new_hostnames.add(subject_hostname)
-                
-                # Update operation and create edges
-                operation.discovered_hostnames = list(new_hostnames)
-                
-                for new_hostname in new_hostnames:
-                    # Add node
-                    new_node = self.data.add_node(
-                        hostname=new_hostname,
-                        depth=depth + 1,
-                        discovery_method=DiscoveryMethod.CERTIFICATE_SUBJECT,
-                        operation_id=operation.operation_id
-                    )
-                    
-                    # Add edge
-                    self.data.add_edge(
-                        source=hostname,
-                        target=new_hostname,
-                        discovery_method=DiscoveryMethod.CERTIFICATE_SUBJECT,
-                        operation_id=operation.operation_id
-                    )
-                    
-                    # Queue operations for new hostname (within depth limit)
-                    if depth + 1 <= self.config.max_depth:
-                        self._queue_dns_operations_for_hostname(new_hostname, depth + 1)
-                
-                logger.debug(f"✅ Certificates for {hostname}: {len(certificates)} certs, {len(new_hostnames)} new hostnames")
-            
-        except Exception as e:
-            operation.success = False
-            operation.error_message = str(e)
-            logger.debug(f"❌ Certificate check failed for {hostname}: {e}")
-        
-        self.data.add_operation(operation)
-    
-    def _perform_external_lookups(self):
-        """Perform external service lookups with operation tracking."""
-        if self.is_hostname_only_mode:
-            logger.info("⭐ Skipping external lookups (hostname-only mode)")
-            return
-        
-        self._update_progress("Performing external service lookups", 85)
-        
-        # Reverse DNS
-        self._perform_reverse_dns_lookups()
-        
-        # Shodan lookups
-        if self.shodan_client:
-            self._perform_shodan_lookups()
-        
-        # VirusTotal lookups
-        if self.virustotal_client:
-            self._perform_virustotal_lookups()
-    
-    def _perform_reverse_dns_lookups(self):
-        """Perform reverse DNS lookups with operation tracking."""
-        logger.info(f"🔄 Performing reverse DNS for {len(self.data.ip_addresses)} IPs")
-        
-        for ip in self.data.ip_addresses:
-            operation = DiscoveryOperation(
-                operation_type=OperationType.DNS_REVERSE,
-                target=ip
-            )
-            
-            try:
-                # Use updated method with operation_id
-                reverse_hostname = self.dns_resolver.reverse_dns_lookup(ip, operation.operation_id)
-                if reverse_hostname:
-                    operation.discovered_hostnames = [reverse_hostname]
-                    operation.success = True
-                    
-                    # Update nodes that have this IP
-                    for node in self.data.nodes.values():
-                        if ip in node.resolved_ips:
-                            node.reverse_dns = reverse_hostname
-                    
-                    logger.debug(f"🔄 Reverse DNS {ip} -> {reverse_hostname}")
-                else:
-                    operation.success = False
-                    
-            except Exception as e:
-                operation.success = False
-                operation.error_message = str(e)
-            
-            self.data.add_operation(operation)
-    
-    def _perform_shodan_lookups(self):
-        """Perform Shodan lookups with operation tracking."""
-        logger.info(f"🕵️ Performing Shodan lookups for {len(self.data.ip_addresses)} IPs")
-        
-        for ip in self.data.ip_addresses:
-            operation = DiscoveryOperation(
-                operation_type=OperationType.SHODAN_LOOKUP,
-                target=ip
-            )
-            
-            try:
-                # Use updated method with operation_id
-                result = self.shodan_client.lookup_ip(ip, operation.operation_id)
-                if result:
-                    operation.shodan_results = [result]
-                    operation.success = True
-                    
-                    # Add to relevant nodes
-                    for node in self.data.nodes.values():
-                        if ip in node.resolved_ips:
-                            node.shodan_results.append(result)
-                    
-                    logger.debug(f"🕵️ Shodan {ip}: {len(result.ports)} ports")
-                else:
-                    operation.success = False
-                    
-            except Exception as e:
-                operation.success = False
-                operation.error_message = str(e)
-            
-            self.data.add_operation(operation)
-    
-    def _perform_virustotal_lookups(self):
-        """Perform VirusTotal lookups with operation tracking."""
-        total_resources = len(self.data.ip_addresses) + len(self.data.nodes)
-        logger.info(f"🛡️ Performing VirusTotal lookups for {total_resources} resources")
-        
-        # Check IPs
-        for ip in self.data.ip_addresses:
-            operation = DiscoveryOperation(
-                operation_type=OperationType.VIRUSTOTAL_IP,
-                target=ip
-            )
-            
-            try:
-                # Use updated method with operation_id
-                result = self.virustotal_client.lookup_ip(ip, operation.operation_id)
-                if result:
-                    operation.virustotal_results = [result]
-                    operation.success = True
-                    
-                    # Add to relevant nodes
-                    for node in self.data.nodes.values():
-                        if ip in node.resolved_ips:
-                            node.virustotal_results.append(result)
-                    
-                    logger.debug(f"🛡️ VirusTotal {ip}: {result.positives}/{result.total}")
-                else:
-                    operation.success = False
-                    
-            except Exception as e:
-                operation.success = False
-                operation.error_message = str(e)
-            
-            self.data.add_operation(operation)
-        
-        # Check domains
-        for hostname, node in self.data.nodes.items():
-            operation = DiscoveryOperation(
-                operation_type=OperationType.VIRUSTOTAL_DOMAIN,
-                target=hostname
-            )
-            
-            try:
-                # Use updated method with operation_id
-                result = self.virustotal_client.lookup_domain(hostname, operation.operation_id)
-                if result:
-                    operation.virustotal_results = [result]
-                    operation.success = True
-                    node.virustotal_results.append(result)
-                    
-                    logger.debug(f"🛡️ VirusTotal {hostname}: {result.positives}/{result.total}")
-                else:
-                    operation.success = False
-                    
-            except Exception as e:
-                operation.success = False
-                operation.error_message = str(e)
-            
-            self.data.add_operation(operation)
-    
-    def _is_valid_hostname(self, hostname: str) -> bool:
-        """Validate hostname format."""
-        if not hostname or '.' not in hostname or len(hostname) > 255:
-            return False
-        
-        # Basic validation
-        parts = hostname.split('.')
-        if len(parts) < 2:
-            return False
-        
-        for part in parts:
-            if not part or len(part) > 63:
-                return False
-        
-        return True
-
-# Backward compatibility
-ReconnaissanceEngine = ForensicReconnaissanceEngine

src/report_generator.py

@@ -1,451 +0,0 @@
-# File: src/report_generator.py
-"""Enhanced report generation with forensic details and discovery graph visualization."""
-
-from datetime import datetime
-from typing import Dict, Any, List, Set
-from .data_structures import ForensicReconData, DiscoveryMethod, OperationType
-import logging
-
-logger = logging.getLogger(__name__)
-
-class ForensicReportGenerator:
-    """Generate comprehensive forensic reports with discovery provenance."""
-    
-    def __init__(self, data: ForensicReconData):
-        self.data = data
-    
-    def generate_text_report(self) -> str:
-        """Generate comprehensive forensic text report."""
-        report = []
-        
-        # Header
-        report.append("=" * 80)
-        report.append("FORENSIC DNS RECONNAISSANCE REPORT")
-        report.append("=" * 80)
-        report.append(f"Scan Start: {self.data.start_time}")
-        if self.data.end_time:
-            report.append(f"Scan End: {self.data.end_time}")
-            duration = self.data.end_time - self.data.start_time
-            report.append(f"Duration: {duration}")
-        report.append(f"Target: {self.data.scan_config.get('target', 'Unknown')}")
-        report.append(f"Max Depth: {self.data.scan_config.get('max_depth', 'Unknown')}")
-        report.append("")
-        
-        # Executive Summary
-        report.append("EXECUTIVE SUMMARY")
-        report.append("-" * 40)
-        stats = self.data.get_stats()
-        report.append(f"Discovered Hostnames: {stats['hostnames']}")
-        report.append(f"IP Addresses Found: {stats['ip_addresses']}")
-        report.append(f"Operations Performed: {stats['operations_performed']}")
-        report.append(f"Discovery Relationships: {stats['discovery_edges']}")
-        report.append(f"DNS Records Collected: {stats['dns_records']}")
-        report.append(f"Total Certificates: {stats['certificates_total']}")
-        report.append(f"  └─ Currently Valid: {stats['certificates_current']}")
-        report.append(f"  └─ Expired: {stats['certificates_expired']}")
-        report.append(f"Shodan Results: {stats['shodan_results']}")
-        report.append(f"VirusTotal Results: {stats['virustotal_results']}")
-        report.append("")
-        
-        # Discovery Graph Analysis
-        graph_analysis = self.data._generate_graph_analysis()
-        report.append("DISCOVERY GRAPH ANALYSIS")
-        report.append("-" * 40)
-        report.append(f"Maximum Discovery Depth: {graph_analysis['max_depth']}")
-        report.append(f"Root Nodes (Initial Targets): {len(graph_analysis['root_nodes'])}")
-        report.append(f"Leaf Nodes (No Further Discoveries): {len(graph_analysis['leaf_nodes'])}")
-        report.append("")
-        
-        # Depth Distribution
-        report.append("Discovery Depth Distribution:")
-        for depth, count in sorted(graph_analysis['depth_distribution'].items()):
-            report.append(f"  Depth {depth}: {count} hostnames")
-        report.append("")
-        
-        # Discovery Methods Distribution
-        report.append("Discovery Methods Used:")
-        for method, count in sorted(graph_analysis['discovery_method_distribution'].items()):
-            report.append(f"  {method}: {count} discoveries")
-        report.append("")
-        
-        # Discovery Tree
-        report.append("DISCOVERY TREE")
-        report.append("-" * 40)
-        report.extend(self._generate_discovery_tree())
-        report.append("")
-        
-        # Detailed Node Analysis
-        report.append("DETAILED NODE ANALYSIS")
-        report.append("-" * 40)
-        report.extend(self._generate_node_details())
-        report.append("")
-        
-        # Operations Timeline
-        report.append("OPERATIONS TIMELINE")
-        report.append("-" * 40)
-        report.extend(self._generate_operations_timeline())
-        report.append("")
-        
-        # Security Analysis
-        security_findings = self._analyze_security_findings()
-        if security_findings:
-            report.append("SECURITY ANALYSIS")
-            report.append("-" * 40)
-            report.extend(security_findings)
-            report.append("")
-        
-        # Certificate Analysis
-        cert_analysis = self._analyze_certificates()
-        if cert_analysis:
-            report.append("CERTIFICATE ANALYSIS")
-            report.append("-" * 40)
-            report.extend(cert_analysis)
-            report.append("")
-        
-        # DNS Record Analysis
-        report.append("DNS RECORD ANALYSIS")
-        report.append("-" * 40)
-        report.extend(self._analyze_dns_records())
-        report.append("")
-        
-        return "\n".join(report)
-    
-    def _generate_discovery_tree(self) -> List[str]:
-        """Generate a tree view of hostname discoveries."""
-        tree_lines = []
-        
-        # Find root nodes
-        graph_analysis = self.data._generate_graph_analysis()
-        root_nodes = graph_analysis['root_nodes']
-        
-        if not root_nodes:
-            tree_lines.append("No root nodes found")
-            return tree_lines
-        
-        # Generate tree for each root
-        for root in sorted(root_nodes):
-            tree_lines.extend(self._build_tree_branch(root, "", set()))
-        
-        return tree_lines
-    
-    def _build_tree_branch(self, hostname: str, prefix: str, visited: Set[str]) -> List[str]:
-        """Build a tree branch for a hostname."""
-        lines = []
-        
-        # Avoid cycles
-        if hostname in visited:
-            lines.append(f"{prefix}{hostname} [CYCLE]")
-            return lines
-        
-        visited.add(hostname)
-        
-        # Get node info
-        node = self.data.get_node(hostname)
-        if not node:
-            lines.append(f"{prefix}{hostname} [NO NODE DATA]")
-            return lines
-        
-        # Node info
-        node_info = f"{hostname} (depth:{node.depth}"
-        if node.resolved_ips:
-            node_info += f", IPs:{len(node.resolved_ips)}"
-        if node.certificates:
-            valid_certs = len(node.get_current_certificates())
-            expired_certs = len(node.get_expired_certificates())
-            node_info += f", certs:{valid_certs}+{expired_certs}"
-        node_info += ")"
-        
-        lines.append(f"{prefix}{node_info}")
-        
-        # Get children
-        children = self.data.get_children(hostname)
-        children.sort()
-        
-        for i, child in enumerate(children):
-            is_last = (i == len(children) - 1)
-            child_prefix = prefix + ("└── " if is_last else "├── ")
-            next_prefix = prefix + ("    " if is_last else "│   ")
-            
-            # Find discovery method for this child
-            discovery_method = "unknown"
-            for edge in self.data.edges:
-                if edge.source_hostname == hostname and edge.target_hostname == child:
-                    discovery_method = edge.discovery_method.value
-                    break
-            
-            lines.append(f"{child_prefix}[{discovery_method}]")
-            lines.extend(self._build_tree_branch(child, next_prefix, visited.copy()))
-        
-        return lines
-    
-    def _generate_node_details(self) -> List[str]:
-        """Generate detailed analysis of each node."""
-        details = []
-        
-        # Sort nodes by depth, then alphabetically
-        sorted_nodes = sorted(self.data.nodes.items(), 
-                            key=lambda x: (x[1].depth, x[0]))
-        
-        for hostname, node in sorted_nodes:
-            details.append(f"\n{hostname} (Depth {node.depth})")
-            details.append("-" * (len(hostname) + 20))
-            
-            # Discovery provenance
-            details.append(f"First Seen: {node.first_seen}")
-            details.append(f"Last Updated: {node.last_updated}")
-            details.append(f"Discovery Methods: {', '.join(m.value for m in node.discovery_methods)}")
-            
-            # Discovery paths
-            paths = self.data.get_discovery_path(hostname)
-            if paths:
-                details.append("Discovery Paths:")
-                for i, path in enumerate(paths[:3]):  # Show max 3 paths
-                    path_str = " -> ".join([f"{src}[{method.value}]{tgt}" for src, tgt, method in path])
-                    details.append(f"  Path {i+1}: {path_str}")
-                if len(paths) > 3:
-                    details.append(f"  ... and {len(paths) - 3} more paths")
-            
-            # DNS status
-            if node.dns_exists is not None:
-                status = "EXISTS" if node.dns_exists else "NOT FOUND"
-                details.append(f"DNS Status: {status} (checked: {node.last_dns_check})")
-            
-            # IP addresses
-            if node.resolved_ips:
-                details.append(f"Resolved IPs: {', '.join(sorted(node.resolved_ips))}")
-            
-            # Reverse DNS
-            if node.reverse_dns:
-                details.append(f"Reverse DNS: {node.reverse_dns}")
-            
-            # DNS records summary
-            total_records = len(node.get_all_dns_records())
-            if total_records > 0:
-                record_types = list(node.dns_records_by_type.keys())
-                details.append(f"DNS Records: {total_records} records ({', '.join(sorted(record_types))})")
-            
-            # Certificates summary
-            current_certs = len(node.get_current_certificates())
-            expired_certs = len(node.get_expired_certificates())
-            if current_certs > 0 or expired_certs > 0:
-                details.append(f"Certificates: {current_certs} valid, {expired_certs} expired")
-            
-            # External results
-            if node.shodan_results:
-                details.append(f"Shodan: {len(node.shodan_results)} results")
-            if node.virustotal_results:
-                vt_detections = sum(r.positives for r in node.virustotal_results)
-                details.append(f"VirusTotal: {len(node.virustotal_results)} scans, {vt_detections} total detections")
-        
-        return details
-    
-    def _generate_operations_timeline(self) -> List[str]:
-        """Generate operations timeline."""
-        timeline = []
-        
-        # Sort operations by timestamp
-        sorted_ops = []
-        for op_id in self.data.operation_timeline:
-            if op_id in self.data.operations:
-                sorted_ops.append(self.data.operations[op_id])
-        
-        # Group operations by type for summary
-        op_summary = {}
-        for op in sorted_ops:
-            op_type = op.operation_type.value
-            if op_type not in op_summary:
-                op_summary[op_type] = {'total': 0, 'successful': 0, 'failed': 0}
-            op_summary[op_type]['total'] += 1
-            if op.success:
-                op_summary[op_type]['successful'] += 1
-            else:
-                op_summary[op_type]['failed'] += 1
-        
-        # Operations summary
-        timeline.append("Operations Summary:")
-        for op_type, counts in sorted(op_summary.items()):
-            timeline.append(f"  {op_type}: {counts['successful']}/{counts['total']} successful")
-        timeline.append("")
-        
-        # Recent operations (last 20)
-        timeline.append("Recent Operations (last 20):")
-        recent_ops = sorted_ops[-20:] if len(sorted_ops) > 20 else sorted_ops
-        
-        for op in recent_ops:
-            timestamp = op.timestamp.strftime("%H:%M:%S.%f")[:-3]
-            status = "✓" if op.success else "✗"
-            target_short = op.target[:30] + "..." if len(op.target) > 30 else op.target
-            
-            timeline.append(f"  {timestamp} {status} {op.operation_type.value:15} {target_short}")
-            
-            # Show key results
-            if op.discovered_hostnames:
-                hostname_list = ", ".join(op.discovered_hostnames[:3])
-                if len(op.discovered_hostnames) > 3:
-                    hostname_list += f" (+{len(op.discovered_hostnames) - 3} more)"
-                timeline.append(f"    └─ Discovered: {hostname_list}")
-            
-            if op.error_message:
-                timeline.append(f"    └─ Error: {op.error_message[:50]}...")
-        
-        return timeline
-    
-    def _analyze_security_findings(self) -> List[str]:
-        """Analyze security-related findings."""
-        findings = []
-        
-        # VirusTotal detections
-        high_risk_resources = []
-        medium_risk_resources = []
-        
-        for node in self.data.nodes.values():
-            for vt_result in node.virustotal_results:
-                if vt_result.positives > 5:
-                    high_risk_resources.append((node.hostname, vt_result))
-                elif vt_result.positives > 0:
-                    medium_risk_resources.append((node.hostname, vt_result))
-        
-        if high_risk_resources:
-            findings.append("🚨 HIGH RISK FINDINGS:")
-            for hostname, vt_result in high_risk_resources:
-                findings.append(f"  {hostname}: {vt_result.positives}/{vt_result.total} detections")
-                findings.append(f"    Report: {vt_result.permalink}")
-        
-        if medium_risk_resources:
-            findings.append("⚠️  MEDIUM RISK FINDINGS:")
-            for hostname, vt_result in medium_risk_resources[:5]:  # Show max 5
-                findings.append(f"  {hostname}: {vt_result.positives}/{vt_result.total} detections")
-            if len(medium_risk_resources) > 5:
-                findings.append(f"  ... and {len(medium_risk_resources) - 5} more resources with detections")
-        
-        # Expired certificates still in use
-        nodes_with_expired_certs = []
-        for hostname, node in self.data.nodes.items():
-            expired = node.get_expired_certificates()
-            current = node.get_current_certificates()
-            if expired and not current:  # Only expired certs, no valid ones
-                nodes_with_expired_certs.append((hostname, len(expired)))
-        
-        if nodes_with_expired_certs:
-            findings.append("📜 CERTIFICATE ISSUES:")
-            for hostname, count in nodes_with_expired_certs:
-                findings.append(f"  {hostname}: {count} expired certificates, no valid ones")
-        
-        return findings
-    
-    def _analyze_certificates(self) -> List[str]:
-        """Analyze certificate findings."""
-        cert_analysis = []
-        
-        # Certificate statistics
-        total_certs = 0
-        valid_certs = 0
-        expired_certs = 0
-        wildcard_certs = 0
-        
-        cert_authorities = {}
-        
-        for node in self.data.nodes.values():
-            for cert in node.certificates:
-                total_certs += 1
-                if cert.is_valid_now:
-                    valid_certs += 1
-                else:
-                    expired_certs += 1
-                
-                if cert.is_wildcard:
-                    wildcard_certs += 1
-                
-                # Count certificate authorities
-                issuer_short = cert.issuer.split(',')[0] if ',' in cert.issuer else cert.issuer
-                cert_authorities[issuer_short] = cert_authorities.get(issuer_short, 0) + 1
-        
-        if total_certs == 0:
-            cert_analysis.append("No certificates found.")
-            return cert_analysis
-        
-        cert_analysis.append(f"Total Certificates: {total_certs}")
-        cert_analysis.append(f"  Currently Valid: {valid_certs}")
-        cert_analysis.append(f"  Expired: {expired_certs}")
-        cert_analysis.append(f"  Wildcard Certificates: {wildcard_certs}")
-        cert_analysis.append("")
-        
-        # Top certificate authorities
-        cert_analysis.append("Certificate Authorities:")
-        sorted_cas = sorted(cert_authorities.items(), key=lambda x: x[1], reverse=True)
-        for ca, count in sorted_cas[:5]:
-            cert_analysis.append(f"  {ca}: {count} certificates")
-        cert_analysis.append("")
-        
-        # Expiring soon (within 30 days)
-        from datetime import timedelta
-        soon = datetime.now() + timedelta(days=30)
-        expiring_soon = []
-        
-        for hostname, node in self.data.nodes.items():
-            for cert in node.get_current_certificates():
-                if cert.not_after <= soon:
-                    expiring_soon.append((hostname, cert.not_after, cert.id))
-        
-        if expiring_soon:
-            cert_analysis.append("Certificates Expiring Soon (within 30 days):")
-            for hostname, expiry, cert_id in sorted(expiring_soon, key=lambda x: x[1]):
-                cert_analysis.append(f"  {hostname}: expires {expiry.strftime('%Y-%m-%d')} (cert ID: {cert_id})")
-        
-        return cert_analysis
-    
-    def _analyze_dns_records(self) -> List[str]:
-        """Analyze DNS record patterns."""
-        dns_analysis = []
-        
-        # Record type distribution
-        record_type_counts = {}
-        total_records = 0
-        
-        for node in self.data.nodes.values():
-            for record_type, records in node.dns_records_by_type.items():
-                record_type_counts[record_type] = record_type_counts.get(record_type, 0) + len(records)
-                total_records += len(records)
-        
-        dns_analysis.append(f"Total DNS Records: {total_records}")
-        dns_analysis.append("Record Type Distribution:")
-        
-        for record_type, count in sorted(record_type_counts.items()):
-            percentage = (count / total_records * 100) if total_records > 0 else 0
-            dns_analysis.append(f"  {record_type}: {count} ({percentage:.1f}%)")
-        dns_analysis.append("")
-        
-        # Interesting findings
-        interesting = []
-        
-        # Multiple MX records
-        multi_mx_nodes = []
-        for hostname, node in self.data.nodes.items():
-            mx_records = node.dns_records_by_type.get('MX', [])
-            if len(mx_records) > 1:
-                multi_mx_nodes.append((hostname, len(mx_records)))
-        
-        if multi_mx_nodes:
-            interesting.append("Multiple MX Records:")
-            for hostname, count in multi_mx_nodes:
-                interesting.append(f"  {hostname}: {count} MX records")
-        
-        # CAA records (security-relevant)
-        caa_nodes = []
-        for hostname, node in self.data.nodes.items():
-            if 'CAA' in node.dns_records_by_type:
-                caa_nodes.append(hostname)
-        
-        if caa_nodes:
-            interesting.append(f"Domains with CAA Records: {len(caa_nodes)}")
-            for hostname in caa_nodes[:5]:  # Show first 5
-                interesting.append(f"  {hostname}")
-        
-        if interesting:
-            dns_analysis.append("Interesting DNS Findings:")
-            dns_analysis.extend(interesting)
-        
-        return dns_analysis
-
-# Backward compatibility
-ReportGenerator = ForensicReportGenerator

src/shodan_client.py

@@ -1,177 +0,0 @@
-# File: src/shodan_client.py
-"""Shodan API integration with forensic operation tracking."""
-
-import requests
-import time
-import logging
-import uuid
-from typing import Optional, Dict, Any, List
-from .data_structures import ShodanResult
-from .config import Config
-
-# Module logger
-logger = logging.getLogger(__name__)
-
-class ShodanClient:
-    """Shodan API client with forensic tracking."""
-    
-    BASE_URL = "https://api.shodan.io"
-    
-    def __init__(self, api_key: str, config: Config):
-        self.api_key = api_key
-        self.config = config
-        self.last_request = 0
-        
-        logger.info(f"🕵️ Shodan client initialized with API key ending in: ...{api_key[-4:] if len(api_key) > 4 else api_key}")
-    
-    def _rate_limit(self):
-        """Apply rate limiting for Shodan."""
-        now = time.time()
-        time_since_last = now - self.last_request
-        min_interval = 1.0 / self.config.SHODAN_RATE_LIMIT
-        
-        if time_since_last < min_interval:
-            sleep_time = min_interval - time_since_last
-            logger.debug(f"⏸️ Shodan rate limiting: sleeping for {sleep_time:.2f}s")
-            time.sleep(sleep_time)
-        
-        self.last_request = time.time()
-    
-    def lookup_ip(self, ip: str, operation_id: Optional[str] = None) -> Optional[ShodanResult]:
-        """Lookup IP address information with forensic tracking."""
-        self._rate_limit()
-        
-        # Generate operation ID if not provided
-        if operation_id is None:
-            operation_id = str(uuid.uuid4())
-        
-        logger.debug(f"🔍 Querying Shodan for IP: {ip} (operation: {operation_id})")
-        
-        try:
-            url = f"{self.BASE_URL}/shodan/host/{ip}"
-            params = {'key': self.api_key}
-            
-            response = requests.get(
-                url, 
-                params=params, 
-                timeout=self.config.HTTP_TIMEOUT,
-                headers={'User-Agent': 'DNS-Recon-Tool/1.0'}
-            )
-            
-            logger.debug(f"📡 Shodan API response for {ip}: {response.status_code}")
-            
-            if response.status_code == 200:
-                data = response.json()
-                
-                ports = []
-                services = {}
-                
-                for service in data.get('data', []):
-                    port = service.get('port')
-                    if port:
-                        ports.append(port)
-                        services[str(port)] = {
-                            'product': service.get('product', ''),
-                            'version': service.get('version', ''),
-                            'banner': service.get('data', '').strip()[:200] if service.get('data') else ''
-                        }
-                
-                # Create ShodanResult with forensic metadata
-                result = ShodanResult(
-                    ip=ip,
-                    ports=sorted(list(set(ports))),
-                    services=services,
-                    organization=data.get('org'),
-                    country=data.get('country_name'),
-                    operation_id=operation_id  # Forensic tracking
-                )
-                
-                logger.info(f"✅ Shodan result for {ip}: {len(result.ports)} ports, org: {result.organization}")
-                return result
-                
-            elif response.status_code == 404:
-                logger.debug(f"ℹ️ IP {ip} not found in Shodan database")
-                return None
-            elif response.status_code == 401:
-                logger.error("❌ Shodan API key is invalid or expired")
-                return None
-            elif response.status_code == 429:
-                logger.warning("⚠️ Shodan API rate limit exceeded")
-                return None
-            else:
-                logger.warning(f"⚠️ Shodan API error for {ip}: HTTP {response.status_code}")
-                try:
-                    error_data = response.json()
-                    logger.debug(f"Shodan error details: {error_data}")
-                except:
-                    pass
-                return None
-                
-        except requests.exceptions.Timeout:
-            logger.warning(f"⏱️ Shodan query timeout for {ip}")
-            return None
-        except requests.exceptions.RequestException as e:
-            logger.error(f"🌐 Shodan network error for {ip}: {e}")
-            return None
-        except Exception as e:
-            logger.error(f"❌ Unexpected error querying Shodan for {ip}: {e}")
-            return None
-    
-    def search_domain(self, domain: str, operation_id: Optional[str] = None) -> List[str]:
-        """Search for IPs associated with a domain with forensic tracking."""
-        self._rate_limit()
-        
-        # Generate operation ID if not provided
-        if operation_id is None:
-            operation_id = str(uuid.uuid4())
-        
-        logger.debug(f"🔍 Searching Shodan for domain: {domain} (operation: {operation_id})")
-        
-        try:
-            url = f"{self.BASE_URL}/shodan/host/search"
-            params = {
-                'key': self.api_key,
-                'query': f'hostname:{domain}',
-                'limit': 100
-            }
-            
-            response = requests.get(
-                url, 
-                params=params, 
-                timeout=self.config.HTTP_TIMEOUT,
-                headers={'User-Agent': 'DNS-Recon-Tool/1.0'}
-            )
-            
-            logger.debug(f"📡 Shodan search response for {domain}: {response.status_code}")
-            
-            if response.status_code == 200:
-                data = response.json()
-                ips = []
-                
-                for match in data.get('matches', []):
-                    ip = match.get('ip_str')
-                    if ip:
-                        ips.append(ip)
-                
-                unique_ips = list(set(ips))
-                logger.info(f"🔍 Shodan search for {domain} found {len(unique_ips)} unique IPs")
-                return unique_ips
-            elif response.status_code == 401:
-                logger.error("❌ Shodan API key is invalid for search")
-                return []
-            elif response.status_code == 429:
-                logger.warning("⚠️ Shodan search rate limit exceeded")
-                return []
-            else:
-                logger.warning(f"⚠️ Shodan search error for {domain}: HTTP {response.status_code}")
-                return []
-                
-        except requests.exceptions.Timeout:
-            logger.warning(f"⏱️ Shodan search timeout for {domain}")
-            return []
-        except requests.exceptions.RequestException as e:
-            logger.error(f"🌐 Shodan search network error for {domain}: {e}")
-            return []
-        except Exception as e:
-            logger.error(f"❌ Unexpected error searching Shodan for {domain}: {e}")
-            return []

src/tld_fetcher.py

@@ -1,213 +0,0 @@
-# File: src/tld_fetcher.py
-"""Fetch and cache IANA TLD list with smart prioritization."""
-
-import requests
-import logging
-from typing import List, Set, Optional, Tuple
-import os
-import time
-
-# Module logger
-logger = logging.getLogger(__name__)
-
-class TLDFetcher:
-    """Fetches and caches IANA TLD list with smart prioritization."""
-    
-    IANA_TLD_URL = "https://data.iana.org/TLD/tlds-alpha-by-domain.txt"
-    CACHE_FILE = "tlds_cache.txt"
-    CACHE_DURATION = 86400  # 24 hours in seconds
-    
-    # Common TLDs that should be checked first (high success rate)
-    PRIORITY_TLDS = {
-        # Generic top-level domains (most common)
-        'com', 'org', 'net', 'edu', 'gov', 'mil', 'int', 'info', 'biz', 'name',
-        'io', 'co', 'me', 'tv', 'cc', 'ly', 'to', 'us', 'uk', 'ca',
-        
-        # Major country codes (high usage)
-        'de', 'fr', 'it', 'es', 'nl', 'be', 'ch', 'at', 'se', 'no', 'dk', 'fi',
-        'au', 'nz', 'jp', 'kr', 'cn', 'hk', 'sg', 'my', 'th', 'in', 'br', 'mx',
-        'ru', 'pl', 'cz', 'hu', 'ro', 'bg', 'hr', 'si', 'sk', 'lt', 'lv', 'ee',
-        'ie', 'pt', 'gr', 'cy', 'mt', 'lu', 'is', 'tr', 'il', 'za', 'ng', 'eg',
-        
-        # Popular new gTLDs (established, not spam-prone)
-        'app', 'dev', 'tech', 'blog', 'news', 'shop', 'store', 'cloud', 'digital',
-        'website', 'site', 'online', 'world', 'global', 'international'
-    }
-    
-    # TLDs to deprioritize (often have wildcard DNS or low-quality domains)
-    DEPRIORITIZED_PATTERNS = [
-        'xn--',  # Internationalized domain names (often less common)
-        # These TLDs are known for high wildcard/parking rates
-        'tk', 'ml', 'ga', 'cf',  # Free TLDs often misused
-        'top', 'win', 'download', 'stream', 'science', 'click', 'link',
-        'loan', 'men', 'racing', 'review', 'party', 'trade', 'date',
-        'cricket', 'accountant', 'faith', 'gdn', 'realtor'
-    ]
-    
-    def __init__(self):
-        self._tlds: Optional[Set[str]] = None
-        self._prioritized_tlds: Optional[Tuple[List[str], List[str], List[str]]] = None
-        logger.info("🌐 TLD fetcher initialized with smart prioritization")
-    
-    def get_tlds(self) -> Set[str]:
-        """Get list of TLDs, using cache if available."""
-        if self._tlds is None:
-            logger.debug("🔍 Loading TLD list...")
-            self._tlds = self._load_tlds()
-            logger.info(f"✅ Loaded {len(self._tlds)} TLDs")
-        return self._tlds
-    
-    def get_prioritized_tlds(self) -> Tuple[List[str], List[str], List[str]]:
-        """Get TLDs sorted by priority: (priority, normal, deprioritized)."""
-        if self._prioritized_tlds is None:
-            all_tlds = self.get_tlds()
-            logger.debug("📊 Categorizing TLDs by priority...")
-            
-            priority_list = []
-            normal_list = []
-            deprioritized_list = []
-            
-            for tld in all_tlds:
-                tld_lower = tld.lower()
-                
-                if tld_lower in self.PRIORITY_TLDS:
-                    priority_list.append(tld_lower)
-                elif any(pattern in tld_lower for pattern in self.DEPRIORITIZED_PATTERNS):
-                    deprioritized_list.append(tld_lower)
-                else:
-                    normal_list.append(tld_lower)
-            
-            # Sort each category alphabetically for consistency
-            priority_list.sort()
-            normal_list.sort()
-            deprioritized_list.sort()
-            
-            self._prioritized_tlds = (priority_list, normal_list, deprioritized_list)
-            
-            logger.info(f"📊 TLD prioritization complete: "
-                       f"{len(priority_list)} priority, "
-                       f"{len(normal_list)} normal, "
-                       f"{len(deprioritized_list)} deprioritized")
-        
-        return self._prioritized_tlds
-    
-    def _load_tlds(self) -> Set[str]:
-        """Load TLDs from cache or fetch from IANA."""
-        if self._is_cache_valid():
-            logger.debug("📂 Loading TLDs from cache")
-            return self._load_from_cache()
-        else:
-            logger.info("🌐 Fetching fresh TLD list from IANA")
-            return self._fetch_and_cache()
-    
-    def _is_cache_valid(self) -> bool:
-        """Check if cache file exists and is recent."""
-        if not os.path.exists(self.CACHE_FILE):
-            logger.debug("❌ TLD cache file does not exist")
-            return False
-        
-        cache_age = time.time() - os.path.getmtime(self.CACHE_FILE)
-        is_valid = cache_age < self.CACHE_DURATION
-        
-        if is_valid:
-            logger.debug(f"✅ TLD cache is valid (age: {cache_age/3600:.1f} hours)")
-        else:
-            logger.debug(f"❌ TLD cache is expired (age: {cache_age/3600:.1f} hours)")
-        
-        return is_valid
-    
-    def _load_from_cache(self) -> Set[str]:
-        """Load TLDs from cache file."""
-        try:
-            with open(self.CACHE_FILE, 'r', encoding='utf-8') as f:
-                tlds = set()
-                for line in f:
-                    line = line.strip().lower()
-                    if line and not line.startswith('#'):
-                        tlds.add(line)
-                
-                logger.info(f"📂 Loaded {len(tlds)} TLDs from cache")
-                return tlds
-        except Exception as e:
-            logger.error(f"❌ Error loading TLD cache: {e}")
-            # Fall back to fetching fresh data
-            return self._fetch_and_cache()
-    
-    def _fetch_and_cache(self) -> Set[str]:
-        """Fetch TLDs from IANA and cache them."""
-        try:
-            logger.info(f"📡 Fetching TLD list from: {self.IANA_TLD_URL}")
-            
-            response = requests.get(
-                self.IANA_TLD_URL, 
-                timeout=30,
-                headers={'User-Agent': 'DNS-Recon-Tool/1.0'}
-            )
-            response.raise_for_status()
-            
-            tlds = set()
-            lines_processed = 0
-            
-            for line in response.text.split('\n'):
-                line = line.strip().lower()
-                if line and not line.startswith('#'):
-                    tlds.add(line)
-                lines_processed += 1
-            
-            logger.info(f"✅ Fetched {len(tlds)} TLDs from IANA (processed {lines_processed} lines)")
-            
-            # Cache the results
-            try:
-                with open(self.CACHE_FILE, 'w', encoding='utf-8') as f:
-                    f.write(response.text)
-                logger.info(f"💾 TLD list cached to {self.CACHE_FILE}")
-            except Exception as cache_error:
-                logger.warning(f"⚠️ Could not cache TLD list: {cache_error}")
-            
-            return tlds
-            
-        except requests.exceptions.Timeout:
-            logger.error("⏱️ Timeout fetching TLD list from IANA")
-            return self._get_fallback_tlds()
-        except requests.exceptions.RequestException as e:
-            logger.error(f"🌐 Network error fetching TLD list: {e}")
-            return self._get_fallback_tlds()
-        except Exception as e:
-            logger.error(f"❌ Unexpected error fetching TLD list: {e}")
-            return self._get_fallback_tlds()
-    
-    def _get_fallback_tlds(self) -> Set[str]:
-        """Return a minimal set of short TLDs if fetch fails."""
-        logger.warning("⚠️ Using fallback TLD list")
-        
-        # Use only short, well-established TLDs as fallback
-        fallback_tlds = {
-            # 2-character TLDs (country codes - most established)
-            'ad', 'ae', 'af', 'ag', 'ai', 'al', 'am', 'ao', 'aq', 'ar', 'as', 'at',
-            'au', 'aw', 'ax', 'az', 'ba', 'bb', 'bd', 'be', 'bf', 'bg', 'bh', 'bi',
-            'bj', 'bl', 'bm', 'bn', 'bo', 'bq', 'br', 'bs', 'bt', 'bv', 'bw', 'by',
-            'bz', 'ca', 'cc', 'cd', 'cf', 'cg', 'ch', 'ci', 'ck', 'cl', 'cm', 'cn',
-            'co', 'cr', 'cu', 'cv', 'cw', 'cx', 'cy', 'cz', 'de', 'dj', 'dk', 'dm',
-            'do', 'dz', 'ec', 'ee', 'eg', 'eh', 'er', 'es', 'et', 'eu', 'fi', 'fj',
-            'fk', 'fm', 'fo', 'fr', 'ga', 'gb', 'gd', 'ge', 'gf', 'gg', 'gh', 'gi',
-            'gl', 'gm', 'gn', 'gp', 'gq', 'gr', 'gs', 'gt', 'gu', 'gw', 'gy', 'hk',
-            'hm', 'hn', 'hr', 'ht', 'hu', 'id', 'ie', 'il', 'im', 'in', 'io', 'iq',
-            'ir', 'is', 'it', 'je', 'jm', 'jo', 'jp', 'ke', 'kg', 'kh', 'ki', 'km',
-            'kn', 'kp', 'kr', 'kw', 'ky', 'kz', 'la', 'lb', 'lc', 'li', 'lk', 'lr',
-            'ls', 'lt', 'lu', 'lv', 'ly', 'ma', 'mc', 'md', 'me', 'mf', 'mg', 'mh',
-            'mk', 'ml', 'mm', 'mn', 'mo', 'mp', 'mq', 'mr', 'ms', 'mt', 'mu', 'mv',
-            'mw', 'mx', 'my', 'mz', 'na', 'nc', 'ne', 'nf', 'ng', 'ni', 'nl', 'no',
-            'np', 'nr', 'nu', 'nz', 'om', 'pa', 'pe', 'pf', 'pg', 'ph', 'pk', 'pl',
-            'pm', 'pn', 'pr', 'ps', 'pt', 'pw', 'py', 'qa', 're', 'ro', 'rs', 'ru',
-            'rw', 'sa', 'sb', 'sc', 'sd', 'se', 'sg', 'sh', 'si', 'sj', 'sk', 'sl',
-            'sm', 'sn', 'so', 'sr', 'ss', 'st', 'sv', 'sx', 'sy', 'sz', 'tc', 'td',
-            'tf', 'tg', 'th', 'tj', 'tk', 'tl', 'tm', 'tn', 'to', 'tr', 'tt', 'tv',
-            'tw', 'tz', 'ua', 'ug', 'uk', 'um', 'us', 'uy', 'uz', 'va', 'vc', 've',
-            'vg', 'vi', 'vn', 'vu', 'wf', 'ws', 'ye', 'yt', 'za', 'zm', 'zw',
-            
-            # 3-character TLDs (generic - most common)
-            'com', 'org', 'net', 'edu', 'gov', 'mil', 'int'
-        }
-        
-        logger.info(f"📋 Using {len(fallback_tlds)} fallback TLDs (≤3 characters)")
-        return fallback_tlds

src/virustotal_client.py

@@ -1,227 +0,0 @@
-# File: src/virustotal_client.py
-"""VirusTotal API integration with forensic operation tracking."""
-
-import requests
-import time
-import logging
-import uuid
-from datetime import datetime
-from typing import Optional
-from .data_structures import VirusTotalResult
-from .config import Config
-
-# Module logger
-logger = logging.getLogger(__name__)
-
-class VirusTotalClient:
-    """VirusTotal API client with forensic tracking."""
-    
-    BASE_URL = "https://www.virustotal.com/vtapi/v2"
-    
-    def __init__(self, api_key: str, config: Config):
-        self.api_key = api_key
-        self.config = config
-        self.last_request = 0
-        
-        logger.info(f"🛡️ VirusTotal client initialized with API key ending in: ...{api_key[-4:] if len(api_key) > 4 else api_key}")
-    
-    def _rate_limit(self):
-        """Apply rate limiting for VirusTotal."""
-        now = time.time()
-        time_since_last = now - self.last_request
-        min_interval = 1.0 / self.config.VIRUSTOTAL_RATE_LIMIT
-        
-        if time_since_last < min_interval:
-            sleep_time = min_interval - time_since_last
-            logger.debug(f"⏸️ VirusTotal rate limiting: sleeping for {sleep_time:.2f}s")
-            time.sleep(sleep_time)
-        
-        self.last_request = time.time()
-    
-    def lookup_ip(self, ip: str, operation_id: Optional[str] = None) -> Optional[VirusTotalResult]:
-        """Lookup IP address reputation with forensic tracking."""
-        self._rate_limit()
-        
-        # Generate operation ID if not provided
-        if operation_id is None:
-            operation_id = str(uuid.uuid4())
-        
-        logger.debug(f"🔍 Querying VirusTotal for IP: {ip} (operation: {operation_id})")
-        
-        try:
-            url = f"{self.BASE_URL}/ip-address/report"
-            params = {
-                'apikey': self.api_key,
-                'ip': ip
-            }
-            
-            response = requests.get(
-                url, 
-                params=params, 
-                timeout=self.config.HTTP_TIMEOUT,
-                headers={'User-Agent': 'DNS-Recon-Tool/1.0'}
-            )
-            
-            logger.debug(f"📡 VirusTotal API response for IP {ip}: {response.status_code}")
-            
-            if response.status_code == 200:
-                data = response.json()
-                
-                logger.debug(f"VirusTotal IP response data keys: {data.keys()}")
-                
-                if data.get('response_code') == 1:
-                    # Count detected URLs
-                    detected_urls = data.get('detected_urls', [])
-                    positives = sum(1 for url in detected_urls if url.get('positives', 0) > 0)
-                    total = len(detected_urls)
-                    
-                    # Parse scan date
-                    scan_date = datetime.now()
-                    if data.get('scan_date'):
-                        try:
-                            scan_date = datetime.fromisoformat(data['scan_date'].replace('Z', '+00:00'))
-                        except ValueError:
-                            try:
-                                scan_date = datetime.strptime(data['scan_date'], '%Y-%m-%d %H:%M:%S')
-                            except ValueError:
-                                logger.debug(f"Could not parse scan_date: {data.get('scan_date')}")
-                    
-                    # Create VirusTotalResult with forensic metadata
-                    result = VirusTotalResult(
-                        resource=ip,
-                        positives=positives,
-                        total=total,
-                        scan_date=scan_date,
-                        permalink=data.get('permalink', f'https://www.virustotal.com/gui/ip-address/{ip}'),
-                        operation_id=operation_id  # Forensic tracking
-                    )
-                    
-                    logger.info(f"✅ VirusTotal result for IP {ip}: {result.positives}/{result.total} detections")
-                    return result
-                elif data.get('response_code') == 0:
-                    logger.debug(f"ℹ️ IP {ip} not found in VirusTotal database")
-                    return None
-                else:
-                    logger.debug(f"VirusTotal returned response_code: {data.get('response_code')}")
-                    return None
-            elif response.status_code == 204:
-                logger.warning("⚠️ VirusTotal API rate limit exceeded")
-                return None
-            elif response.status_code == 403:
-                logger.error("❌ VirusTotal API key is invalid or lacks permissions")
-                return None
-            else:
-                logger.warning(f"⚠️ VirusTotal API error for IP {ip}: HTTP {response.status_code}")
-                try:
-                    error_data = response.json()
-                    logger.debug(f"VirusTotal error details: {error_data}")
-                except:
-                    pass
-                return None
-                    
-        except requests.exceptions.Timeout:
-            logger.warning(f"⏱️ VirusTotal query timeout for IP {ip}")
-            return None
-        except requests.exceptions.RequestException as e:
-            logger.error(f"🌐 VirusTotal network error for IP {ip}: {e}")
-            return None
-        except Exception as e:
-            logger.error(f"❌ Unexpected error querying VirusTotal for IP {ip}: {e}")
-            return None
-    
-    def lookup_domain(self, domain: str, operation_id: Optional[str] = None) -> Optional[VirusTotalResult]:
-        """Lookup domain reputation with forensic tracking."""
-        self._rate_limit()
-        
-        # Generate operation ID if not provided
-        if operation_id is None:
-            operation_id = str(uuid.uuid4())
-        
-        logger.debug(f"🔍 Querying VirusTotal for domain: {domain} (operation: {operation_id})")
-        
-        try:
-            url = f"{self.BASE_URL}/domain/report"
-            params = {
-                'apikey': self.api_key,
-                'domain': domain
-            }
-            
-            response = requests.get(
-                url, 
-                params=params, 
-                timeout=self.config.HTTP_TIMEOUT,
-                headers={'User-Agent': 'DNS-Recon-Tool/1.0'}
-            )
-            
-            logger.debug(f"📡 VirusTotal API response for domain {domain}: {response.status_code}")
-            
-            if response.status_code == 200:
-                data = response.json()
-                
-                logger.debug(f"VirusTotal domain response data keys: {data.keys()}")
-                
-                if data.get('response_code') == 1:
-                    # Count detected URLs
-                    detected_urls = data.get('detected_urls', [])
-                    positives = sum(1 for url in detected_urls if url.get('positives', 0) > 0)
-                    total = len(detected_urls)
-                    
-                    # Also check for malicious/suspicious categories
-                    categories = data.get('categories', [])
-                    if any(cat in ['malicious', 'suspicious', 'phishing', 'malware'] 
-                          for cat in categories):
-                        positives += 1
-                    
-                    # Parse scan date
-                    scan_date = datetime.now()
-                    if data.get('scan_date'):
-                        try:
-                            scan_date = datetime.fromisoformat(data['scan_date'].replace('Z', '+00:00'))
-                        except ValueError:
-                            try:
-                                scan_date = datetime.strptime(data['scan_date'], '%Y-%m-%d %H:%M:%S')
-                            except ValueError:
-                                logger.debug(f"Could not parse scan_date: {data.get('scan_date')}")
-                    
-                    # Create VirusTotalResult with forensic metadata
-                    result = VirusTotalResult(
-                        resource=domain,
-                        positives=positives,
-                        total=max(total, 1),  # Ensure total is at least 1
-                        scan_date=scan_date,
-                        permalink=data.get('permalink', f'https://www.virustotal.com/gui/domain/{domain}'),
-                        operation_id=operation_id  # Forensic tracking
-                    )
-                    
-                    logger.info(f"✅ VirusTotal result for domain {domain}: {result.positives}/{result.total} detections")
-                    return result
-                elif data.get('response_code') == 0:
-                    logger.debug(f"ℹ️ Domain {domain} not found in VirusTotal database")
-                    return None
-                else:
-                    logger.debug(f"VirusTotal returned response_code: {data.get('response_code')}")
-                    return None
-            elif response.status_code == 204:
-                logger.warning("⚠️ VirusTotal API rate limit exceeded")
-                return None
-            elif response.status_code == 403:
-                logger.error("❌ VirusTotal API key is invalid or lacks permissions")
-                return None
-            else:
-                logger.warning(f"⚠️ VirusTotal API error for domain {domain}: HTTP {response.status_code}")
-                try:
-                    error_data = response.json()
-                    logger.debug(f"VirusTotal error details: {error_data}")
-                except:
-                    pass
-                return None
-                    
-        except requests.exceptions.Timeout:
-            logger.warning(f"⏱️ VirusTotal query timeout for domain {domain}")
-            return None
-        except requests.exceptions.RequestException as e:
-            logger.error(f"🌐 VirusTotal network error for domain {domain}: {e}")
-            return None
-        except Exception as e:
-            logger.error(f"❌ Unexpected error querying VirusTotal for domain {domain}: {e}")
-            return None

src/web_app.py

@@ -1,393 +0,0 @@
-# File: src/web_app.py
-"""Flask web application for forensic reconnaissance tool."""
-
-from flask import Flask, render_template, request, jsonify, send_from_directory
-import threading
-import time
-import logging
-from .config import Config
-from .reconnaissance import ForensicReconnaissanceEngine
-from .report_generator import ForensicReportGenerator
-from .data_structures import ForensicReconData
-
-# Set up logging for this module
-logger = logging.getLogger(__name__)
-
-# Global variables for tracking ongoing scans
-active_scans = {}
-scan_lock = threading.Lock()
-
-def create_app(config: Config):
-    """Create Flask application."""
-    app = Flask(__name__, 
-                template_folder='../templates',
-                static_folder='../static')
-    
-    app.config['SECRET_KEY'] = 'forensic-recon-tool-secret-key'
-    
-    # Set up logging for web app
-    config.setup_logging(cli_mode=False)
-    logger.info("🌐 Forensic web application initialized")
-    
-    @app.route('/')
-    def index():
-        """Main page."""
-        return render_template('index.html')
-    
-    @app.route('/api/scan', methods=['POST'])
-    def start_scan():
-        """Start a new forensic reconnaissance scan."""
-        try:
-            data = request.get_json()
-            target = data.get('target')
-            scan_config = Config.from_args(
-                shodan_key=data.get('shodan_key'),
-                virustotal_key=data.get('virustotal_key'),
-                max_depth=data.get('max_depth', 2)
-            )
-            
-            if not target:
-                logger.warning("⚠️ Scan request missing target")
-                return jsonify({'error': 'Target is required'}), 400
-            
-            # Generate scan ID
-            scan_id = f"{target}_{int(time.time())}"
-            logger.info(f"🚀 Starting new forensic scan: {scan_id} for target: {target}")
-            
-            # Create shared ForensicReconData object for live updates
-            shared_data = ForensicReconData()
-            shared_data.scan_config = {
-                'target': target,
-                'max_depth': scan_config.max_depth,
-                'shodan_enabled': scan_config.shodan_key is not None,
-                'virustotal_enabled': scan_config.virustotal_key is not None
-            }
-            
-            # Initialize scan data with the shared forensic data object
-            with scan_lock:
-                active_scans[scan_id] = {
-                    'status': 'starting',
-                    'progress': 0,
-                    'message': 'Initializing forensic scan...',
-                    'data': shared_data,  # Share the forensic data object from the start
-                    'error': None,
-                    'live_stats': shared_data.get_stats(),  # Use forensic stats
-                    'latest_discoveries': []
-                }
-            
-            # Start forensic reconnaissance in background thread
-            thread = threading.Thread(
-                target=run_forensic_reconnaissance_background,
-                args=(scan_id, target, scan_config, shared_data)
-            )
-            thread.daemon = True
-            thread.start()
-            
-            return jsonify({'scan_id': scan_id})
-        
-        except Exception as e:
-            logger.error(f"❌ Error starting scan: {e}", exc_info=True)
-            return jsonify({'error': str(e)}), 500
-    
-    @app.route('/api/scan/<scan_id>/status')
-    def get_scan_status(scan_id):
-        """Get scan status and progress with live forensic discoveries."""
-        with scan_lock:
-            if scan_id not in active_scans:
-                return jsonify({'error': 'Scan not found'}), 404
-            
-            scan_data = active_scans[scan_id].copy()
-        
-        # Update live stats from forensic data if available
-        if scan_data.get('data') and hasattr(scan_data['data'], 'get_stats'):
-            scan_data['live_stats'] = scan_data['data'].get_stats()
-        
-        # Don't include the full forensic data object in status (too large)
-        if 'data' in scan_data:
-            del scan_data['data']
-        
-        return jsonify(scan_data)
-    
-    @app.route('/api/scan/<scan_id>/report')
-    def get_scan_report(scan_id):
-        """Get forensic scan report."""
-        with scan_lock:
-            if scan_id not in active_scans:
-                return jsonify({'error': 'Scan not found'}), 404
-            
-            scan_data = active_scans[scan_id]
-            
-            if scan_data['status'] != 'completed' or not scan_data['data']:
-                return jsonify({'error': 'Scan not completed'}), 400
-        
-        try:
-            # Generate forensic report
-            report_gen = ForensicReportGenerator(scan_data['data'])
-            
-            return jsonify({
-                'json_report': scan_data['data'].to_json(),
-                'text_report': report_gen.generate_text_report()
-            })
-        except Exception as e:
-            logger.error(f"❌ Error generating report for {scan_id}: {e}", exc_info=True)
-            return jsonify({'error': f'Failed to generate report: {str(e)}'}), 500
-
-
-    @app.route('/api/scan/<scan_id>/graph')
-    def get_scan_graph(scan_id):
-        """Get graph data for visualization."""
-        with scan_lock:
-            if scan_id not in active_scans:
-                return jsonify({'error': 'Scan not found'}), 404
-            
-            scan_data = active_scans[scan_id]
-            
-            if scan_data['status'] != 'completed' or not scan_data['data']:
-                return jsonify({'error': 'Scan not completed'}), 400
-        
-        try:
-            forensic_data = scan_data['data']
-            
-            # Extract nodes for graph
-            nodes = []
-            for hostname, node in forensic_data.nodes.items():
-                # Determine node color based on depth
-                color_map = {
-                    0: '#00ff41',  # Green for root
-                    1: '#ff9900',  # Orange for depth 1
-                    2: '#ff6b6b',  # Red for depth 2
-                    3: '#4ecdc4',  # Teal for depth 3
-                    4: '#45b7d1',  # Blue for depth 4+
-                }
-                color = color_map.get(node.depth, '#666666')
-                
-                # Calculate node size based on number of connections and data
-                connections = len(forensic_data.get_children(hostname)) + len(forensic_data.get_parents(hostname))
-                dns_records = len(node.get_all_dns_records())
-                certificates = len(node.certificates)
-                
-                # Size based on importance (connections + data)
-                size = max(8, min(20, 8 + connections * 2 + dns_records // 3 + certificates))
-                
-                nodes.append({
-                    'id': hostname,
-                    'label': hostname,
-                    'depth': node.depth,
-                    'color': color,
-                    'size': size,
-                    'dns_records': dns_records,
-                    'certificates': certificates,
-                    'ip_addresses': list(node.resolved_ips),
-                    'discovery_methods': [method.value for method in node.discovery_methods],
-                    'first_seen': node.first_seen.isoformat() if node.first_seen else None
-                })
-            
-            # Extract edges for graph
-            edges = []
-            for edge in forensic_data.edges:
-                # Skip synthetic TLD expansion edges for cleaner visualization
-                if edge.source_hostname.startswith('tld_expansion:'):
-                    continue
-                    
-                # Color edges by discovery method
-                method_colors = {
-                    'initial_target': '#00ff41',
-                    'tld_expansion': '#ff9900', 
-                    'dns_record_value': '#4ecdc4',
-                    'certificate_subject': '#ff6b6b',
-                    'dns_subdomain_extraction': '#45b7d1'
-                }
-                
-                edges.append({
-                    'source': edge.source_hostname,
-                    'target': edge.target_hostname,
-                    'method': edge.discovery_method.value,
-                    'color': method_colors.get(edge.discovery_method.value, '#666666'),
-                    'operation_id': edge.operation_id,
-                    'timestamp': edge.timestamp.isoformat() if edge.timestamp else None
-                })
-            
-            # Graph statistics
-            stats = {
-                'node_count': len(nodes),
-                'edge_count': len(edges),
-                'max_depth': max([node['depth'] for node in nodes]) if nodes else 0,
-                'discovery_methods': list(set([edge['method'] for edge in edges])),
-                'root_nodes': [node['id'] for node in nodes if node['depth'] == 0]
-            }
-            
-            return jsonify({
-                'nodes': nodes,
-                'edges': edges,
-                'stats': stats
-            })
-            
-        except Exception as e:
-            logger.error(f"⚠️ Error generating graph data for {scan_id}: {e}", exc_info=True)
-            return jsonify({'error': f'Failed to generate graph: {str(e)}'}), 500
-    
-    @app.route('/api/scan/<scan_id>/live-data')
-    def get_live_scan_data(scan_id):
-        """Get current forensic reconnaissance data (for real-time updates)."""
-        with scan_lock:
-            if scan_id not in active_scans:
-                return jsonify({'error': 'Scan not found'}), 404
-            
-            scan_data = active_scans[scan_id]
-            forensic_data = scan_data['data']
-            
-            if not forensic_data:
-                return jsonify({
-                    'hostnames': [],
-                    'ip_addresses': [],
-                    'stats': {
-                        'hostnames': 0,
-                        'ip_addresses': 0,
-                        'discovery_edges': 0,
-                        'operations_performed': 0,
-                        'dns_records': 0,
-                        'certificates_total': 0,
-                        'certificates_current': 0,
-                        'certificates_expired': 0,
-                        'shodan_results': 0,
-                        'virustotal_results': 0
-                    },
-                    'latest_discoveries': []
-                })
-            
-            # Extract data from forensic structure for frontend
-            try:
-                hostnames = sorted(list(forensic_data.nodes.keys()))
-                ip_addresses = sorted(list(forensic_data.ip_addresses))
-                stats = forensic_data.get_stats()
-                
-                # Generate activity log from recent operations
-                latest_discoveries = []
-                recent_operations = forensic_data.operation_timeline[-10:]  # Last 10 operations
-                
-                for op_id in recent_operations:
-                    if op_id in forensic_data.operations:
-                        operation = forensic_data.operations[op_id]
-                        activity_entry = {
-                            'timestamp': operation.timestamp.timestamp(),
-                            'message': f"{operation.operation_type.value}: {operation.target}"
-                        }
-                        
-                        # Add result summary
-                        if operation.discovered_hostnames:
-                            activity_entry['message'] += f" → {len(operation.discovered_hostnames)} hostnames"
-                        if operation.discovered_ips:
-                            activity_entry['message'] += f" → {len(operation.discovered_ips)} IPs"
-                        
-                        latest_discoveries.append(activity_entry)
-                
-                # Update scan data with latest discoveries
-                scan_data['latest_discoveries'] = latest_discoveries
-                
-                return jsonify({
-                    'hostnames': hostnames,
-                    'ip_addresses': ip_addresses,
-                    'stats': stats,
-                    'latest_discoveries': latest_discoveries
-                })
-                
-            except Exception as e:
-                logger.error(f"❌ Error extracting live data for {scan_id}: {e}", exc_info=True)
-                # Return minimal data structure
-                return jsonify({
-                    'hostnames': [],
-                    'ip_addresses': [],
-                    'stats': {
-                        'hostnames': len(forensic_data.nodes) if forensic_data.nodes else 0,
-                        'ip_addresses': len(forensic_data.ip_addresses) if forensic_data.ip_addresses else 0,
-                        'discovery_edges': 0,
-                        'operations_performed': 0,
-                        'dns_records': 0,
-                        'certificates_total': 0,
-                        'certificates_current': 0,
-                        'certificates_expired': 0,
-                        'shodan_results': 0,
-                        'virustotal_results': 0
-                    },
-                    'latest_discoveries': []
-                })
-    
-    return app
-
-def run_forensic_reconnaissance_background(scan_id: str, target: str, config: Config, shared_data: ForensicReconData):
-    """Run forensic reconnaissance in background thread with shared forensic data object."""
-    
-    def update_progress(message: str, percentage: int = None):
-        """Update scan progress and live forensic statistics."""
-        with scan_lock:
-            if scan_id in active_scans:
-                active_scans[scan_id]['message'] = message
-                if percentage is not None:
-                    active_scans[scan_id]['progress'] = percentage
-                
-                # Update live stats from the shared forensic data object
-                if shared_data:
-                    active_scans[scan_id]['live_stats'] = shared_data.get_stats()
-                    
-                    # Add to latest discoveries (keep last 10)
-                    if 'latest_discoveries' not in active_scans[scan_id]:
-                        active_scans[scan_id]['latest_discoveries'] = []
-                    
-                    # Create activity entry
-                    activity_entry = {
-                        'timestamp': time.time(),
-                        'message': message
-                    }
-                    
-                    active_scans[scan_id]['latest_discoveries'].append(activity_entry)
-                    
-                    # Keep only last 10 discoveries
-                    active_scans[scan_id]['latest_discoveries'] = \
-                        active_scans[scan_id]['latest_discoveries'][-10:]
-        
-        logger.info(f"[{scan_id}] {message} ({percentage}%)" if percentage else f"[{scan_id}] {message}")
-    
-    try:
-        logger.info(f"🔧 Initializing forensic reconnaissance engine for scan: {scan_id}")
-        
-        # Initialize forensic engine
-        engine = ForensicReconnaissanceEngine(config)
-        engine.set_progress_callback(update_progress)
-        
-        # IMPORTANT: Pass the shared forensic data object to the engine
-        engine.set_shared_data(shared_data)
-        
-        # Update status
-        with scan_lock:
-            active_scans[scan_id]['status'] = 'running'
-        
-        logger.info(f"🚀 Starting forensic reconnaissance for: {target}")
-        
-        # Run forensic reconnaissance - this will populate the shared_data object incrementally
-        final_data = engine.run_reconnaissance(target)
-        
-        logger.info(f"✅ Forensic reconnaissance completed for scan: {scan_id}")
-        
-        # Update with final results (the shared_data should already be populated)
-        with scan_lock:
-            active_scans[scan_id]['status'] = 'completed'
-            active_scans[scan_id]['progress'] = 100
-            active_scans[scan_id]['message'] = 'Forensic reconnaissance completed'
-            active_scans[scan_id]['data'] = final_data  # This should be the same as shared_data
-            active_scans[scan_id]['live_stats'] = final_data.get_stats()
-        
-        # Log final forensic statistics
-        final_stats = final_data.get_stats()
-        logger.info(f"📊 Final forensic stats for {scan_id}: {final_stats}")
-        
-        # Log discovery graph analysis
-        graph_analysis = final_data._generate_graph_analysis()
-        logger.info(f"🌐 Discovery graph: {len(final_data.nodes)} nodes, {len(final_data.edges)} edges, max depth: {graph_analysis['max_depth']}")
-    
-    except Exception as e:
-        logger.error(f"❌ Error in forensic reconnaissance for {scan_id}: {e}", exc_info=True)
-        # Handle errors
-        with scan_lock:
-            active_scans[scan_id]['status'] = 'error'
-            active_scans[scan_id]['error'] = str(e)
-            active_scans[scan_id]['message'] = f'Error: {str(e)}'

static/js/graph.js

@@ -0,0 +1,925 @@
+/**
+ * Graph visualization module for DNSRecon
+ * Handles network graph rendering using vis.js
+ */
+
+class GraphManager {
+    constructor(containerId) {
+        this.container = document.getElementById(containerId);
+        this.network = null;
+        this.nodes = new vis.DataSet();
+        this.edges = new vis.DataSet();
+        this.isInitialized = false;
+        this.currentLayout = 'physics';
+        this.nodeInfoPopup = null;
+
+        this.options = {
+            nodes: {
+                shape: 'dot',
+                size: 15,
+                font: {
+                    size: 12,
+                    color: '#c7c7c7',
+                    face: 'Roboto Mono, monospace',
+                    background: 'rgba(26, 26, 26, 0.9)',
+                    strokeWidth: 2,
+                    strokeColor: '#000000'
+                },
+                borderWidth: 2,
+                borderColor: '#444',
+                scaling: {
+                    min: 10,
+                    max: 30,
+                    label: {
+                        enabled: true,
+                        min: 8,
+                        max: 16
+                    }
+                },
+                chosen: {
+                    node: (values, id, selected, hovering) => {
+                        values.borderColor = '#00ff41';
+                        values.borderWidth = 3;
+                    }
+                }
+            },
+            edges: {
+                width: 2,
+                color: {
+                    color: '#555',
+                    highlight: '#00ff41',
+                    hover: '#ff9900',
+                    inherit: false
+                },
+                font: {
+                    size: 10,
+                    color: '#999',
+                    face: 'Roboto Mono, monospace',
+                    background: 'rgba(26, 26, 26, 0.8)',
+                    strokeWidth: 1,
+                    strokeColor: '#000000'
+                },
+                arrows: {
+                    to: {
+                        enabled: true,
+                        scaleFactor: 1,
+                        type: 'arrow'
+                    }
+                },
+                smooth: {
+                    enabled: true,
+                    type: 'dynamic',
+                    roundness: 0.6
+                },
+                chosen: {
+                    edge: (values, id, selected, hovering) => {
+                        values.color = '#00ff41';
+                        values.width = 4;
+                    }
+                }
+            },
+            physics: {
+                enabled: true,
+                stabilization: {
+                    enabled: true,
+                    iterations: 150,
+                    updateInterval: 50
+                },
+                barnesHut: {
+                    gravitationalConstant: -3000,
+                    centralGravity: 0.4,
+                    springLength: 120,
+                    springConstant: 0.05,
+                    damping: 0.1,
+                    avoidOverlap: 0.2
+                },
+                maxVelocity: 30,
+                minVelocity: 0.1,
+                solver: 'barnesHut',
+                timestep: 0.4,
+                adaptiveTimestep: true
+            },
+            interaction: {
+                hover: true,
+                hoverConnectedEdges: true,
+                selectConnectedEdges: true,
+                tooltipDelay: 300,
+                hideEdgesOnDrag: false,
+                hideNodesOnDrag: false,
+                zoomView: true,
+                dragView: true,
+                multiselect: true
+            },
+            layout: {
+                improvedLayout: true,
+                randomSeed: 2
+            }
+        };
+
+        this.createNodeInfoPopup();
+    }
+
+    /**
+     * Create floating node info popup
+     */
+    createNodeInfoPopup() {
+        this.nodeInfoPopup = document.createElement('div');
+        this.nodeInfoPopup.className = 'node-info-popup';
+        this.nodeInfoPopup.style.display = 'none';
+        document.body.appendChild(this.nodeInfoPopup);
+    }
+
+    /**
+     * Initialize the network graph
+     */
+    initialize() {
+        if (this.isInitialized) {
+            return;
+        }
+
+        try {
+            const data = {
+                nodes: this.nodes,
+                edges: this.edges
+            };
+
+            this.network = new vis.Network(this.container, data, this.options);
+            this.setupNetworkEvents();
+            this.isInitialized = true;
+
+            // Hide placeholder
+            const placeholder = this.container.querySelector('.graph-placeholder');
+            if (placeholder) {
+                placeholder.style.display = 'none';
+            }
+
+            // Add graph controls
+            this.addGraphControls();
+
+            console.log('Graph initialized successfully');
+        } catch (error) {
+            console.error('Failed to initialize graph:', error);
+            this.showError('Failed to initialize visualization');
+        }
+    }
+
+    /**
+     * Add interactive graph controls
+     */
+    addGraphControls() {
+        const controlsContainer = document.createElement('div');
+        controlsContainer.className = 'graph-controls';
+        controlsContainer.innerHTML = `
+            <button class="graph-control-btn" id="graph-fit" title="Fit to Screen">[FIT]</button>
+            <button class="graph-control-btn" id="graph-physics" title="Toggle Physics">[PHYSICS]</button>
+            <button class="graph-control-btn" id="graph-cluster" title="Cluster Nodes">[CLUSTER]</button>
+        `;
+
+        this.container.appendChild(controlsContainer);
+
+        // Add control event listeners
+        document.getElementById('graph-fit').addEventListener('click', () => this.fitView());
+        document.getElementById('graph-physics').addEventListener('click', () => this.togglePhysics());
+        document.getElementById('graph-cluster').addEventListener('click', () => this.toggleClustering());
+    }
+
+    /**
+     * Setup network event handlers
+     */
+    setupNetworkEvents() {
+        if (!this.network) return;
+
+        // Node click event with details
+        this.network.on('click', (params) => {
+            if (params.nodes.length > 0) {
+                const nodeId = params.nodes[0];
+                if (this.network.isCluster(nodeId)) {
+                    this.network.openCluster(nodeId);
+                } else {
+                    const node = this.nodes.get(nodeId);
+                    if (node) {
+                        this.showNodeDetails(node);
+                        this.highlightNodeConnections(nodeId);
+                    }
+                }
+            } else {
+                this.clearHighlights();
+            }
+        });
+
+        // Hover events
+        this.network.on('hoverNode', (params) => {
+            const nodeId = params.node;
+            const node = this.nodes.get(nodeId);
+            if (node) {
+                this.highlightConnectedNodes(nodeId, true);
+            }
+        });
+
+        this.network.on('oncontext', (params) => {
+            params.event.preventDefault();
+        });
+
+        // Stabilization events with progress
+        this.network.on('stabilizationProgress', (params) => {
+            const progress = params.iterations / params.total;
+            this.updateStabilizationProgress(progress);
+        });
+
+        this.network.on('stabilizationIterationsDone', () => {
+            this.onStabilizationComplete();
+        });
+
+        // Selection events
+        this.network.on('select', (params) => {
+            console.log('Selected nodes:', params.nodes);
+            console.log('Selected edges:', params.edges);
+        });
+    }
+
+    /**
+     * @param {Object} graphData - Graph data from backend
+     */
+    updateGraph(graphData) {
+        if (!graphData || !graphData.nodes || !graphData.edges) {
+            console.warn('Invalid graph data received');
+            return;
+        }
+
+        try {
+            // Initialize if not already done
+            if (!this.isInitialized) {
+                this.initialize();
+            }
+
+            const largeEntityMap = new Map();
+            graphData.nodes.forEach(node => {
+                if (node.type === 'large_entity' && node.attributes && Array.isArray(node.attributes.nodes)) {
+                    node.attributes.nodes.forEach(nodeId => {
+                        largeEntityMap.set(nodeId, node.id);
+                    });
+                }
+            });
+
+            const processedNodes = graphData.nodes.map(node => {
+                const processed = this.processNode(node);
+                if (largeEntityMap.has(node.id)) {
+                    processed.hidden = true;
+                }
+                return processed;
+            });
+
+            const mergedEdges = {};
+            graphData.edges.forEach(edge => {
+                const fromNode = largeEntityMap.has(edge.from) ? largeEntityMap.get(edge.from) : edge.from;
+                const toNode = largeEntityMap.has(edge.to) ? largeEntityMap.get(edge.to) : edge.to;
+                const mergeKey = `${fromNode}-${toNode}-${edge.label}`;
+
+                if (!mergedEdges[mergeKey]) {
+                    mergedEdges[mergeKey] = {
+                        ...edge,
+                        from: fromNode,
+                        to: toNode,
+                        count: 0,
+                        confidence_score: 0
+                    };
+                }
+
+                mergedEdges[mergeKey].count++;
+                if (edge.confidence_score > mergedEdges[mergeKey].confidence_score) {
+                    mergedEdges[mergeKey].confidence_score = edge.confidence_score;
+                }
+            });
+
+            const processedEdges = Object.values(mergedEdges).map(edge => {
+                const processed = this.processEdge(edge);
+                if (edge.count > 1) {
+                    processed.label = `${edge.label} (${edge.count})`;
+                }
+                return processed;
+            });
+
+            // Update datasets with animation
+            const existingNodeIds = this.nodes.getIds();
+            const existingEdgeIds = this.edges.getIds();
+
+            // Add new nodes with fade-in animation
+            const newNodes = processedNodes.filter(node => !existingNodeIds.includes(node.id));
+            const newEdges = processedEdges.filter(edge => !existingEdgeIds.includes(edge.id));
+
+            // Update existing data
+            this.nodes.update(processedNodes);
+            this.edges.update(processedEdges);
+
+            // Highlight new additions briefly
+            if (newNodes.length > 0 || newEdges.length > 0) {
+                setTimeout(() => this.highlightNewElements(newNodes, newEdges), 100);
+            }
+
+            // Auto-fit view for small graphs or first update
+            if (processedNodes.length <= 10 || existingNodeIds.length === 0) {
+                setTimeout(() => this.fitView(), 800);
+            }
+
+            console.log(`Graph updated: ${processedNodes.length} nodes, ${processedEdges.length} edges (${newNodes.length} new nodes, ${newEdges.length} new edges)`);
+        } catch (error) {
+            console.error('Failed to update graph:', error);
+            this.showError('Failed to update visualization');
+        }
+    }
+
+    /**
+     * Process node data with styling and metadata
+     * @param {Object} node - Raw node data
+     * @returns {Object} Processed node data
+     */
+    processNode(node) {
+        const processedNode = {
+            id: node.id,
+            label: this.formatNodeLabel(node.id, node.type),
+            color: this.getNodeColor(node.type),
+            size: this.getNodeSize(node.type),
+            borderColor: this.getNodeBorderColor(node.type),
+            shape: this.getNodeShape(node.type),
+            attributes: node.attributes || {},
+            description: node.description || '',
+            metadata: node.metadata || {},
+            type: node.type,
+            incoming_edges: node.incoming_edges || [],
+            outgoing_edges: node.outgoing_edges || []
+        };
+
+        // Add confidence-based styling
+        if (node.confidence) {
+            processedNode.borderWidth = Math.max(2, Math.floor(node.confidence * 5));
+        }
+
+        // Style based on certificate validity
+        if (node.type === 'domain') {
+            if (node.attributes && node.attributes.certificates && node.attributes.certificates.has_valid_cert === false) {
+                processedNode.color = { background: '#888888', border: '#666666' };
+            }
+        }
+        
+        // Handle merged correlation objects (similar to large entities)
+        if (node.type === 'correlation_object') {
+            const metadata = node.metadata || {};
+            const values = metadata.values || [];
+            const mergeCount = metadata.merge_count || 1;
+            
+            if (mergeCount > 1) {
+                // Display as merged correlation container
+                processedNode.label = `Correlations (${mergeCount})`;
+                processedNode.title = `Merged correlation container with ${mergeCount} values: ${values.slice(0, 3).join(', ')}${values.length > 3 ? '...' : ''}`;
+                processedNode.borderWidth = 3; // Thicker border for merged nodes
+            } else {
+                // Single correlation value
+                const value = Array.isArray(values) && values.length > 0 ? values[0] : (metadata.value || 'Unknown');
+                const displayValue = typeof value === 'string' && value.length > 20 ? value.substring(0, 17) + '...' : value;
+                processedNode.label = `${displayValue}`;
+                processedNode.title = `Correlation: ${value}`;
+            }
+        }
+
+        return processedNode;
+    }
+
+    /**
+     * Process edge data with styling and metadata
+     * @param {Object} edge - Raw edge data
+     * @returns {Object} Processed edge data
+     */
+    processEdge(edge) {
+        const confidence = edge.confidence_score || 0;
+        const processedEdge = {
+            id: `${edge.from}-${edge.to}`,
+            from: edge.from,
+            to: edge.to,
+            label: this.formatEdgeLabel(edge.label, confidence),
+            title: this.createEdgeTooltip(edge),
+            width: this.getEdgeWidth(confidence),
+            color: this.getEdgeColor(confidence),
+            dashes: confidence < 0.6 ? [5, 5] : false,
+            metadata: {
+                relationship_type: edge.label,
+                confidence_score: confidence,
+                source_provider: edge.source_provider,
+                discovery_timestamp: edge.discovery_timestamp
+            }
+        };
+
+
+
+        return processedEdge;
+    }
+
+    /**
+     * Format node label for display
+     * @param {string} nodeId - Node identifier
+     * @param {string} nodeType - Node type
+     * @returns {string} Formatted label
+     */
+    formatNodeLabel(nodeId, nodeType) {
+        if (typeof nodeId !== 'string') return '';
+        if (nodeId.length > 20) {
+            return nodeId.substring(0, 17) + '...';
+        }
+        return nodeId;
+    }
+
+    /**
+     * Format edge label for display
+     * @param {string} relationshipType - Type of relationship
+     * @param {number} confidence - Confidence score
+     * @returns {string} Formatted label
+     */
+    formatEdgeLabel(relationshipType, confidence) {
+        if (!relationshipType) return '';
+
+        const confidenceText = confidence >= 0.8 ? '●' : confidence >= 0.6 ? '◐' : '○';
+        return `${relationshipType} ${confidenceText}`;
+    }
+    
+    /**
+     * Get node color based on type
+     * @param {string} nodeType - Node type
+     * @returns {string} Color value
+     */
+    getNodeColor(nodeType) {
+        const colors = {
+            'domain': '#00ff41',     // Green
+            'ip': '#ff9900',         // Amber
+            'asn': '#00aaff',         // Blue
+            'large_entity': '#ff6b6b', // Red for large entities
+            'correlation_object': '#9620c0ff'
+        };
+        return colors[nodeType] || '#ffffff';
+    }
+
+
+    /**
+     * Get node border color based on type
+     * @param {string} nodeType - Node type
+     * @returns {string} Border color value
+     */
+    getNodeBorderColor(nodeType) {
+        const borderColors = {
+            'domain': '#00aa2e',
+            'ip': '#cc7700',
+            'asn': '#0088cc',
+            'correlation_object': '#c235c9ff'
+        };
+        return borderColors[nodeType] || '#666666';
+    }
+
+    /**
+     * Get node size based on type
+     * @param {string} nodeType - Node type
+     * @returns {number} Node size
+     */
+    getNodeSize(nodeType) {
+        const sizes = {
+            'domain': 12,
+            'ip': 14,
+            'asn': 16,
+            'correlation_object': 8,
+            'large_entity': 5
+        };
+        return sizes[nodeType] || 12;
+    }
+    
+    /**
+     * Get node shape based on type
+     * @param {string} nodeType - Node type
+     * @returns {string} Shape name
+     */
+    getNodeShape(nodeType) {
+        const shapes = {
+            'domain': 'dot',
+            'ip': 'square',
+            'asn': 'triangle',
+            'correlation_object': 'hexagon',
+            'large_entity': 'database'
+        };
+        return shapes[nodeType] || 'dot';
+    }
+    
+    /**
+     * Get edge color based on confidence
+     * @param {number} confidence - Confidence score
+     * @returns {string} Edge color
+     */
+    getEdgeColor(confidence) {
+        if (confidence >= 0.8) {
+            return '#00ff41'; // High confidence - green
+        } else if (confidence >= 0.6) {
+            return '#ff9900'; // Medium confidence - amber
+        } else {
+            return '#666666'; // Low confidence - gray
+        }
+    }
+
+    /**
+     * Get edge width based on confidence
+     * @param {number} confidence - Confidence score
+     * @returns {number} Edge width
+     */
+    getEdgeWidth(confidence) {
+        if (confidence >= 0.8) {
+            return 3;
+        } else if (confidence >= 0.6) {
+            return 2;
+        } else {
+            return 1;
+        }
+    }
+    
+    /**
+     * Create edge tooltip with correct provider information
+     * @param {Object} edge - Edge data
+     * @returns {string} HTML tooltip content
+     */
+    createEdgeTooltip(edge) {
+        let tooltip = `<div style="font-family: 'Roboto Mono', monospace; font-size: 11px;">`;
+        tooltip += `<div style="color: #00ff41; font-weight: bold; margin-bottom: 4px;">${edge.label || 'Relationship'}</div>`;
+        tooltip += `<div style="color: #999; margin-bottom: 2px;">Confidence: ${(edge.confidence_score * 100).toFixed(1)}%</div>`;
+        
+        if (edge.source_provider) {
+            tooltip += `<div style="color: #999; margin-bottom: 2px;">Provider: ${edge.source_provider}</div>`;
+        }
+        
+        if (edge.discovery_timestamp) {
+            const date = new Date(edge.discovery_timestamp);
+            tooltip += `<div style="color: #666; font-size: 10px;">Discovered: ${date.toLocaleString()}</div>`;
+        }
+        
+        tooltip += `</div>`;
+        return tooltip;
+    }
+
+    /**
+     * Determine if node is important based on connections or metadata
+     * @param {Object} node - Node data
+     * @returns {boolean} True if node is important
+     */
+    isImportantNode(node) {
+        // Mark nodes as important based on criteria
+        if (node.type === 'domain' && node.id.includes('www.')) return false;
+        if (node.metadata && node.metadata.connection_count > 3) return true;
+        if (node.type === 'asn') return true;
+        return false;
+    }
+
+    /**
+     * Show node details in modal
+     * @param {Object} node - Node object
+     */
+    showNodeDetails(node) {
+        // Trigger custom event for main application to handle
+        const event = new CustomEvent('nodeSelected', {
+            detail: { node }
+        });
+        document.dispatchEvent(event);
+    }
+    
+    /**
+     * Hide node info popup
+     */
+    hideNodeInfoPopup() {
+        if (this.nodeInfoPopup) {
+            this.nodeInfoPopup.style.display = 'none';
+        }
+    }
+    
+    /**
+     * Highlight node connections
+     * @param {string} nodeId - Node to highlight
+     */
+    highlightNodeConnections(nodeId) {
+        const connectedNodes = this.network.getConnectedNodes(nodeId);
+        const connectedEdges = this.network.getConnectedEdges(nodeId);
+        
+        // Update node colors
+        const nodeUpdates = connectedNodes.map(id => ({
+            id: id,
+            borderColor: '#ff9900',
+            borderWidth: 3
+        }));
+        
+        nodeUpdates.push({
+            id: nodeId,
+            borderColor: '#00ff41',
+            borderWidth: 4
+        });
+        
+        // Update edge colors
+        const edgeUpdates = connectedEdges.map(id => ({
+            id: id,
+            color: { color: '#ff9900' },
+            width: 3
+        }));
+        
+        this.nodes.update(nodeUpdates);
+        this.edges.update(edgeUpdates);
+        
+        // Store for cleanup
+        this.highlightedElements = {
+            nodes: connectedNodes.concat([nodeId]),
+            edges: connectedEdges
+        };
+    }
+
+    /**
+     * Highlight connected nodes on hover
+     * @param {string} nodeId - Node ID
+     * @param {boolean} highlight - Whether to highlight or unhighlight
+     */
+    highlightConnectedNodes(nodeId, highlight) {
+        const connectedNodes = this.network.getConnectedNodes(nodeId);
+        const connectedEdges = this.network.getConnectedEdges(nodeId);
+        
+        if (highlight) {
+            // Dim all other elements
+            this.dimUnconnectedElements([nodeId, ...connectedNodes], connectedEdges);
+        }
+    }
+
+    /**
+     * Dim elements not connected to the specified nodes
+     * @param {Array} nodeIds - Node IDs to keep highlighted
+     * @param {Array} edgeIds - Edge IDs to keep highlighted
+     */
+    dimUnconnectedElements(nodeIds, edgeIds) {
+        const allNodes = this.nodes.get();
+        const allEdges = this.edges.get();
+        
+        const nodeUpdates = allNodes.map(node => ({
+            id: node.id,
+            opacity: nodeIds.includes(node.id) ? 1 : 0.3
+        }));
+        
+        const edgeUpdates = allEdges.map(edge => ({
+            id: edge.id,
+            opacity: edgeIds.includes(edge.id) ? 1 : 0.1
+        }));
+        
+        this.nodes.update(nodeUpdates);
+        this.edges.update(edgeUpdates);
+    }
+    
+    /**
+     * Clear all highlights
+     */
+    clearHighlights() {
+        if (this.highlightedElements) {
+            // Reset highlighted nodes
+            const nodeUpdates = this.highlightedElements.nodes.map(id => {
+                const originalNode = this.nodes.get(id);
+                return {
+                    id: id,
+                    borderColor: this.getNodeBorderColor(originalNode.type),
+                    borderWidth: 2
+                };
+            });
+            
+            // Reset highlighted edges
+            const edgeUpdates = this.highlightedElements.edges.map(id => {
+                const originalEdge = this.edges.get(id);
+                return {
+                    id: id,
+                    color: this.getEdgeColor(originalEdge.metadata ? originalEdge.metadata.confidence_score : 0.5),
+                    width: this.getEdgeWidth(originalEdge.metadata ? originalEdge.metadata.confidence_score : 0.5)
+                };
+            });
+            
+            this.nodes.update(nodeUpdates);
+            this.edges.update(edgeUpdates);
+            
+            this.highlightedElements = null;
+        }
+    }
+
+    /**
+     * Clear hover highlights
+     */
+    clearHoverHighlights() {
+        const allNodes = this.nodes.get();
+        const allEdges = this.edges.get();
+        
+        const nodeUpdates = allNodes.map(node => ({ id: node.id, opacity: 1 }));
+        const edgeUpdates = allEdges.map(edge => ({ id: edge.id, opacity: 1 }));
+        
+        this.nodes.update(nodeUpdates);
+        this.edges.update(edgeUpdates);
+    }
+
+    /**
+     * Highlight newly added elements
+     * @param {Array} newNodes - New nodes
+     * @param {Array} newEdges - New edges
+     */
+    highlightNewElements(newNodes, newEdges) {
+        // Briefly highlight new nodes
+        const nodeHighlights = newNodes.map(node => ({
+            id: node.id,
+            borderColor: '#00ff41',
+            borderWidth: 4
+        }));
+        
+        // Briefly highlight new edges
+        const edgeHighlights = newEdges.map(edge => ({
+            id: edge.id,
+            color: '#00ff41',
+            width: 4
+        }));
+        
+        this.nodes.update(nodeHighlights);
+        this.edges.update(edgeHighlights);
+        
+        // Reset after animation
+        setTimeout(() => {
+            const nodeResets = newNodes.map(node => ({
+                id: node.id,
+                borderColor: this.getNodeBorderColor(node.type),
+                borderWidth: 2,
+            }));
+            
+            const edgeResets = newEdges.map(edge => ({
+                id: edge.id,
+                color: this.getEdgeColor(edge.metadata ? edge.metadata.confidence_score : 0.5),
+                width: this.getEdgeWidth(edge.metadata ? edge.metadata.confidence_score : 0.5)
+            }));
+            
+            this.nodes.update(nodeResets);
+            this.edges.update(edgeResets);
+        }, 2000);
+    }
+
+    /**
+     * Update stabilization progress
+     * @param {number} progress - Progress value (0-1)
+     */
+    updateStabilizationProgress(progress) {
+        // Could show a progress indicator if needed
+        console.log(`Graph stabilization: ${(progress * 100).toFixed(1)}%`);
+    }
+
+    /**
+     * Handle stabilization completion
+     */
+    onStabilizationComplete() {
+        console.log('Graph stabilization complete');
+    }
+
+    /**
+     * Focus view on specific node
+     * @param {string} nodeId - Node to focus on
+     */
+    focusOnNode(nodeId) {
+        const nodePosition = this.network.getPositions([nodeId]);
+        if (nodePosition[nodeId]) {
+            this.network.moveTo({
+                position: nodePosition[nodeId],
+                scale: 1.5,
+                animation: {
+                    duration: 1000,
+                    easingFunction: 'easeInOutQuart'
+                }
+            });
+        }
+    }
+    
+    /**
+     * Toggle physics simulation
+     */
+    togglePhysics() {
+        const currentPhysics = this.network.physics.physicsEnabled;
+        this.network.setOptions({ physics: !currentPhysics });
+        
+        const button = document.getElementById('graph-physics');
+        if (button) {
+            button.textContent = currentPhysics ? '[PHYSICS OFF]' : '[PHYSICS ON]';
+            button.style.color = currentPhysics ? '#ff9900' : '#00ff41';
+        }
+    }
+
+    /**
+     * Toggle node clustering
+     */
+    toggleClustering() {
+        if (this.network.isCluster('domain-cluster')) {
+            this.network.openCluster('domain-cluster');
+        } else {
+            const clusterOptions = {
+                joinCondition: (nodeOptions) => {
+                    return nodeOptions.type === 'domain';
+                },
+                clusterNodeProperties: {
+                    id: 'domain-cluster',
+                    label: 'Domains',
+                    shape: 'database',
+                    color: '#00ff41',
+                    borderWidth: 3,
+                }
+            };
+            this.network.cluster(clusterOptions);
+        }
+    }
+
+    /**
+     * Fit the view to show all nodes
+     */
+    fitView() {
+        if (this.network) {
+            this.network.fit({
+                animation: {
+                    duration: 1000,
+                    easingFunction: 'easeInOutQuad'
+                }
+            });
+        }
+    }
+
+    /**
+     * Clear the graph
+     */
+    clear() {
+        this.nodes.clear();
+        this.edges.clear();
+
+        // Show placeholder
+        const placeholder = this.container.querySelector('.graph-placeholder');
+        if (placeholder) {
+            placeholder.style.display = 'flex';
+        }
+    }
+
+    /**
+     * Show error message
+     * @param {string} message - Error message
+     */
+    showError(message) {
+        const placeholder = this.container.querySelector('.graph-placeholder .placeholder-text');
+        if (placeholder) {
+            placeholder.textContent = `Error: ${message}`;
+            placeholder.style.color = '#ff6b6b';
+        }
+    }
+
+    /**
+     * Get network statistics
+     * @returns {Object} Statistics object
+     */
+    getStatistics() {
+        return {
+            nodeCount: this.nodes.length,
+            edgeCount: this.edges.length,
+            //isStabilized: this.network ? this.network.isStabilized() : false
+        };
+    }
+
+    /**
+     * Apply filters to the graph
+     * @param {string} nodeType - The type of node to show ('all' for no filter)
+     * @param {number} minConfidence - The minimum confidence score for edges to be visible
+     */
+    applyFilters(nodeType, minConfidence) {
+        console.log(`Applying filters: nodeType=${nodeType}, minConfidence=${minConfidence}`);
+
+        const nodeUpdates = [];
+        const edgeUpdates = [];
+
+        const allNodes = this.nodes.get({ returnType: 'Object' });
+        const allEdges = this.edges.get();
+
+        // Determine which nodes are visible based on the nodeType filter
+        for (const nodeId in allNodes) {
+            const node = allNodes[nodeId];
+            const isVisible = (nodeType === 'all' || node.type === nodeType);
+            nodeUpdates.push({ id: nodeId, hidden: !isVisible });
+        }
+
+        // Update nodes first to determine edge visibility
+        this.nodes.update(nodeUpdates);
+
+        // Determine which edges are visible based on confidence and connected nodes
+        for (const edge of allEdges) {
+            const sourceNode = this.nodes.get(edge.from);
+            const targetNode = this.nodes.get(edge.to);
+            const confidence = edge.metadata ? edge.metadata.confidence_score : 0;
+
+            const isVisible = confidence >= minConfidence &&
+                              sourceNode && !sourceNode.hidden &&
+                              targetNode && !targetNode.hidden;
+            
+            edgeUpdates.push({ id: edge.id, hidden: !isVisible });
+        }
+
+        this.edges.update(edgeUpdates);
+
+        console.log('Filters applied.');
+    }
+}
+
+// Export for use in main.js
+window.GraphManager = GraphManager;

static/style.css

@@ -1,688 +0,0 @@
-/*
-    ███████╗██████╗ ███████╗ ██████╗████████╗     ██████╗  ██████╗ ██╗  ██╗███████╗
-    ██╔════╝██╔══██╗██╔════╝██╔═══██╗╚══██╔══╝    ██╔═══██╗██╔═══██╗╚██╗██╔╝██╔════╝
-    ███████╗██████╔╝█████╗  ██║   ██║   ██║       ██║   ██║██║   ██║ ╚███╔╝ ███████╗
-    ╚════██║██╔══██╗██╔══╝  ██║   ██║   ██║       ██║   ██║██║   ██║ ██╔██╗ ╚════██║
-    ███████║██║  ██║███████╗╚██████╔╝   ██║       ╚██████╔╝╚██████╔╝██╔╝ ██╗███████║
-    ╚══════╝╚═╝  ╚═╝╚══════╝ ╚═════╝    ╚═╝        ╚═════╝  ╚═════╝ ╚═╝  ╚═╝╚══════╝
-    
-    TACTICAL THEME - DNS RECONNAISSANCE INTERFACE
-    STYLE OVERRIDE
-*/
-
-* {
-    margin: 0;
-    padding: 0;
-    box-sizing: border-box;
-}
-
-body {
-    font-family: 'Roboto Mono', 'Lucida Console', Monaco, monospace;
-    line-height: 1.6;
-    color: #c7c7c7; /* Light grey for readability */
-    /* Dark, textured background for a gritty feel */
-    background-color: #1a1a1a;
-    background-image: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='4' height='4' viewBox='0 0 4 4'%3E%3Cpath fill='%23333333' fill-opacity='0.4' d='M1 3h1v1H1V3zm2-2h1v1H3V1z'%3E%3C/path%3E%3C/svg%3E");
-    min-height: 100vh;
-}
-
-.container {
-    max-width: 1200px;
-    margin: 0 auto;
-    padding: 20px;
-}
-
-header {
-    text-align: center;
-    color: #e0e0e0;
-    margin-bottom: 40px;
-    border-bottom: 1px solid #444;
-    padding-bottom: 20px;
-}
-
-header h1 {
-    font-family: 'Special Elite', 'Courier New', monospace; /* Stencil / Typewriter font */
-    font-size: 2.8rem;
-    color: #00ff41; /* Night-vision green */
-    text-shadow: 0 0 5px rgba(0, 255, 65, 0.5);
-    margin-bottom: 10px;
-    letter-spacing: 2px;
-}
-
-header p {
-    font-size: 1.1rem;
-    color: #a0a0a0;
-}
-
-.scan-form, .progress-section, .results-section {
-    background: #2a2a2a; /* Dark charcoal */
-    border-radius: 4px; /* Sharper edges */
-    border: 1px solid #444;
-    box-shadow: inset 0 0 15px rgba(0,0,0,0.5);
-    padding: 30px;
-    margin-bottom: 25px;
-}
-
-.scan-form h2, .progress-section h2, .results-section h2 {
-    margin-bottom: 20px;
-    color: #e0e0e0;
-    border-bottom: 1px solid #555;
-    padding-bottom: 10px;
-    text-transform: uppercase; /* Military style */
-    letter-spacing: 1px;
-}
-
-.form-group {
-    margin-bottom: 20px;
-}
-
-.form-group label {
-    display: block;
-    margin-bottom: 8px;
-    font-weight: 600;
-    color: #b0b0b0;
-    text-transform: uppercase;
-    font-size: 0.9rem;
-}
-
-.form-group input, .form-group select {
-    width: 100%;
-    padding: 12px;
-    background: #1a1a1a;
-    border: 1px solid #555;
-    border-radius: 2px;
-    font-size: 16px;
-    color: #00ff41; /* Green text for input fields */
-    font-family: 'Roboto Mono', monospace;
-    transition: all 0.2s ease-in-out;
-}
-
-.form-group input:focus, .form-group select:focus {
-    outline: none;
-    border-color: #ff9900; /* Amber focus color */
-    box-shadow: 0 0 5px rgba(255, 153, 0, 0.5);
-}
-
-.api-keys {
-    background: rgba(0,0,0,0.3);
-    padding: 20px;
-    border-radius: 4px;
-    border: 1px solid #444;
-    margin: 20px 0;
-}
-
-.api-keys h3 {
-    margin-bottom: 15px;
-    color: #c7c7c7;
-}
-
-.btn-primary, .btn-secondary {
-    padding: 12px 24px;
-    border: 1px solid #666;
-    border-radius: 2px;
-    font-size: 16px;
-    font-weight: 600;
-    cursor: pointer;
-    transition: all 0.2s ease-in-out;
-    margin-right: 10px;
-    margin-bottom: 10px;
-    text-transform: uppercase;
-    letter-spacing: 1px;
-}
-
-.btn-primary {
-    background: #2c5c34; /* Dark military green */
-    color: #e0e0e0;
-    border-color: #3b7b46;
-}
-
-.btn-primary:hover {
-    background: #3b7b46; /* Lighter green on hover */
-    color: #fff;
-    border-color: #4cae5c;
-}
-
-.btn-secondary {
-    background: #4a4a4a; /* Dark grey */
-    color: #c7c7c7;
-    border-color: #666;
-}
-
-.btn-secondary:hover {
-    background: #5a5a5a;
-}
-
-.btn-secondary.active {
-    background: #6a4f2a; /* Amber/Brown for active state */
-    color: #fff;
-    border-color: #ff9900;
-}
-
-.progress-bar {
-    width: 100%;
-    height: 20px;
-    background: #1a1a1a;
-    border: 1px solid #555;
-    border-radius: 2px;
-    overflow: hidden;
-    margin-bottom: 15px;
-    padding: 2px;
-}
-
-.progress-fill {
-    height: 100%;
-    background: #ff9900; /* Solid amber progress fill */
-    width: 0%;
-    transition: width 0.3s ease;
-    border-radius: 0;
-}
-
-#progressMessage {
-    font-weight: 500;
-    color: #a0a0a0;
-    margin-bottom: 20px;
-}
-
-.scan-controls {
-    text-align: center;
-}
-
-.results-controls {
-    margin-bottom: 20px;
-    text-align: center;
-}
-
-.report-container {
-    background: #0a0a0a; /* Near-black terminal background */
-    border-radius: 4px;
-    border: 1px solid #333;
-    padding: 20px;
-    max-height: 600px;
-    overflow-y: auto;
-    box-shadow: inset 0 0 10px #000;
-}
-
-#reportContent {
-    color: #00ff41; /* Classic terminal green */
-    font-family: 'Courier New', monospace;
-    font-size: 14px;
-    line-height: 1.4;
-    white-space: pre-wrap;
-    word-wrap: break-word;
-}
-
-.hostname-list, .ip-list {
-    display: flex;
-    flex-wrap: wrap;
-    gap: 5px;
-}
-
-.discovery-item {
-    background: #2a2a2a;
-    color: #00ff41;
-    padding: 2px 6px;
-    border-radius: 2px;
-    font-family: 'Courier New', monospace;
-    font-size: 0.8rem;
-    border: 1px solid #444;
-}
-
-.activity-list {
-    max-height: 150px;
-    overflow-y: auto;
-}
-
-.activity-item {
-    color: #a0a0a0;
-    font-family: 'Courier New', monospace;
-    font-size: 0.8rem;
-    padding: 2px 0;
-    border-bottom: 1px solid #333;
-}
-
-.activity-item:last-child {
-    border-bottom: none;
-}
-
-/* Live Discoveries Base Styling */
-.live-discoveries {
-    background: rgba(0, 20, 0, 0.6);
-    border: 1px solid #00ff41;
-    border-radius: 4px;
-    padding: 20px;
-    margin-top: 20px;
-}
-
-.live-discoveries h3 {
-    color: #00ff41;
-    margin-bottom: 15px;
-    text-transform: uppercase;
-    letter-spacing: 1px;
-}
-
-/* Enhanced styling for live discoveries when shown in results view */
-.results-section .live-discoveries {
-    background: rgba(0, 40, 0, 0.8);
-    border: 2px solid #00ff41;
-    border-radius: 4px;
-    padding: 20px;
-    margin-bottom: 25px;
-    box-shadow: 0 0 10px rgba(0, 255, 65, 0.3);
-}
-
-.results-section .live-discoveries h3 {
-    color: #00ff41;
-    text-shadow: 0 0 3px rgba(0, 255, 65, 0.5);
-}
-
-/* Ensure the progress section flows nicely when showing both progress and results */
-.progress-section.with-results {
-    margin-bottom: 0;
-    border-bottom: none;
-}
-
-.results-section.with-live-data {
-    border-top: 1px solid #444;
-    padding-top: 20px;
-}
-
-/* Better spacing for the combined view */
-.progress-section + .results-section {
-    margin-top: 0;
-}
-
-/* Hide specific progress elements while keeping the section visible */
-.progress-section .progress-bar.hidden,
-.progress-section #progressMessage.hidden,
-.progress-section .scan-controls.hidden {
-    display: none !important;
-}
-
-.stats-grid {
-    display: grid;
-    grid-template-columns: repeat(auto-fit, minmax(150px, 1fr));
-    gap: 15px;
-    margin-bottom: 20px;
-}
-
-.stat-item {
-    display: flex;
-    justify-content: space-between;
-    align-items: center;
-    padding: 8px 12px;
-    background: rgba(0, 0, 0, 0.5);
-    border: 1px solid #333;
-    border-radius: 2px;
-}
-
-.stat-label {
-    color: #a0a0a0;
-    font-size: 0.9rem;
-}
-
-.stat-value {
-    color: #00ff41;
-    font-weight: bold;
-    font-family: 'Courier New', monospace;
-    transition: background-color 0.3s ease;
-}
-
-/* Animation for final stats highlight */
-@keyframes finalHighlight {
-    0% { background-color: #ff9900; }
-    100% { background-color: transparent; }
-}
-
-.stat-value.final {
-    animation: finalHighlight 2s ease-in-out;
-}
-
-.discoveries-list {
-    margin-top: 20px;
-}
-
-.discoveries-list h4 {
-    color: #ff9900;
-    margin-bottom: 15px;
-    border-bottom: 1px solid #444;
-    padding-bottom: 5px;
-}
-
-.discovery-section {
-    margin-bottom: 15px;
-    padding: 10px;
-    background: rgba(0, 0, 0, 0.3);
-    border: 1px solid #333;
-    border-radius: 2px;
-}
-
-.discovery-section strong {
-    color: #c7c7c7;
-    display: block;
-    margin-bottom: 8px;
-    font-size: 0.9rem;
-}
-
-/* Tactical loading spinner */
-.loading {
-    display: inline-block;
-    width: 20px;
-    height: 20px;
-    border: 3px solid rgba(199, 199, 199, 0.3);
-    border-radius: 50%;
-    border-top-color: #00ff41; /* Night-vision green spinner */
-    animation: spin 1s linear infinite;
-}
-
-@keyframes spin {
-    to { transform: rotate(360deg); }
-}
-
-/* Responsive design adjustments */
-@media (max-width: 768px) {
-    .container {
-        padding: 10px;
-    }
-    
-    header h1 {
-        font-size: 2.2rem;
-    }
-    
-    .scan-form, .progress-section, .results-section {
-        padding: 20px;
-    }
-    
-    .btn-primary, .btn-secondary {
-        width: 100%;
-        margin-right: 0;
-    }
-    
-    .results-controls {
-        display: flex;
-        flex-wrap: wrap;
-        justify-content: center;
-    }
-    
-    .results-controls button {
-        flex: 1;
-        min-width: 120px;
-    }
-    
-    .stats-grid {
-        grid-template-columns: repeat(2, 1fr);
-        gap: 10px;
-    }
-    
-    .stat-item {
-        padding: 6px 8px;
-    }
-    
-    .stat-label, .stat-value {
-        font-size: 0.8rem;
-    }
-    
-    .hostname-list, .ip-list {
-        flex-direction: column;
-        align-items: flex-start;
-    }
-
-    /* Responsive adjustments for the combined view */
-    .results-section .live-discoveries {
-        padding: 15px;
-        margin-bottom: 15px;
-    }
-    
-    .results-section .live-discoveries .stats-grid {
-        grid-template-columns: repeat(2, 1fr);
-        gap: 10px;
-    }
-}
-
-
-/* Add this CSS to the existing style.css file */
-
-/* Graph Section */
-.graph-section {
-    background: #2a2a2a;
-    border-radius: 4px;
-    border: 1px solid #444;
-    box-shadow: inset 0 0 15px rgba(0,0,0,0.5);
-    padding: 30px;
-    margin-bottom: 25px;
-}
-
-.graph-controls {
-    margin-bottom: 20px;
-    text-align: center;
-}
-
-.graph-controls button {
-    margin: 0 5px;
-}
-
-.graph-legend {
-    display: flex;
-    justify-content: space-between;
-    margin-bottom: 20px;
-    padding: 15px;
-    background: rgba(0,0,0,0.3);
-    border-radius: 4px;
-    border: 1px solid #444;
-    flex-wrap: wrap;
-}
-
-.graph-legend h4, .graph-legend h5 {
-    color: #e0e0e0;
-    margin-bottom: 10px;
-    text-transform: uppercase;
-    letter-spacing: 1px;
-}
-
-.legend-items, .legend-methods {
-    display: flex;
-    flex-direction: column;
-    gap: 8px;
-}
-
-.legend-item {
-    display: flex;
-    align-items: center;
-    gap: 10px;
-    font-size: 0.9rem;
-    color: #c7c7c7;
-}
-
-.legend-color {
-    width: 16px;
-    height: 16px;
-    border-radius: 50%;
-    border: 1px solid #666;
-}
-
-.method-item {
-    font-size: 0.9rem;
-    color: #a0a0a0;
-    margin-bottom: 4px;
-}
-
-.graph-container {
-    position: relative;
-    width: 100%;
-    height: 600px;
-    background: #0a0a0a;
-    border-radius: 4px;
-    border: 1px solid #333;
-    margin-bottom: 20px;
-    overflow: hidden;
-}
-
-#discoveryGraph {
-    width: 100%;
-    height: 100%;
-    cursor: grab;
-}
-
-#discoveryGraph:active {
-    cursor: grabbing;
-}
-
-.graph-tooltip {
-    position: absolute;
-    background: rgba(0, 0, 0, 0.9);
-    color: #00ff41;
-    padding: 10px;
-    border-radius: 4px;
-    border: 1px solid #00ff41;
-    font-family: 'Courier New', monospace;
-    font-size: 0.8rem;
-    pointer-events: none;
-    opacity: 0;
-    transition: opacity 0.2s ease;
-    z-index: 1000;
-    max-width: 300px;
-    line-height: 1.4;
-}
-
-.graph-tooltip.visible {
-    opacity: 1;
-}
-
-.graph-info {
-    display: flex;
-    justify-content: space-between;
-    align-items: flex-start;
-    flex-wrap: wrap;
-    gap: 20px;
-}
-
-.graph-stats {
-    display: flex;
-    gap: 20px;
-    flex-wrap: wrap;
-}
-
-.selected-node-info {
-    background: rgba(0, 40, 0, 0.8);
-    border: 1px solid #00ff41;
-    border-radius: 4px;
-    padding: 15px;
-    min-width: 250px;
-}
-
-.selected-node-info h4 {
-    color: #00ff41;
-    margin-bottom: 10px;
-    text-transform: uppercase;
-    letter-spacing: 1px;
-}
-
-#nodeDetails {
-    font-family: 'Courier New', monospace;
-    font-size: 0.8rem;
-    line-height: 1.4;
-    color: #c7c7c7;
-}
-
-#nodeDetails .detail-item {
-    margin-bottom: 8px;
-    border-bottom: 1px solid #333;
-    padding-bottom: 4px;
-}
-
-#nodeDetails .detail-item:last-child {
-    border-bottom: none;
-}
-
-#nodeDetails .detail-label {
-    color: #00ff41;
-    font-weight: bold;
-}
-
-#nodeDetails .detail-value {
-    color: #c7c7c7;
-    margin-left: 10px;
-}
-
-/* Graph Nodes and Links Styling (applied via D3) */
-.graph-node {
-    stroke: #333;
-    stroke-width: 2px;
-    cursor: pointer;
-    transition: all 0.3s ease;
-}
-
-.graph-node:hover {
-    stroke: #fff;
-    stroke-width: 3px;
-}
-
-.graph-node.selected {
-    stroke: #ff9900;
-    stroke-width: 4px;
-}
-
-.graph-link {
-    stroke-opacity: 0.6;
-    stroke-width: 2px;
-    transition: all 0.3s ease;
-}
-
-.graph-link:hover {
-    stroke-opacity: 1;
-    stroke-width: 3px;
-}
-
-.graph-link.highlighted {
-    stroke-opacity: 1;
-    stroke-width: 3px;
-}
-
-.graph-label {
-    font-family: 'Courier New', monospace;
-    font-size: 10px;
-    fill: #c7c7c7;
-    text-anchor: middle;
-    pointer-events: none;
-    opacity: 0.8;
-}
-
-.graph-label.visible {
-    opacity: 1;
-}
-
-/* Responsive adjustments for graph */
-@media (max-width: 768px) {
-    .graph-container {
-        height: 400px;
-    }
-    
-    .graph-legend {
-        flex-direction: column;
-        gap: 15px;
-    }
-    
-    .legend-items, .legend-methods {
-        flex-direction: row;
-        flex-wrap: wrap;
-        gap: 15px;
-    }
-    
-    .graph-info {
-        flex-direction: column;
-        align-items: stretch;
-    }
-    
-    .graph-stats {
-        justify-content: center;
-    }
-    
-    .selected-node-info {
-        min-width: auto;
-        width: 100%;
-    }
-    
-    .graph-tooltip {
-        font-size: 0.7rem;
-        max-width: 250px;
-    }
-}

templates/index.html

@@ -3,117 +3,246 @@
 <head>
     <meta charset="UTF-8">
     <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>DNS Reconnaissance Tool</title>
-    <link rel="stylesheet" href="{{ url_for('static', filename='style.css') }}">
+    <title>DNSRecon - Infrastructure Reconnaissance</title>
+    <link rel="stylesheet" href="{{ url_for('static', filename='css/main.css') }}">
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/vis/4.21.0/vis.min.js"></script>
+    <link href="https://cdnjs.cloudflare.com/ajax/libs/vis/4.21.0/vis.min.css" rel="stylesheet" type="text/css">
+    <link href="https://fonts.googleapis.com/css2?family=Roboto+Mono:wght@300;400;500;700&family=Special+Elite&display=swap" rel="stylesheet">
 </head>
 <body>
     <div class="container">
-        <header>
-            <h1>🔍 DNS Reconnaissance Tool</h1>
-            <p>Comprehensive domain and IP intelligence gathering</p>
+        <header class="header">
+            <div class="header-content">
+                <div class="logo">
+                    <span class="logo-icon">[DNS]</span>
+                    <span class="logo-text">RECON</span>
+                </div>
+                <div class="status-indicator">
+                    <span id="connection-status" class="status-dot"></span>
+                    <span class="status-text">System Online</span>
+                </div>
+            </div>
         </header>
 
-        <div class="scan-form" id="scanForm">
-            <h2>Start New Scan</h2>
-            
-            <div class="form-group">
-                <label for="target">Target (domain.com or hostname):</label>
-                <input type="text" id="target" placeholder="example.com or example" required>
-            </div>
-            
-            <div class="form-group">
-                <label for="maxDepth">Max Recursion Depth:</label>
-                <select id="maxDepth">
-                    <option value="1">1</option>
-                    <option value="2" selected>2</option>
-                    <option value="3">3</option>
-                    <option value="4">4</option>
-                    <option value="5">5</option>
-                </select>
-            </div>
-            
-            <div class="api-keys">
-                <h3>Optional API Keys</h3>
-                <div class="form-group">
-                    <label for="shodanKey">Shodan API Key:</label>
-                    <input type="password" id="shodanKey" placeholder="Optional - for port scanning data">
+        <main class="main-content">
+            <section class="control-panel">
+                <div class="panel-header">
+                    <h2>Target Configuration</h2>
                 </div>
                 
-                <div class="form-group">
-                    <label for="virustotalKey">VirusTotal API Key:</label>
-                    <input type="password" id="virustotalKey" placeholder="Optional - for security analysis">
+                <div class="form-container">
+                    <div class="input-group">
+                        <label for="target-domain">Target Domain</label>
+                        <input type="text" id="target-domain" placeholder="example.com" autocomplete="off">
+                    </div>
+                    
+                    <div class="input-group">
+                        <label for="max-depth">Recursion Depth</label>
+                        <select id="max-depth">
+                            <option value="1">Depth 1 - Direct relationships</option>
+                            <option value="2" selected>Depth 2 - Recommended</option>
+                            <option value="3">Depth 3 - Extended analysis</option>
+                            <option value="4">Depth 4 - Deep reconnaissance</option>
+                            <option value="5">Depth 5 - Maximum depth</option>
+                        </select>
+                    </div>
+                    
+                    <div class="button-group">
+                        <button id="start-scan" class="btn btn-primary">
+                            <span class="btn-icon">[RUN]</span>
+                            <span>Start Reconnaissance</span>
+                        </button>
+                        <button id="add-to-graph" class="btn btn-primary">
+                            <span class="btn-icon">[ADD]</span>
+                            <span>Add to Graph</span>
+                        </button>
+                        <button id="stop-scan" class="btn btn-secondary" disabled>
+                            <span class="btn-icon">[STOP]</span>
+                            <span>Terminate Scan</span>
+                        </button>
+                        <button id="export-results" class="btn btn-secondary">
+                            <span class="btn-icon">[EXPORT]</span>
+                            <span>Download Results</span>
+                        </button>
+                        <button id="configure-api-keys" class="btn btn-secondary">
+                            <span class="btn-icon">[API]</span>
+                            <span>Configure API Keys</span>
+                        </button>
+                    </div>
+                </div>
+            </section>
+
+            <section class="status-panel">
+                <div class="panel-header">
+                    <h2>Reconnaissance Status</h2>
+                </div>
+                
+                <div class="status-content">
+                    <div class="status-row">
+                        <span class="status-label">Current Status:</span>
+                        <span id="scan-status" class="status-value">Idle</span>
+                    </div>
+                    <div class="status-row">
+                        <span class="status-label">Target:</span>
+                        <span id="target-display" class="status-value">None</span>
+                    </div>
+                    <div class="status-row">
+                        <span class="status-label">Depth:</span>
+                        <span id="depth-display" class="status-value">0/0</span>
+                    </div>
+                    <div class="status-row">
+                        <span class="status-label">Relationships:</span>
+                        <span id="relationships-display" class="status-value">0</span>
+                    </div>
+                </div>
+                
+                <div class="progress-container">
+                    <div class="progress-info">
+                        <span id="progress-label">Progress:</span>
+                        <span id="progress-compact">0/0</span>
+                    </div>
+                    <div class="progress-bar">
+                        <div id="progress-fill" class="progress-fill"></div>
+                    </div>
+                </div>
+            </section>
+
+            <section class="visualization-panel">
+                <div class="panel-header">
+                    <h2>Infrastructure Map</h2>
+                    <div class="view-controls">
+                        <div class="filter-group">
+                            <label for="node-type-filter">Node Type:</label>
+                            <select id="node-type-filter">
+                                <option value="all">All</option>
+                                <option value="domain">Domain</option>
+                                <option value="ip">IP</option>
+                                <option value="asn">ASN</option>
+                                <option value="correlation_object">Correlation Object</option>
+                                <option value="large_entity">Large Entity</option>
+                            </select>
+                        </div>
+                        <div class="filter-group">
+                            <label for="confidence-filter">Min Confidence:</label>
+                            <input type="range" id="confidence-filter" min="0" max="1" step="0.1" value="0">
+                            <span id="confidence-value">0</span>
+                        </div>
+                    </div>
+                </div>
+                
+                <div id="network-graph" class="graph-container">
+                    <div class="graph-placeholder">
+                        <div class="placeholder-content">
+                            <div class="placeholder-icon">[○]</div>
+                            <div class="placeholder-text">Infrastructure map will appear here</div>
+                            <div class="placeholder-subtext">Start a reconnaissance scan to visualize relationships</div>
+                        </div>
+                    </div>
+                </div>
+                
+                <div class="legend">
+                    <div class="legend-item">
+                        <div class="legend-color" style="background-color: #00ff41;"></div>
+                        <span>Domains</span>
+                    </div>
+                    <div class="legend-item">
+                        <div class="legend-color" style="background-color: #ff9900;"></div>
+                        <span>IP Addresses</span>
+                    </div>
+                    <div class="legend-item">
+                        <div class="legend-color" style="background-color: #c7c7c7;"></div>
+                        <span>Domain (invalid cert)</span>
+                    </div>
+                    <div class="legend-item">
+                        <div class="legend-color" style="background-color: #9d4edd;"></div>
+                        <span>Correlation Objects</span>
+                    </div>
+                    <div class="legend-item">
+                        <div class="legend-edge high-confidence"></div>
+                        <span>High Confidence</span>
+                    </div>
+                    <div class="legend-item">
+                        <div class="legend-edge medium-confidence"></div>
+                        <span>Medium Confidence</span>
+                    </div>
+                    <div class="legend-item">
+                        <div class="legend-color" style="background-color: #ff6b6b;"></div>
+                        <span>Large Entity</span>
+                    </div>
+                </div>
+            </section>
+
+            <section class="provider-panel">
+                <div class="panel-header">
+                    <h2>Data Providers</h2>
+                </div>
+                
+                <div id="provider-list" class="provider-list">
+                    </div>
+            </section>
+        </main>
+
+        <footer class="footer">
+            <div class="footer-content">
+                <span>v0.0.0rc</span>
+                <span class="footer-separator">|</span>
+                <span>Passive Infrastructure Reconnaissance</span>
+                <span class="footer-separator">|</span>
+                <span id="session-id">Session: Loading...</span>
+            </div>
+        </footer>
+
+        <div id="node-modal" class="modal">
+            <div class="modal-content">
+                <div class="modal-header">
+                    <h3 id="modal-title">Node Details</h3>
+                    <button id="modal-close" class="modal-close">[×]</button>
+                </div>
+                <div class="modal-body">
+                    <div id="modal-details">
+                        </div>
                 </div>
             </div>
-            
-            <button id="startScan" class="btn-primary">Start Reconnaissance</button>
         </div>
 
-        <div class="progress-section" id="progressSection" style="display: none;">
-            <h2>Scan Progress</h2>
-            <div class="progress-bar">
-                <div class="progress-fill" id="progressFill"></div>
-            </div>
-            <p id="progressMessage">Initializing...</p>
-            <div class="scan-controls">
-                <button id="newScan" class="btn-secondary">New Scan</button>
-            </div>
-        </div>
-
-        <div class="results-section" id="resultsSection" style="display: none;">
-            <h2>Reconnaissance Results</h2>
-            
-            <div class="results-controls">
-                <button id="showJson" class="btn-secondary">Show JSON</button>
-                <button id="showText" class="btn-secondary active">Show Text Report</button>
-                <button id="showGraphView" class="btn-secondary">Show Graph</button>
-                <button id="downloadJson" class="btn-secondary">Download JSON</button>
-                <button id="downloadText" class="btn-secondary">Download Text</button>
-            </div>
-            
-            <div class="report-container">
-                <pre id="reportContent"></pre>
-            </div>
-        </div>
-
-        <div class="graph-section" id="graphSection" style="display: none;">
-            <h2>Discovery Graph</h2>
-            
-            <div class="graph-stats">
-                <div class="stat-item">
-                    <span class="stat-label">Nodes:</span>
-                    <span id="graphNodes" class="stat-value">0</span>
+        <div id="api-key-modal" class="modal">
+            <div class="modal-content">
+                <div class="modal-header">
+                    <h3>Configure API Keys</h3>
+                    <button id="api-key-modal-close" class="modal-close">[×]</button>
                 </div>
-                <div class="stat-item">
-                    <span class="stat-label">Edges:</span>
-                    <span id="graphEdges" class="stat-value">0</span>
-                </div>
-                <div class="stat-item">
-                    <span class="stat-label">Max Depth:</span>
-                    <span id="graphDepth" class="stat-value">0</span>
+                <div class="modal-body">
+                    <p class="modal-description">
+                        Enter your API keys for enhanced data providers. Keys are stored in memory for the current session only and are never saved to disk.
+                    </p>
+                    <div id="api-key-inputs">
+                        </div>
+                    <div class="button-group" style="flex-direction: row; justify-content: flex-end;">
+                        <button id="reset-api-keys" class="btn btn-secondary">
+                            <span>Reset</span>
+                        </button>
+                        <button id="save-api-keys" class="btn btn-primary">
+                            <span>Save Keys</span>
+                        </button>
+                    </div>
                 </div>
             </div>
-
-            <div class="graph-controls">
-                <button id="showGraph" class="btn-secondary active">Show Graph</button>
-                <button id="hideGraph" class="btn-secondary">Hide Graph</button>
-                <button id="resetZoom" class="btn-secondary">Reset Zoom</button>
-                <button id="toggleLabels" class="btn-secondary">Show Labels</button>
-            </div>
-            
-            <div class="graph-display-area">
-                <svg id="discoveryGraph"></svg>
-                <div id="graphTooltip" class="graph-tooltip"></div>
-            </div>
-
-            <div class="selected-node-details" id="selectedNodeInfo" style="display: none;">
-                <h3>Selected Node Details</h3>
-                <div id="nodeDetails"></div>
-            </div>
-        </div>
         </div>
+    </div>
 
-    <script src="https://cdnjs.cloudflare.com/ajax/libs/d3/7.8.5/d3.min.js"></script>
-    <script src="{{ url_for('static', filename='script.js') }}"></script>
-
+    <script>
+        function copyToClipboard(elementId) {
+            const element = document.getElementById(elementId);
+            const textToCopy = element.innerText;
+            navigator.clipboard.writeText(textToCopy).then(() => {
+                // Optional: Show a success message
+                console.log('Copied to clipboard');
+            }).catch(err => {
+                console.error('Failed to copy: ', err);
+            });
+        }
+    </script>
+    <script src="{{ url_for('static', filename='js/graph.js') }}"></script>
+    <script src="{{ url_for('static', filename='js/main.js') }}"></script>
 </body>
 </html>

utils/helpers.py

@@ -0,0 +1,50 @@
+def _is_valid_domain(domain: str) -> bool:
+    """
+    Basic domain validation.
+
+    Args:
+        domain: Domain string to validate
+
+    Returns:
+        True if domain appears valid
+    """
+    if not domain or len(domain) > 253:
+        return False
+
+    # Check for valid characters and structure
+    parts = domain.split('.')
+    if len(parts) < 2:
+        return False
+
+    for part in parts:
+        if not part or len(part) > 63:
+            return False
+        if not part.replace('-', '').replace('_', '').isalnum():
+            return False
+
+    return True
+
+def _is_valid_ip(ip: str) -> bool:
+    """
+    Basic IP address validation.
+
+    Args:
+        ip: IP address string to validate
+
+    Returns:
+        True if IP appears valid
+    """
+    try:
+        parts = ip.split('.')
+        if len(parts) != 4:
+            return False
+
+        for part in parts:
+            num = int(part)
+            if not 0 <= num <= 255:
+                return False
+
+        return True
+
+    except (ValueError, AttributeError):
+        return False