mstoeck3/dnscope
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

data-model #2

Merged
mstoeck3 merged 20 commits from data-model into main 2025-09-17 21:56:18 +00:00
Conversation 0 Commits 20 Files Changed 19 +3940 -4311
19 changed files with 3940 additions and 4311 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.env.example
View File

@ -25,10 +25,10 @@ DEFAULT_RECURSION_DEPTH=2
# Default timeout for provider API requests in seconds. # Default timeout for provider API requests in seconds.
DEFAULT_TIMEOUT=30 DEFAULT_TIMEOUT=30
# The number of concurrent provider requests to make. # The number of concurrent provider requests to make.
MAX_CONCURRENT_REQUESTS=5 MAX_CONCURRENT_REQUESTS=1
# The number of results from a provider that triggers the "large entity" grouping. # The number of results from a provider that triggers the "large entity" grouping.
LARGE_ENTITY_THRESHOLD=100 LARGE_ENTITY_THRESHOLD=100
# The number of times to retry a target if a provider fails. # The number of times to retry a target if a provider fails.
MAX_RETRIES_PER_TARGET=8 MAX_RETRIES_PER_TARGET=8
# How long cached provider responses are stored (in hours). # How long cached provider responses are stored (in hours).
CACHE_EXPIRY_HOURS=12 CACHE_TIMEOUT_HOURS=12

473
app.py
View File

@ -10,46 +10,63 @@ import traceback
from flask import Flask, render_template, request, jsonify, send_file, session from flask import Flask, render_template, request, jsonify, send_file, session
from datetime import datetime, timezone, timedelta from datetime import datetime, timezone, timedelta
import io import io
import os
from core.session_manager import session_manager from core.session_manager import session_manager
from config import config from config import config
from core.graph_manager import NodeType from core.graph_manager import NodeType
from utils.helpers import is_valid_target from utils.helpers import is_valid_target
from decimal import Decimal
app = Flask(__name__) app = Flask(__name__)
# Use centralized configuration for Flask settings
app.config['SECRET_KEY'] = config.flask_secret_key app.config['SECRET_KEY'] = config.flask_secret_key
app.config['PERMANENT_SESSION_LIFETIME'] = timedelta(hours=config.flask_permanent_session_lifetime_hours) app.config['PERMANENT_SESSION_LIFETIME'] = timedelta(hours=config.flask_permanent_session_lifetime_hours)
def get_user_scanner(): def get_user_scanner():
""" """
Retrieves the scanner for the current session, or creates a new Retrieves the scanner for the current session, or creates a new one if none exists.
session and scanner if one doesn't exist.
""" """
# Get current Flask session info for debugging
current_flask_session_id = session.get('dnsrecon_session_id') current_flask_session_id = session.get('dnsrecon_session_id')
# Try to get existing session
if current_flask_session_id: if current_flask_session_id:
existing_scanner = session_manager.get_session(current_flask_session_id) existing_scanner = session_manager.get_session(current_flask_session_id)
if existing_scanner: if existing_scanner:
return current_flask_session_id, existing_scanner return current_flask_session_id, existing_scanner
# Create new session if none exists
print("Creating new session as none was found...")
new_session_id = session_manager.create_session() new_session_id = session_manager.create_session()
new_scanner = session_manager.get_session(new_session_id) new_scanner = session_manager.get_session(new_session_id)
if not new_scanner: if not new_scanner:
raise Exception("Failed to create new scanner session") raise Exception("Failed to create new scanner session")
# Store in Flask session
session['dnsrecon_session_id'] = new_session_id session['dnsrecon_session_id'] = new_session_id
session.permanent = True session.permanent = True
return new_session_id, new_scanner return new_session_id, new_scanner
class CustomJSONEncoder(json.JSONEncoder):
"""Custom JSON encoder to handle non-serializable objects."""
def default(self, obj):
if isinstance(obj, datetime):
return obj.isoformat()
elif isinstance(obj, set):
return list(obj)
elif isinstance(obj, Decimal):
return float(obj)
elif hasattr(obj, '__dict__'):
# For custom objects, try to serialize their dict representation
try:
return obj.__dict__
except:
return str(obj)
elif hasattr(obj, 'value') and hasattr(obj, 'name'):
# For enum objects
return obj.value
else:
# For any other non-serializable object, convert to string
return str(obj)
@app.route('/') @app.route('/')
def index(): def index():
"""Serve the main web interface.""" """Serve the main web interface."""
@ -59,11 +76,8 @@ def index():
@app.route('/api/scan/start', methods=['POST']) @app.route('/api/scan/start', methods=['POST'])
def start_scan(): def start_scan():
""" """
Start a new reconnaissance scan. Creates a new isolated scanner if Starts a new reconnaissance scan.
clear_graph is true, otherwise adds to the existing one.
""" """
print("=== API: /api/scan/start called ===")
try: try:
data = request.get_json() data = request.get_json()
if not data or 'target' not in data: if not data or 'target' not in data:
@ -72,47 +86,28 @@ def start_scan():
target = data['target'].strip() target = data['target'].strip()
max_depth = data.get('max_depth', config.default_recursion_depth) max_depth = data.get('max_depth', config.default_recursion_depth)
clear_graph = data.get('clear_graph', True) clear_graph = data.get('clear_graph', True)
force_rescan_target = data.get('force_rescan_target', None) # **FIX**: Get the new parameter force_rescan_target = data.get('force_rescan_target', None)
print(f"Parsed - target: '{target}', max_depth: {max_depth}, clear_graph: {clear_graph}, force_rescan: {force_rescan_target}")
# Validation
if not target: if not target:
return jsonify({'success': False, 'error': 'Target cannot be empty'}), 400 return jsonify({'success': False, 'error': 'Target cannot be empty'}), 400
if not is_valid_target(target): if not is_valid_target(target):
return jsonify({'success': False, 'error': 'Invalid target format. Please enter a valid domain or IP address.'}), 400 return jsonify({'success': False, 'error': 'Invalid target format.'}), 400
if not isinstance(max_depth, int) or not 1 <= max_depth <= 5: if not isinstance(max_depth, int) or not 1 <= max_depth <= 5:
return jsonify({'success': False, 'error': 'Max depth must be an integer between 1 and 5'}), 400 return jsonify({'success': False, 'error': 'Max depth must be an integer between 1 and 5'}), 400
user_session_id, scanner = None, None user_session_id, scanner = get_user_scanner()
if clear_graph:
print("Clear graph requested: Creating a new, isolated scanner session.")
old_session_id = session.get('dnsrecon_session_id')
if old_session_id:
session_manager.terminate_session(old_session_id)
user_session_id = session_manager.create_session()
session['dnsrecon_session_id'] = user_session_id
session.permanent = True
scanner = session_manager.get_session(user_session_id)
else:
print("Adding to existing graph: Reusing the current scanner session.")
user_session_id, scanner = get_user_scanner()
if not scanner: if not scanner:
return jsonify({'success': False, 'error': 'Failed to get or create a scanner instance.'}), 500 return jsonify({'success': False, 'error': 'Failed to get scanner instance.'}), 500
print(f"Using scanner {id(scanner)} in session {user_session_id}") success = scanner.start_scan(target, max_depth, clear_graph=clear_graph, force_rescan_target=force_rescan_target)
success = scanner.start_scan(target, max_depth, clear_graph=clear_graph, force_rescan_target=force_rescan_target) # **FIX**: Pass the new parameter
if success: if success:
return jsonify({ return jsonify({
'success': True, 'success': True,
'message': 'Scan started successfully', 'message': 'Scan started successfully',
'scan_id': scanner.logger.session_id, 'scan_id': scanner.logger.session_id,
'user_session_id': user_session_id, 'user_session_id': user_session_id
}) })
else: else:
return jsonify({ return jsonify({
@ -121,170 +116,98 @@ def start_scan():
}), 409 }), 409
except Exception as e: except Exception as e:
print(f"ERROR: Exception in start_scan endpoint: {e}")
traceback.print_exc() traceback.print_exc()
return jsonify({'success': False, 'error': f'Internal server error: {str(e)}'}), 500 return jsonify({'success': False, 'error': f'Internal server error: {str(e)}'}), 500
@app.route('/api/scan/stop', methods=['POST']) @app.route('/api/scan/stop', methods=['POST'])
def stop_scan(): def stop_scan():
"""Stop the current scan with immediate GUI feedback.""" """Stop the current scan."""
print("=== API: /api/scan/stop called ===")
try: try:
# Get user-specific scanner
user_session_id, scanner = get_user_scanner() user_session_id, scanner = get_user_scanner()
print(f"Stopping scan for session: {user_session_id}")
if not scanner: if not scanner:
return jsonify({ return jsonify({'success': False, 'error': 'No scanner found for session'}), 404
'success': False,
'error': 'No scanner found for session'
}), 404
# Ensure session ID is set
if not scanner.session_id: if not scanner.session_id:
scanner.session_id = user_session_id scanner.session_id = user_session_id
# Use the stop mechanism scanner.stop_scan()
success = scanner.stop_scan()
# Also set the Redis stop signal directly for extra reliability
session_manager.set_stop_signal(user_session_id) session_manager.set_stop_signal(user_session_id)
# Force immediate status update
session_manager.update_scanner_status(user_session_id, 'stopped') session_manager.update_scanner_status(user_session_id, 'stopped')
# Update the full scanner state
session_manager.update_session_scanner(user_session_id, scanner) session_manager.update_session_scanner(user_session_id, scanner)
print(f"Stop scan completed. Success: {success}, Scanner status: {scanner.status}")
return jsonify({ return jsonify({
'success': True, 'success': True,
'message': 'Scan stop requested - termination initiated', 'message': 'Scan stop requested',
'user_session_id': user_session_id, 'user_session_id': user_session_id
'scanner_status': scanner.status,
'stop_method': 'cross_process'
}) })
except Exception as e: except Exception as e:
print(f"ERROR: Exception in stop_scan endpoint: {e}")
traceback.print_exc() traceback.print_exc()
return jsonify({ return jsonify({'success': False, 'error': f'Internal server error: {str(e)}'}), 500
'success': False,
'error': f'Internal server error: {str(e)}'
}), 500
@app.route('/api/scan/status', methods=['GET']) @app.route('/api/scan/status', methods=['GET'])
def get_scan_status(): def get_scan_status():
"""Get current scan status with error handling.""" """Get current scan status."""
try: try:
# Get user-specific scanner
user_session_id, scanner = get_user_scanner() user_session_id, scanner = get_user_scanner()
if not scanner: if not scanner:
# Return default idle status if no scanner
return jsonify({ return jsonify({
'success': True, 'success': True,
'status': { 'status': {
'status': 'idle', 'status': 'idle', 'target_domain': None, 'current_depth': 0,
'target_domain': None, 'max_depth': 0, 'progress_percentage': 0.0,
'current_depth': 0,
'max_depth': 0,
'current_indicator': '',
'total_indicators_found': 0,
'indicators_processed': 0,
'progress_percentage': 0.0,
'enabled_providers': [],
'graph_statistics': {},
'user_session_id': user_session_id 'user_session_id': user_session_id
} }
}) })
# Ensure session ID is set
if not scanner.session_id: if not scanner.session_id:
scanner.session_id = user_session_id scanner.session_id = user_session_id
status = scanner.get_scan_status() status = scanner.get_scan_status()
status['user_session_id'] = user_session_id status['user_session_id'] = user_session_id
# Additional debug info return jsonify({'success': True, 'status': status})
status['debug_info'] = {
'scanner_object_id': id(scanner),
'session_id_set': bool(scanner.session_id),
'has_scan_thread': bool(scanner.scan_thread and scanner.scan_thread.is_alive())
}
return jsonify({
'success': True,
'status': status
})
except Exception as e: except Exception as e:
print(f"ERROR: Exception in get_scan_status endpoint: {e}")
traceback.print_exc() traceback.print_exc()
return jsonify({ return jsonify({
'success': False, 'success': False, 'error': f'Internal server error: {str(e)}',
'error': f'Internal server error: {str(e)}', 'fallback_status': {'status': 'error', 'progress_percentage': 0.0}
'fallback_status': {
'status': 'error',
'target_domain': None,
'current_depth': 0,
'max_depth': 0,
'progress_percentage': 0.0
}
}), 500 }), 500
@app.route('/api/graph', methods=['GET']) @app.route('/api/graph', methods=['GET'])
def get_graph_data(): def get_graph_data():
"""Get current graph data with error handling.""" """Get current graph data."""
try: try:
# Get user-specific scanner
user_session_id, scanner = get_user_scanner() user_session_id, scanner = get_user_scanner()
if not scanner: empty_graph = {
# Return empty graph if no scanner 'nodes': [], 'edges': [],
return jsonify({ 'statistics': {'node_count': 0, 'edge_count': 0}
'success': True, }
'graph': {
'nodes': [],
'edges': [],
'statistics': {
'node_count': 0,
'edge_count': 0,
'creation_time': datetime.now(timezone.utc).isoformat(),
'last_modified': datetime.now(timezone.utc).isoformat()
}
},
'user_session_id': user_session_id
})
graph_data = scanner.get_graph_data() if not scanner:
return jsonify({ return jsonify({'success': True, 'graph': empty_graph, 'user_session_id': user_session_id})
'success': True,
'graph': graph_data, graph_data = scanner.get_graph_data() or empty_graph
'user_session_id': user_session_id
}) return jsonify({'success': True, 'graph': graph_data, 'user_session_id': user_session_id})
except Exception as e: except Exception as e:
print(f"ERROR: Exception in get_graph_data endpoint: {e}")
traceback.print_exc() traceback.print_exc()
return jsonify({ return jsonify({
'success': False, 'success': False, 'error': f'Internal server error: {str(e)}',
'error': f'Internal server error: {str(e)}', 'fallback_graph': {'nodes': [], 'edges': [], 'statistics': {}}
'fallback_graph': {
'nodes': [],
'edges': [],
'statistics': {'node_count': 0, 'edge_count': 0}
}
}), 500 }), 500
@app.route('/api/graph/large-entity/extract', methods=['POST']) @app.route('/api/graph/large-entity/extract', methods=['POST'])
def extract_from_large_entity(): def extract_from_large_entity():
"""Extract a node from a large entity, making it a standalone node.""" """Extract a node from a large entity."""
try: try:
data = request.get_json() data = request.get_json()
large_entity_id = data.get('large_entity_id') large_entity_id = data.get('large_entity_id')
@ -306,13 +229,12 @@ def extract_from_large_entity():
return jsonify({'success': False, 'error': f'Failed to extract node {node_id}.'}), 500 return jsonify({'success': False, 'error': f'Failed to extract node {node_id}.'}), 500
except Exception as e: except Exception as e:
print(f"ERROR: Exception in extract_from_large_entity endpoint: {e}")
traceback.print_exc() traceback.print_exc()
return jsonify({'success': False, 'error': f'Internal server error: {str(e)}'}), 500 return jsonify({'success': False, 'error': f'Internal server error: {str(e)}'}), 500
@app.route('/api/graph/node/<node_id>', methods=['DELETE']) @app.route('/api/graph/node/<node_id>', methods=['DELETE'])
def delete_graph_node(node_id): def delete_graph_node(node_id):
"""Delete a node from the graph for the current user session.""" """Delete a node from the graph."""
try: try:
user_session_id, scanner = get_user_scanner() user_session_id, scanner = get_user_scanner()
if not scanner: if not scanner:
@ -321,14 +243,12 @@ def delete_graph_node(node_id):
success = scanner.graph.remove_node(node_id) success = scanner.graph.remove_node(node_id)
if success: if success:
# Persist the change
session_manager.update_session_scanner(user_session_id, scanner) session_manager.update_session_scanner(user_session_id, scanner)
return jsonify({'success': True, 'message': f'Node {node_id} deleted successfully.'}) return jsonify({'success': True, 'message': f'Node {node_id} deleted successfully.'})
else: else:
return jsonify({'success': False, 'error': f'Node {node_id} not found in graph.'}), 404 return jsonify({'success': False, 'error': f'Node {node_id} not found.'}), 404
except Exception as e: except Exception as e:
print(f"ERROR: Exception in delete_graph_node endpoint: {e}")
traceback.print_exc() traceback.print_exc()
return jsonify({'success': False, 'error': f'Internal server error: {str(e)}'}), 500 return jsonify({'success': False, 'error': f'Internal server error: {str(e)}'}), 500
@ -349,7 +269,6 @@ def revert_graph_action():
action_data = data['data'] action_data = data['data']
if action_type == 'delete': if action_type == 'delete':
# Re-add the node
node_to_add = action_data.get('node') node_to_add = action_data.get('node')
if node_to_add: if node_to_add:
scanner.graph.add_node( scanner.graph.add_node(
@ -360,56 +279,73 @@ def revert_graph_action():
metadata=node_to_add.get('metadata') metadata=node_to_add.get('metadata')
) )
# Re-add the edges
edges_to_add = action_data.get('edges', []) edges_to_add = action_data.get('edges', [])
for edge in edges_to_add: for edge in edges_to_add:
# Add edge only if both nodes exist to prevent errors
if scanner.graph.graph.has_node(edge['from']) and scanner.graph.graph.has_node(edge['to']): if scanner.graph.graph.has_node(edge['from']) and scanner.graph.graph.has_node(edge['to']):
scanner.graph.add_edge( scanner.graph.add_edge(
source_id=edge['from'], source_id=edge['from'], target_id=edge['to'],
target_id=edge['to'],
relationship_type=edge['metadata']['relationship_type'], relationship_type=edge['metadata']['relationship_type'],
confidence_score=edge['metadata']['confidence_score'], confidence_score=edge['metadata']['confidence_score'],
source_provider=edge['metadata']['source_provider'], source_provider=edge['metadata']['source_provider'],
raw_data=edge.get('raw_data', {}) raw_data=edge.get('raw_data', {})
) )
# Persist the change
session_manager.update_session_scanner(user_session_id, scanner) session_manager.update_session_scanner(user_session_id, scanner)
return jsonify({'success': True, 'message': 'Delete action reverted successfully.'}) return jsonify({'success': True, 'message': 'Delete action reverted successfully.'})
return jsonify({'success': False, 'error': f'Unknown revert action type: {action_type}'}), 400 return jsonify({'success': False, 'error': f'Unknown revert action type: {action_type}'}), 400
except Exception as e: except Exception as e:
print(f"ERROR: Exception in revert_graph_action endpoint: {e}")
traceback.print_exc() traceback.print_exc()
return jsonify({'success': False, 'error': f'Internal server error: {str(e)}'}), 500 return jsonify({'success': False, 'error': f'Internal server error: {str(e)}'}), 500
@app.route('/api/export', methods=['GET']) @app.route('/api/export', methods=['GET'])
def export_results(): def export_results():
"""Export complete scan results as downloadable JSON for the user session.""" """Export scan results as a JSON file with improved error handling."""
try: try:
# Get user-specific scanner
user_session_id, scanner = get_user_scanner() user_session_id, scanner = get_user_scanner()
# Get complete results if not scanner:
results = scanner.export_results() return jsonify({'success': False, 'error': 'No active scanner session found'}), 404
# Add session information to export # Get export data with error handling
try:
results = scanner.export_results()
except Exception as e:
return jsonify({'success': False, 'error': f'Failed to gather export data: {str(e)}'}), 500
# Add export metadata
results['export_metadata'] = { results['export_metadata'] = {
'user_session_id': user_session_id, 'user_session_id': user_session_id,
'export_timestamp': datetime.now(timezone.utc).isoformat(), 'export_timestamp': datetime.now(timezone.utc).isoformat(),
'export_type': 'user_session_results' 'export_version': '1.0.0',
'forensic_integrity': 'maintained'
} }
# Create filename with timestamp # Generate filename with forensic naming convention
timestamp = datetime.now(timezone.utc).strftime('%Y%m%d_%H%M%S') timestamp = datetime.now(timezone.utc).strftime('%Y%m%d_%H%M%S')
target = scanner.current_target or 'unknown' target = scanner.current_target or 'unknown'
filename = f"dnsrecon_{target}_{timestamp}_{user_session_id[:8]}.json" # Sanitize target for filename
safe_target = "".join(c for c in target if c.isalnum() or c in ('-', '_', '.')).rstrip()
filename = f"dnsrecon_{safe_target}_{timestamp}.json"
# Create in-memory file # Serialize with custom encoder and error handling
json_data = json.dumps(results, indent=2, ensure_ascii=False) try:
json_data = json.dumps(results, indent=2, cls=CustomJSONEncoder, ensure_ascii=False)
except Exception as e:
# If custom encoder fails, try a more aggressive approach
try:
# Convert problematic objects to strings recursively
cleaned_results = _clean_for_json(results)
json_data = json.dumps(cleaned_results, indent=2, ensure_ascii=False)
except Exception as e2:
return jsonify({
'success': False,
'error': f'JSON serialization failed: {str(e2)}'
}), 500
# Create file object
file_obj = io.BytesIO(json_data.encode('utf-8')) file_obj = io.BytesIO(json_data.encode('utf-8'))
return send_file( return send_file(
@ -420,71 +356,70 @@ def export_results():
) )
except Exception as e: except Exception as e:
print(f"ERROR: Exception in export_results endpoint: {e}")
traceback.print_exc() traceback.print_exc()
return jsonify({ return jsonify({
'success': False, 'success': False,
'error': f'Export failed: {str(e)}' 'error': f'Export failed: {str(e)}',
'error_type': type(e).__name__
}), 500 }), 500
def _clean_for_json(obj, max_depth=10, current_depth=0):
"""
Recursively clean an object to make it JSON serializable.
Handles circular references and problematic object types.
"""
if current_depth > max_depth:
return f"<max_depth_exceeded_{type(obj).__name__}>"
@app.route('/api/providers', methods=['GET']) if obj is None or isinstance(obj, (bool, int, float, str)):
def get_providers(): return obj
"""Get information about available providers for the user session.""" elif isinstance(obj, datetime):
return obj.isoformat()
try: elif isinstance(obj, (set, frozenset)):
# Get user-specific scanner return list(obj)
user_session_id, scanner = get_user_scanner() elif isinstance(obj, dict):
cleaned = {}
if scanner: for key, value in obj.items():
# Updated debug print to be consistent with the new progress bar logic try:
completed_tasks = scanner.indicators_completed # Ensure key is string
total_tasks = scanner.total_tasks_ever_enqueued clean_key = str(key) if not isinstance(key, str) else key
print(f"DEBUG: Task Progress - Completed: {completed_tasks}, Total Enqueued: {total_tasks}") cleaned[clean_key] = _clean_for_json(value, max_depth, current_depth + 1)
else: except Exception:
print("DEBUG: No active scanner session found.") cleaned[str(key)] = f"<serialization_error_{type(value).__name__}>"
return cleaned
provider_info = scanner.get_provider_info() elif isinstance(obj, (list, tuple)):
cleaned = []
return jsonify({ for item in obj:
'success': True, try:
'providers': provider_info, cleaned.append(_clean_for_json(item, max_depth, current_depth + 1))
'user_session_id': user_session_id except Exception:
}) cleaned.append(f"<serialization_error_{type(item).__name__}>")
return cleaned
except Exception as e: elif hasattr(obj, '__dict__'):
print(f"ERROR: Exception in get_providers endpoint: {e}") try:
traceback.print_exc() return _clean_for_json(obj.__dict__, max_depth, current_depth + 1)
return jsonify({ except Exception:
'success': False, return str(obj)
'error': f'Internal server error: {str(e)}' elif hasattr(obj, 'value'):
}), 500 # For enum-like objects
return obj.value
else:
return str(obj)
@app.route('/api/config/api-keys', methods=['POST']) @app.route('/api/config/api-keys', methods=['POST'])
def set_api_keys(): def set_api_keys():
""" """Set API keys for the current session."""
Set API keys for providers for the user session only.
"""
try: try:
data = request.get_json() data = request.get_json()
if data is None: if data is None:
return jsonify({ return jsonify({'success': False, 'error': 'No API keys provided'}), 400
'success': False,
'error': 'No API keys provided'
}), 400
# Get user-specific scanner and config
user_session_id, scanner = get_user_scanner() user_session_id, scanner = get_user_scanner()
session_config = scanner.config session_config = scanner.config
updated_providers = [] updated_providers = []
# Iterate over the API keys provided in the request data
for provider_name, api_key in data.items(): for provider_name, api_key in data.items():
# This allows us to both set and clear keys. The config
# handles enabling/disabling based on if the key is empty.
api_key_value = str(api_key or '').strip() api_key_value = str(api_key or '').strip()
success = session_config.set_api_key(provider_name.lower(), api_key_value) success = session_config.set_api_key(provider_name.lower(), api_key_value)
@ -492,60 +427,136 @@ def set_api_keys():
updated_providers.append(provider_name) updated_providers.append(provider_name)
if updated_providers: if updated_providers:
# Reinitialize scanner providers to apply the new keys
scanner._initialize_providers() scanner._initialize_providers()
# Persist the updated scanner object back to the user's session
session_manager.update_session_scanner(user_session_id, scanner) session_manager.update_session_scanner(user_session_id, scanner)
return jsonify({ return jsonify({
'success': True, 'success': True,
'message': f'API keys updated for session {user_session_id}: {", ".join(updated_providers)}', 'message': f'API keys updated for: {", ".join(updated_providers)}',
'updated_providers': updated_providers,
'user_session_id': user_session_id 'user_session_id': user_session_id
}) })
else: else:
return jsonify({ return jsonify({'success': False, 'error': 'No valid API keys were provided.'}), 400
'success': False,
'error': 'No valid API keys were provided or provider names were incorrect.'
}), 400
except Exception as e: except Exception as e:
print(f"ERROR: Exception in set_api_keys endpoint: {e}")
traceback.print_exc() traceback.print_exc()
return jsonify({'success': False, 'error': f'Internal server error: {str(e)}'}), 500
@app.route('/api/providers', methods=['GET'])
def get_providers():
"""Get enhanced information about available providers including API key sources."""
try:
user_session_id, scanner = get_user_scanner()
base_provider_info = scanner.get_provider_info()
# Enhance provider info with API key source information
enhanced_provider_info = {}
for provider_name, info in base_provider_info.items():
enhanced_info = dict(info) # Copy base info
if info['requires_api_key']:
# Determine API key source and configuration status
api_key = scanner.config.get_api_key(provider_name)
backend_api_key = os.getenv(f'{provider_name.upper()}_API_KEY')
if backend_api_key:
# API key configured via backend/environment
enhanced_info.update({
'api_key_configured': True,
'api_key_source': 'backend',
'api_key_help': f'API key configured via environment variable {provider_name.upper()}_API_KEY'
})
elif api_key:
# API key configured via web interface
enhanced_info.update({
'api_key_configured': True,
'api_key_source': 'frontend',
'api_key_help': f'API key set via web interface (session-only)'
})
else:
# No API key configured
enhanced_info.update({
'api_key_configured': False,
'api_key_source': None,
'api_key_help': f'Requires API key to enable {info["display_name"]} integration'
})
else:
# Provider doesn't require API key
enhanced_info.update({
'api_key_configured': True, # Always "configured" for non-API providers
'api_key_source': None,
'api_key_help': None
})
enhanced_provider_info[provider_name] = enhanced_info
return jsonify({ return jsonify({
'success': False, 'success': True,
'error': f'Internal server error: {str(e)}' 'providers': enhanced_provider_info,
}), 500 'user_session_id': user_session_id
})
except Exception as e:
traceback.print_exc()
return jsonify({'success': False, 'error': f'Internal server error: {str(e)}'}), 500
@app.route('/api/config/providers', methods=['POST'])
def configure_providers():
"""Configure provider settings (enable/disable)."""
try:
data = request.get_json()
if data is None:
return jsonify({'success': False, 'error': 'No provider settings provided'}), 400
user_session_id, scanner = get_user_scanner()
session_config = scanner.config
updated_providers = []
for provider_name, settings in data.items():
provider_name_clean = provider_name.lower().strip()
if 'enabled' in settings:
# Update the enabled state in session config
session_config.enabled_providers[provider_name_clean] = settings['enabled']
updated_providers.append(provider_name_clean)
if updated_providers:
# Reinitialize providers with new settings
scanner._initialize_providers()
session_manager.update_session_scanner(user_session_id, scanner)
return jsonify({
'success': True,
'message': f'Provider settings updated for: {", ".join(updated_providers)}',
'user_session_id': user_session_id
})
else:
return jsonify({'success': False, 'error': 'No valid provider settings were provided.'}), 400
except Exception as e:
traceback.print_exc()
return jsonify({'success': False, 'error': f'Internal server error: {str(e)}'}), 500
@app.errorhandler(404) @app.errorhandler(404)
def not_found(error): def not_found(error):
"""Handle 404 errors.""" """Handle 404 errors."""
return jsonify({ return jsonify({'success': False, 'error': 'Endpoint not found'}), 404
'success': False,
'error': 'Endpoint not found'
}), 404
@app.errorhandler(500) @app.errorhandler(500)
def internal_error(error): def internal_error(error):
"""Handle 500 errors.""" """Handle 500 errors."""
print(f"ERROR: 500 Internal Server Error: {error}")
traceback.print_exc() traceback.print_exc()
return jsonify({ return jsonify({'success': False, 'error': 'Internal server error'}), 500
'success': False,
'error': 'Internal server error'
}), 500
if __name__ == '__main__': if __name__ == '__main__':
print("Starting DNSRecon Flask application with user session support...")
# Load configuration from environment
config.load_from_env() config.load_from_env()
# Start Flask application
print(f"Starting server on {config.flask_host}:{config.flask_port}")
app.run( app.run(
host=config.flask_host, host=config.flask_host,
port=config.flask_port, port=config.flask_port,

60
config.py
View File

@ -21,11 +21,10 @@ class Config:
# --- General Settings --- # --- General Settings ---
self.default_recursion_depth = 2 self.default_recursion_depth = 2
self.default_timeout = 30 self.default_timeout = 60
self.max_concurrent_requests = 5 self.max_concurrent_requests = 1
self.large_entity_threshold = 100 self.large_entity_threshold = 100
self.max_retries_per_target = 8 self.max_retries_per_target = 8
self.cache_expiry_hours = 12
# --- Provider Caching Settings --- # --- Provider Caching Settings ---
self.cache_timeout_hours = 6 # Provider-specific cache timeout self.cache_timeout_hours = 6 # Provider-specific cache timeout
@ -69,7 +68,6 @@ class Config:
self.max_concurrent_requests = int(os.getenv('MAX_CONCURRENT_REQUESTS', self.max_concurrent_requests)) self.max_concurrent_requests = int(os.getenv('MAX_CONCURRENT_REQUESTS', self.max_concurrent_requests))
self.large_entity_threshold = int(os.getenv('LARGE_ENTITY_THRESHOLD', self.large_entity_threshold)) self.large_entity_threshold = int(os.getenv('LARGE_ENTITY_THRESHOLD', self.large_entity_threshold))
self.max_retries_per_target = int(os.getenv('MAX_RETRIES_PER_TARGET', self.max_retries_per_target)) self.max_retries_per_target = int(os.getenv('MAX_RETRIES_PER_TARGET', self.max_retries_per_target))
self.cache_expiry_hours = int(os.getenv('CACHE_EXPIRY_HOURS', self.cache_expiry_hours))
self.cache_timeout_hours = int(os.getenv('CACHE_TIMEOUT_HOURS', self.cache_timeout_hours)) self.cache_timeout_hours = int(os.getenv('CACHE_TIMEOUT_HOURS', self.cache_timeout_hours))
# Override Flask and session settings # Override Flask and session settings
@ -87,6 +85,60 @@ class Config:
self.enabled_providers[provider] = True self.enabled_providers[provider] = True
return True return True
def set_provider_enabled(self, provider: str, enabled: bool) -> bool:
"""
Set provider enabled status for the session.
Args:
provider: Provider name
enabled: Whether the provider should be enabled
Returns:
True if the setting was applied successfully
"""
provider_key = provider.lower()
self.enabled_providers[provider_key] = enabled
return True
def get_provider_enabled(self, provider: str) -> bool:
"""
Get provider enabled status.
Args:
provider: Provider name
Returns:
True if the provider is enabled
"""
provider_key = provider.lower()
return self.enabled_providers.get(provider_key, True) # Default to enabled
def bulk_set_provider_settings(self, provider_settings: dict) -> dict:
"""
Set multiple provider settings at once.
Args:
provider_settings: Dict of provider_name -> {'enabled': bool, ...}
Returns:
Dict with results for each provider
"""
results = {}
for provider_name, settings in provider_settings.items():
provider_key = provider_name.lower()
try:
if 'enabled' in settings:
self.enabled_providers[provider_key] = settings['enabled']
results[provider_key] = {'success': True, 'enabled': settings['enabled']}
else:
results[provider_key] = {'success': False, 'error': 'No enabled setting provided'}
except Exception as e:
results[provider_key] = {'success': False, 'error': str(e)}
return results
def get_api_key(self, provider: str) -> Optional[str]: def get_api_key(self, provider: str) -> Optional[str]:
"""Get API key for a provider.""" """Get API key for a provider."""
return self.api_keys.get(provider) return self.api_keys.get(provider)

565
core/graph_manager.py
View File

@ -1,8 +1,10 @@
# core/graph_manager.py # dnsrecon-reduced/core/graph_manager.py
""" """
Graph data model for DNSRecon using NetworkX. Graph data model for DNSRecon using NetworkX.
Manages in-memory graph storage with confidence scoring and forensic metadata. Manages in-memory graph storage with confidence scoring and forensic metadata.
Now fully compatible with the unified ProviderResult data model.
UPDATED: Fixed correlation exclusion keys to match actual attribute names.
""" """
import re import re
from datetime import datetime, timezone from datetime import datetime, timezone
@ -16,7 +18,8 @@ class NodeType(Enum):
"""Enumeration of supported node types.""" """Enumeration of supported node types."""
DOMAIN = "domain" DOMAIN = "domain"
IP = "ip" IP = "ip"
ASN = "asn" ISP = "isp"
CA = "ca"
LARGE_ENTITY = "large_entity" LARGE_ENTITY = "large_entity"
CORRELATION_OBJECT = "correlation_object" CORRELATION_OBJECT = "correlation_object"
@ -28,6 +31,7 @@ class GraphManager:
""" """
Thread-safe graph manager for DNSRecon infrastructure mapping. Thread-safe graph manager for DNSRecon infrastructure mapping.
Uses NetworkX for in-memory graph storage with confidence scoring. Uses NetworkX for in-memory graph storage with confidence scoring.
Compatible with unified ProviderResult data model.
""" """
def __init__(self): def __init__(self):
@ -39,6 +43,31 @@ class GraphManager:
# Compile regex for date filtering for efficiency # Compile regex for date filtering for efficiency
self.date_pattern = re.compile(r'^\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}') self.date_pattern = re.compile(r'^\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}')
# FIXED: Exclude cert_issuer_name since we already create proper CA relationships
self.EXCLUDED_KEYS = [
# Certificate metadata that creates noise or has dedicated node types
'cert_source', # Always 'crtsh' for crtsh provider
'cert_common_name',
'cert_validity_period_days', # Numerical, not useful for correlation
'cert_issuer_name', # FIXED: Has dedicated CA nodes, don't correlate
#'cert_certificate_id', # Unique per certificate
#'cert_serial_number', # Unique per certificate
'cert_entry_timestamp', # Timestamp, filtered by date regex anyway
'cert_not_before', # Date, filtered by date regex anyway
'cert_not_after', # Date, filtered by date regex anyway
# DNS metadata that creates noise
'dns_ttl', # TTL values are not meaningful for correlation
# Shodan metadata that might create noise
'timestamp', # Generic timestamp fields
'last_update', # Generic timestamp fields
#'org', # Too generic, causes false correlations
#'isp', # Too generic, causes false correlations
# Generic noisy attributes
'updated_timestamp', # Any timestamp field
'discovery_timestamp', # Any timestamp field
'query_timestamp', # Any timestamp field
]
def __getstate__(self): def __getstate__(self):
"""Prepare GraphManager for pickling, excluding compiled regex.""" """Prepare GraphManager for pickling, excluding compiled regex."""
state = self.__dict__.copy() state = self.__dict__.copy()
@ -52,245 +81,138 @@ class GraphManager:
self.__dict__.update(state) self.__dict__.update(state)
self.date_pattern = re.compile(r'^\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}') self.date_pattern = re.compile(r'^\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}')
def _update_correlation_index(self, node_id: str, data: Any, path: List[str] = [], parent_attr: str = ""): def process_correlations_for_node(self, node_id: str):
"""Recursively traverse metadata and add hashable values to the index with better path tracking.""" """
if path is None: UPDATED: Process correlations for a given node with enhanced tracking.
path = [] Now properly tracks which attribute/provider created each correlation.
"""
if isinstance(data, dict): if not self.graph.has_node(node_id):
for key, value in data.items():
self._update_correlation_index(node_id, value, path + [key], key)
elif isinstance(data, list):
for i, item in enumerate(data):
# Instead of just using [i], include the parent attribute context
list_path_component = f"[{i}]" if not parent_attr else f"{parent_attr}[{i}]"
self._update_correlation_index(node_id, item, path + [list_path_component], parent_attr)
else:
self._add_to_correlation_index(node_id, data, ".".join(path), parent_attr)
def _add_to_correlation_index(self, node_id: str, value: Any, path_str: str, parent_attr: str = ""):
"""Add a hashable value to the correlation index, filtering out noise."""
if not isinstance(value, (str, int, float, bool)) or value is None:
return return
# Ignore certain paths that contain noisy, non-unique identifiers node_attributes = self.graph.nodes[node_id].get('attributes', [])
if any(keyword in path_str.lower() for keyword in ['count', 'total', 'timestamp', 'date']):
return
# Filter out common low-entropy values and date-like strings # Process each attribute for potential correlations
if isinstance(value, str): for attr in node_attributes:
# FIXED: Prevent correlation on date/time strings. attr_name = attr.get('name')
if self.date_pattern.match(value): attr_value = attr.get('value')
return attr_provider = attr.get('provider', 'unknown')
if len(value) < 4 or value.lower() in ['true', 'false', 'unknown', 'none', 'crt.sh']:
return
elif isinstance(value, int) and (abs(value) < 1024 or abs(value) > 65535):
return # Ignore small integers and common port numbers
elif isinstance(value, bool):
return # Ignore boolean values
# Add the valuable correlation data to the index # IMPROVED: More comprehensive exclusion logic
if value not in self.correlation_index: should_exclude = (
self.correlation_index[value] = {} # Check against excluded keys (exact match or substring)
if node_id not in self.correlation_index[value]: any(excluded_key in attr_name or attr_name == excluded_key for excluded_key in self.EXCLUDED_KEYS) or
self.correlation_index[value][node_id] = [] # Invalid value types
not isinstance(attr_value, (str, int, float, bool)) or
attr_value is None or
# Boolean values are not useful for correlation
isinstance(attr_value, bool) or
# String values that are too short or are dates
(isinstance(attr_value, str) and (
len(attr_value) < 4 or
self.date_pattern.match(attr_value) or
# Exclude common generic values that create noise
attr_value.lower() in ['unknown', 'none', 'null', 'n/a', 'true', 'false', '0', '1']
)) or
# Numerical values that are likely to be unique identifiers
(isinstance(attr_value, (int, float)) and (
attr_value == 0 or # Zero values are not meaningful
attr_value == 1 or # One values are too common
abs(attr_value) > 1000000 # Very large numbers are likely IDs
))
)
# Store both the full path and the parent attribute for better edge labeling if should_exclude:
correlation_entry = { continue
'path': path_str,
'parent_attr': parent_attr,
'meaningful_attr': self._extract_meaningful_attribute(path_str, parent_attr)
}
if correlation_entry not in self.correlation_index[value][node_id]: # Initialize correlation tracking for this value
self.correlation_index[value][node_id].append(correlation_entry) if attr_value not in self.correlation_index:
self.correlation_index[attr_value] = {
def _extract_meaningful_attribute(self, path_str: str, parent_attr: str = "") -> str: 'nodes': set(),
"""Extract the most meaningful attribute name from a path string.""" 'sources': [] # Track which provider/attribute combinations contributed
if not path_str:
return "unknown"
path_parts = path_str.split('.')
# Look for the last non-array-index part
for part in reversed(path_parts):
# Skip array indices like [0], [1], etc.
if not (part.startswith('[') and part.endswith(']') and part[1:-1].isdigit()):
# Clean up compound names like "hostnames[0]" to just "hostnames"
clean_part = re.sub(r'\[\d+\]$', '', part)
if clean_part:
return clean_part
# Fallback to parent attribute if available
if parent_attr:
return parent_attr
# Last resort - use the first meaningful part
for part in path_parts:
if not (part.startswith('[') and part.endswith(']') and part[1:-1].isdigit()):
clean_part = re.sub(r'\[\d+\]$', '', part)
if clean_part:
return clean_part
return "correlation"
def _check_for_correlations(self, new_node_id: str, data: Any, path: List[str] = [], parent_attr: str = "") -> List[Dict]:
"""Recursively traverse metadata to find correlations with existing data."""
if path is None:
path = []
all_correlations = []
if isinstance(data, dict):
for key, value in data.items():
if key == 'source': # Avoid correlating on the provider name
continue
all_correlations.extend(self._check_for_correlations(new_node_id, value, path + [key], key))
elif isinstance(data, list):
for i, item in enumerate(data):
list_path_component = f"[{i}]" if not parent_attr else f"{parent_attr}[{i}]"
all_correlations.extend(self._check_for_correlations(new_node_id, item, path + [list_path_component], parent_attr))
else:
value = data
if value in self.correlation_index:
existing_nodes_with_paths = self.correlation_index[value]
unique_nodes = set(existing_nodes_with_paths.keys())
unique_nodes.add(new_node_id)
if len(unique_nodes) < 2:
return all_correlations # Correlation must involve at least two distinct nodes
new_source = {
'node_id': new_node_id,
'path': ".".join(path),
'parent_attr': parent_attr,
'meaningful_attr': self._extract_meaningful_attribute(".".join(path), parent_attr)
} }
all_sources = [new_source]
for node_id, path_entries in existing_nodes_with_paths.items(): # Add this node and source information
for entry in path_entries: self.correlation_index[attr_value]['nodes'].add(node_id)
if isinstance(entry, dict):
all_sources.append({
'node_id': node_id,
'path': entry['path'],
'parent_attr': entry.get('parent_attr', ''),
'meaningful_attr': entry.get('meaningful_attr', self._extract_meaningful_attribute(entry['path'], entry.get('parent_attr', '')))
})
else:
# Handle legacy string-only entries
all_sources.append({
'node_id': node_id,
'path': str(entry),
'parent_attr': '',
'meaningful_attr': self._extract_meaningful_attribute(str(entry))
})
all_correlations.append({ # Track the source of this correlation value
'value': value, source_info = {
'sources': all_sources, 'node_id': node_id,
'nodes': list(unique_nodes) 'provider': attr_provider,
}) 'attribute': attr_name,
return all_correlations 'path': f"{attr_provider}_{attr_name}"
}
def add_node(self, node_id: str, node_type: NodeType, attributes: Optional[Dict[str, Any]] = None, # Add source if not already present (avoid duplicates)
description: str = "", metadata: Optional[Dict[str, Any]] = None) -> bool: existing_sources = [s for s in self.correlation_index[attr_value]['sources']
"""Add a node to the graph, update attributes, and process correlations.""" if s['node_id'] == node_id and s['path'] == source_info['path']]
is_new_node = not self.graph.has_node(node_id) if not existing_sources:
if is_new_node: self.correlation_index[attr_value]['sources'].append(source_info)
self.graph.add_node(node_id, type=node_type.value,
added_timestamp=datetime.now(timezone.utc).isoformat(),
attributes=attributes or {},
description=description,
metadata=metadata or {})
else:
# Safely merge new attributes into existing attributes
if attributes:
existing_attributes = self.graph.nodes[node_id].get('attributes', {})
existing_attributes.update(attributes)
self.graph.nodes[node_id]['attributes'] = existing_attributes
if description:
self.graph.nodes[node_id]['description'] = description
if metadata:
existing_metadata = self.graph.nodes[node_id].get('metadata', {})
existing_metadata.update(metadata)
self.graph.nodes[node_id]['metadata'] = existing_metadata
if attributes and node_type != NodeType.CORRELATION_OBJECT: # Create correlation node if we have multiple nodes with this value
correlations = self._check_for_correlations(node_id, attributes) if len(self.correlation_index[attr_value]['nodes']) > 1:
for corr in correlations: self._create_enhanced_correlation_node_and_edges(attr_value, self.correlation_index[attr_value])
value = corr['value']
# STEP 1: Substring check against all existing nodes def _create_enhanced_correlation_node_and_edges(self, value, correlation_data):
if self._correlation_value_matches_existing_node(value): """
# Skip creating correlation node - would be redundant UPDATED: Create correlation node and edges with raw provider data (no formatting).
continue """
correlation_node_id = f"corr_{hash(str(value)) & 0x7FFFFFFF}"
nodes = correlation_data['nodes']
sources = correlation_data['sources']
eligible_nodes = set(corr['nodes']) # Create or update correlation node
if not self.graph.has_node(correlation_node_id):
# Use raw provider/attribute data - no formatting
provider_counts = {}
for source in sources:
# Keep original provider and attribute names
key = f"{source['provider']}_{source['attribute']}"
provider_counts[key] = provider_counts.get(key, 0) + 1
if len(eligible_nodes) < 2: # Use the most common provider/attribute as the primary label (raw)
# Need at least 2 nodes to create a correlation primary_source = max(provider_counts.items(), key=lambda x: x[1])[0] if provider_counts else "unknown_correlation"
continue
# STEP 3: Check for existing correlation node with same connection pattern metadata = {
correlation_nodes_with_pattern = self._find_correlation_nodes_with_same_pattern(eligible_nodes) 'value': value,
'correlated_nodes': list(nodes),
'sources': sources,
'primary_source': primary_source,
'correlation_count': len(nodes)
}
if correlation_nodes_with_pattern: self.add_node(correlation_node_id, NodeType.CORRELATION_OBJECT, metadata=metadata)
# STEP 4: Merge with existing correlation node #print(f"Created correlation node {correlation_node_id} for value '{value}' with {len(nodes)} nodes")
target_correlation_node = correlation_nodes_with_pattern[0]
self._merge_correlation_values(target_correlation_node, value, corr)
else:
# STEP 5: Create new correlation node for eligible nodes only
correlation_node_id = f"corr_{abs(hash(str(sorted(eligible_nodes))))}"
self.add_node(correlation_node_id, NodeType.CORRELATION_OBJECT,
metadata={'values': [value], 'sources': corr['sources'],
'correlated_nodes': list(eligible_nodes)})
# Create edges from eligible nodes to this correlation node with better labeling # Create edges from each node to the correlation node
for c_node_id in eligible_nodes: for source in sources:
if self.graph.has_node(c_node_id): node_id = source['node_id']
# Find the best attribute name for this node provider = source['provider']
meaningful_attr = self._find_best_attribute_name_for_node(c_node_id, corr['sources']) attribute = source['attribute']
relationship_type = f"c_{meaningful_attr}"
self.add_edge(c_node_id, correlation_node_id, relationship_type, confidence_score=0.9)
self._update_correlation_index(node_id, attributes) if self.graph.has_node(node_id) and not self.graph.has_edge(node_id, correlation_node_id):
# Format relationship label as "corr_provider_attribute"
relationship_label = f"corr_{provider}_{attribute}"
self.last_modified = datetime.now(timezone.utc).isoformat() self.add_edge(
return is_new_node source_id=node_id,
target_id=correlation_node_id,
relationship_type=relationship_label,
confidence_score=0.9,
source_provider=provider,
raw_data={
'correlation_value': value,
'original_attribute': attribute,
'correlation_type': 'attribute_matching'
}
)
def _find_best_attribute_name_for_node(self, node_id: str, sources: List[Dict]) -> str: #print(f"Added correlation edge: {node_id} -> {correlation_node_id} ({relationship_label})")
"""Find the best attribute name for a correlation edge by looking at the sources."""
node_sources = [s for s in sources if s['node_id'] == node_id]
if not node_sources:
return "correlation"
# Use the meaningful_attr if available
for source in node_sources:
meaningful_attr = source.get('meaningful_attr')
if meaningful_attr and meaningful_attr != "unknown":
return meaningful_attr
# Fallback to parent_attr
for source in node_sources:
parent_attr = source.get('parent_attr')
if parent_attr:
return parent_attr
# Last resort - extract from path
for source in node_sources:
path = source.get('path', '')
if path:
extracted = self._extract_meaningful_attribute(path)
if extracted != "unknown":
return extracted
return "correlation"
def _has_direct_edge_bidirectional(self, node_a: str, node_b: str) -> bool: def _has_direct_edge_bidirectional(self, node_a: str, node_b: str) -> bool:
""" """
Check if there's a direct edge between two nodes in either direction. Check if there's a direct edge between two nodes in either direction.
Returns True if node_a→node_b OR node_b→node_a exists. Returns True if node_aâ†'node_b OR node_bâ†'node_a exists.
""" """
return (self.graph.has_edge(node_a, node_b) or return (self.graph.has_edge(node_a, node_b) or
self.graph.has_edge(node_b, node_a)) self.graph.has_edge(node_b, node_a))
@ -382,19 +304,60 @@ class GraphManager:
f"across {node_count} nodes" f"across {node_count} nodes"
) )
def add_node(self, node_id: str, node_type: NodeType, attributes: Optional[List[Dict[str, Any]]] = None,
description: str = "", metadata: Optional[Dict[str, Any]] = None) -> bool:
"""
Add a node to the graph, update attributes, and process correlations.
Now compatible with unified data model - attributes are dictionaries from converted StandardAttribute objects.
"""
is_new_node = not self.graph.has_node(node_id)
if is_new_node:
self.graph.add_node(node_id, type=node_type.value,
added_timestamp=datetime.now(timezone.utc).isoformat(),
attributes=attributes or [], # Store as a list from the start
description=description,
metadata=metadata or {})
else:
# Safely merge new attributes into the existing list of attributes
if attributes:
existing_attributes = self.graph.nodes[node_id].get('attributes', [])
# Handle cases where old data might still be in dictionary format
if not isinstance(existing_attributes, list):
existing_attributes = []
# Create a set of existing attribute names for efficient duplicate checking
existing_attr_names = {attr['name'] for attr in existing_attributes}
for new_attr in attributes:
if new_attr['name'] not in existing_attr_names:
existing_attributes.append(new_attr)
existing_attr_names.add(new_attr['name'])
self.graph.nodes[node_id]['attributes'] = existing_attributes
if description:
self.graph.nodes[node_id]['description'] = description
if metadata:
existing_metadata = self.graph.nodes[node_id].get('metadata', {})
existing_metadata.update(metadata)
self.graph.nodes[node_id]['metadata'] = existing_metadata
self.last_modified = datetime.now(timezone.utc).isoformat()
return is_new_node
def add_edge(self, source_id: str, target_id: str, relationship_type: str, def add_edge(self, source_id: str, target_id: str, relationship_type: str,
confidence_score: float = 0.5, source_provider: str = "unknown", confidence_score: float = 0.5, source_provider: str = "unknown",
raw_data: Optional[Dict[str, Any]] = None) -> bool: raw_data: Optional[Dict[str, Any]] = None) -> bool:
"""Add or update an edge between two nodes, ensuring nodes exist.""" """
UPDATED: Add or update an edge between two nodes with raw relationship labels.
"""
if not self.graph.has_node(source_id) or not self.graph.has_node(target_id): if not self.graph.has_node(source_id) or not self.graph.has_node(target_id):
return False return False
new_confidence = confidence_score new_confidence = confidence_score
if relationship_type.startswith("c_"): # UPDATED: Use raw relationship type - no formatting
edge_label = relationship_type edge_label = relationship_type
else:
edge_label = f"{source_provider}_{relationship_type}"
if self.graph.has_edge(source_id, target_id): if self.graph.has_edge(source_id, target_id):
# If edge exists, update confidence if the new score is higher. # If edge exists, update confidence if the new score is higher.
@ -404,7 +367,7 @@ class GraphManager:
self.graph.edges[source_id, target_id]['updated_by'] = source_provider self.graph.edges[source_id, target_id]['updated_by'] = source_provider
return False return False
# Add a new edge with all attributes. # Add a new edge with raw attributes
self.graph.add_edge(source_id, target_id, self.graph.add_edge(source_id, target_id,
relationship_type=edge_label, relationship_type=edge_label,
confidence_score=new_confidence, confidence_score=new_confidence,
@ -423,13 +386,19 @@ class GraphManager:
return False return False
node_data = self.graph.nodes[large_entity_id] node_data = self.graph.nodes[large_entity_id]
attributes = node_data.get('attributes', {}) attributes = node_data.get('attributes', [])
# Find the 'nodes' attribute dictionary in the list
nodes_attr = next((attr for attr in attributes if attr.get('name') == 'nodes'), None)
# Remove from the list of member nodes # Remove from the list of member nodes
if 'nodes' in attributes and node_id_to_extract in attributes['nodes']: if nodes_attr and 'value' in nodes_attr and isinstance(nodes_attr['value'], list) and node_id_to_extract in nodes_attr['value']:
attributes['nodes'].remove(node_id_to_extract) nodes_attr['value'].remove(node_id_to_extract)
# Update the count
attributes['count'] = len(attributes['nodes']) # Find the 'count' attribute and update it
count_attr = next((attr for attr in attributes if attr.get('name') == 'count'), None)
if count_attr:
count_attr['value'] = len(nodes_attr['value'])
else: else:
# This can happen if the node was already extracted, which is not an error. # This can happen if the node was already extracted, which is not an error.
print(f"Warning: Node {node_id_to_extract} not found in the 'nodes' list of {large_entity_id}.") print(f"Warning: Node {node_id_to_extract} not found in the 'nodes' list of {large_entity_id}.")
@ -448,11 +417,21 @@ class GraphManager:
# Clean up the correlation index # Clean up the correlation index
keys_to_delete = [] keys_to_delete = []
for value, nodes in self.correlation_index.items(): for value, data in self.correlation_index.items():
if node_id in nodes: if isinstance(data, dict) and 'nodes' in data:
del nodes[node_id] # Updated correlation structure
if not nodes: # If no other nodes are associated with this value, remove it if node_id in data['nodes']:
keys_to_delete.append(value) data['nodes'].discard(node_id)
# Remove sources for this node
data['sources'] = [s for s in data['sources'] if s['node_id'] != node_id]
if not data['nodes']: # If no other nodes are associated, remove it
keys_to_delete.append(value)
else:
# Legacy correlation structure (fallback)
if isinstance(data, set) and node_id in data:
data.discard(node_id)
if not data:
keys_to_delete.append(value)
for key in keys_to_delete: for key in keys_to_delete:
if key in self.correlation_index: if key in self.correlation_index:
@ -473,54 +452,59 @@ class GraphManager:
"""Get all nodes of a specific type.""" """Get all nodes of a specific type."""
return [n for n, d in self.graph.nodes(data=True) if d.get('type') == node_type.value] return [n for n, d in self.graph.nodes(data=True) if d.get('type') == node_type.value]
def get_neighbors(self, node_id: str) -> List[str]:
"""Get all unique neighbors (predecessors and successors) for a node."""
if not self.graph.has_node(node_id):
return []
return list(set(self.graph.predecessors(node_id)) | set(self.graph.successors(node_id)))
def get_high_confidence_edges(self, min_confidence: float = 0.8) -> List[Tuple[str, str, Dict]]: def get_high_confidence_edges(self, min_confidence: float = 0.8) -> List[Tuple[str, str, Dict]]:
"""Get edges with confidence score above a given threshold.""" """Get edges with confidence score above a given threshold."""
return [(u, v, d) for u, v, d in self.graph.edges(data=True) return [(u, v, d) for u, v, d in self.graph.edges(data=True)
if d.get('confidence_score', 0) >= min_confidence] if d.get('confidence_score', 0) >= min_confidence]
def get_graph_data(self) -> Dict[str, Any]: def get_graph_data(self) -> Dict[str, Any]:
"""Export graph data formatted for frontend visualization.""" """
Export graph data formatted for frontend visualization.
SIMPLIFIED: No certificate styling - frontend handles all visual styling.
"""
nodes = [] nodes = []
for node_id, attrs in self.graph.nodes(data=True): for node_id, attrs in self.graph.nodes(data=True):
node_data = {'id': node_id, 'label': node_id, 'type': attrs.get('type', 'unknown'), node_data = {
'attributes': attrs.get('attributes', {}), 'id': node_id,
'description': attrs.get('description', ''), 'label': node_id,
'metadata': attrs.get('metadata', {}), 'type': attrs.get('type', 'unknown'),
'added_timestamp': attrs.get('added_timestamp')} 'attributes': attrs.get('attributes', []), # Raw attributes list
# Customize node appearance based on type and attributes 'description': attrs.get('description', ''),
node_type = node_data['type'] 'metadata': attrs.get('metadata', {}),
attributes = node_data['attributes'] 'added_timestamp': attrs.get('added_timestamp')
if node_type == 'domain' and attributes.get('certificates', {}).get('has_valid_cert') is False: }
node_data['color'] = {'background': '#c7c7c7', 'border': '#999'} # Gray for invalid cert
# Add incoming and outgoing edges to node data # Add incoming and outgoing edges to node data
if self.graph.has_node(node_id): if self.graph.has_node(node_id):
node_data['incoming_edges'] = [{'from': u, 'data': d} for u, _, d in self.graph.in_edges(node_id, data=True)] node_data['incoming_edges'] = [
node_data['outgoing_edges'] = [{'to': v, 'data': d} for _, v, d in self.graph.out_edges(node_id, data=True)] {'from': u, 'data': d} for u, _, d in self.graph.in_edges(node_id, data=True)
]
node_data['outgoing_edges'] = [
{'to': v, 'data': d} for _, v, d in self.graph.out_edges(node_id, data=True)
]
nodes.append(node_data) nodes.append(node_data)
edges = [] edges = []
for source, target, attrs in self.graph.edges(data=True): for source, target, attrs in self.graph.edges(data=True):
edges.append({'from': source, 'to': target, edges.append({
'label': attrs.get('relationship_type', ''), 'from': source,
'confidence_score': attrs.get('confidence_score', 0), 'to': target,
'source_provider': attrs.get('source_provider', ''), 'label': attrs.get('relationship_type', ''),
'discovery_timestamp': attrs.get('discovery_timestamp')}) 'confidence_score': attrs.get('confidence_score', 0),
'source_provider': attrs.get('source_provider', ''),
'discovery_timestamp': attrs.get('discovery_timestamp')
})
return { return {
'nodes': nodes, 'edges': edges, 'nodes': nodes,
'edges': edges,
'statistics': self.get_statistics()['basic_metrics'] 'statistics': self.get_statistics()['basic_metrics']
} }
def export_json(self) -> Dict[str, Any]: def export_json(self) -> Dict[str, Any]:
"""Export complete graph data as a JSON-serializable dictionary.""" """Export complete graph data as a JSON-serializable dictionary."""
graph_data = nx.node_link_data(self.graph) # Use NetworkX's built-in robust serializer graph_data = nx.node_link_data(self.graph, edges="edges")
return { return {
'export_metadata': { 'export_metadata': {
'export_timestamp': datetime.now(timezone.utc).isoformat(), 'export_timestamp': datetime.now(timezone.utc).isoformat(),
@ -528,15 +512,20 @@ class GraphManager:
'last_modified': self.last_modified, 'last_modified': self.last_modified,
'total_nodes': self.get_node_count(), 'total_nodes': self.get_node_count(),
'total_edges': self.get_edge_count(), 'total_edges': self.get_edge_count(),
'graph_format': 'dnsrecon_v1_nodeling' 'graph_format': 'dnsrecon_v1_unified_model'
}, },
'graph': graph_data, 'graph': graph_data,
'statistics': self.get_statistics() 'statistics': self.get_statistics()
} }
def _get_confidence_distribution(self) -> Dict[str, int]: def _get_confidence_distribution(self) -> Dict[str, int]:
"""Get distribution of edge confidence scores.""" """Get distribution of edge confidence scores with empty graph handling."""
distribution = {'high': 0, 'medium': 0, 'low': 0} distribution = {'high': 0, 'medium': 0, 'low': 0}
# FIXED: Handle empty graph case
if self.get_edge_count() == 0:
return distribution
for _, _, data in self.graph.edges(data=True): for _, _, data in self.graph.edges(data=True):
confidence = data.get('confidence_score', 0) confidence = data.get('confidence_score', 0)
if confidence >= 0.8: if confidence >= 0.8:
@ -548,22 +537,42 @@ class GraphManager:
return distribution return distribution
def get_statistics(self) -> Dict[str, Any]: def get_statistics(self) -> Dict[str, Any]:
"""Get comprehensive statistics about the graph.""" """Get comprehensive statistics about the graph with proper empty graph handling."""
stats = {'basic_metrics': {'total_nodes': self.get_node_count(),
'total_edges': self.get_edge_count(), # FIXED: Handle empty graph case properly
'creation_time': self.creation_time, node_count = self.get_node_count()
'last_modified': self.last_modified}, edge_count = self.get_edge_count()
'node_type_distribution': {}, 'relationship_type_distribution': {},
'confidence_distribution': self._get_confidence_distribution(), stats = {
'provider_distribution': {}} 'basic_metrics': {
# Calculate distributions 'total_nodes': node_count,
for node_type in NodeType: 'total_edges': edge_count,
stats['node_type_distribution'][node_type.value] = self.get_nodes_by_type(node_type).__len__() 'creation_time': self.creation_time,
for _, _, data in self.graph.edges(data=True): 'last_modified': self.last_modified
rel_type = data.get('relationship_type', 'unknown') },
stats['relationship_type_distribution'][rel_type] = stats['relationship_type_distribution'].get(rel_type, 0) + 1 'node_type_distribution': {},
provider = data.get('source_provider', 'unknown') 'relationship_type_distribution': {},
stats['provider_distribution'][provider] = stats['provider_distribution'].get(provider, 0) + 1 'confidence_distribution': self._get_confidence_distribution(),
'provider_distribution': {}
}
# FIXED: Only calculate distributions if we have data
if node_count > 0:
# Calculate node type distributions
for node_type in NodeType:
count = len(self.get_nodes_by_type(node_type))
if count > 0: # Only include types that exist
stats['node_type_distribution'][node_type.value] = count
if edge_count > 0:
# Calculate edge distributions
for _, _, data in self.graph.edges(data=True):
rel_type = data.get('relationship_type', 'unknown')
stats['relationship_type_distribution'][rel_type] = stats['relationship_type_distribution'].get(rel_type, 0) + 1
provider = data.get('source_provider', 'unknown')
stats['provider_distribution'][provider] = stats['provider_distribution'].get(provider, 0) + 1
return stats return stats
def clear(self) -> None: def clear(self) -> None:

4
core/logger.py
View File

@ -152,7 +152,7 @@ class ForensicLogger:
# Log to standard logger # Log to standard logger
if error: if error:
self.logger.error(f"API Request Failed - {provider}: {url} - {error}") self.logger.error(f"API Request Failed.")
else: else:
self.logger.info(f"API Request - {provider}: {url} - Status: {status_code}") self.logger.info(f"API Request - {provider}: {url} - Status: {status_code}")
@ -197,7 +197,7 @@ class ForensicLogger:
self.logger.info(f"Scan Started - Target: {target_domain}, Depth: {recursion_depth}") self.logger.info(f"Scan Started - Target: {target_domain}, Depth: {recursion_depth}")
self.logger.info(f"Enabled Providers: {', '.join(enabled_providers)}") self.logger.info(f"Enabled Providers: {', '.join(enabled_providers)}")
self.session_metadata['target_domains'].add(target_domain) self.session_metadata['target_domains'].update(target_domain)
def log_scan_complete(self) -> None: def log_scan_complete(self) -> None:
"""Log the completion of a reconnaissance scan.""" """Log the completion of a reconnaissance scan."""

107
core/provider_result.py Normal file
View File

@ -0,0 +1,107 @@
# dnsrecon-reduced/core/provider_result.py
"""
Unified data model for DNSRecon passive reconnaissance.
Standardizes the data structure across all providers to ensure consistent processing.
"""
from typing import Any, Optional, List, Dict
from dataclasses import dataclass, field
from datetime import datetime, timezone
@dataclass
class StandardAttribute:
"""A unified data structure for a single piece of information about a node."""
target_node: str
name: str
value: Any
type: str
provider: str
confidence: float
timestamp: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
metadata: Optional[Dict[str, Any]] = field(default_factory=dict)
def __post_init__(self):
"""Validate the attribute after initialization."""
if not isinstance(self.confidence, (int, float)) or not 0.0 <= self.confidence <= 1.0:
raise ValueError(f"Confidence must be between 0.0 and 1.0, got {self.confidence}")
@dataclass
class Relationship:
"""A unified data structure for a directional link between two nodes."""
source_node: str
target_node: str
relationship_type: str
confidence: float
provider: str
timestamp: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
raw_data: Optional[Dict[str, Any]] = field(default_factory=dict)
def __post_init__(self):
"""Validate the relationship after initialization."""
if not isinstance(self.confidence, (int, float)) or not 0.0 <= self.confidence <= 1.0:
raise ValueError(f"Confidence must be between 0.0 and 1.0, got {self.confidence}")
@dataclass
class ProviderResult:
"""A container for all data returned by a provider from a single query."""
attributes: List[StandardAttribute] = field(default_factory=list)
relationships: List[Relationship] = field(default_factory=list)
def add_attribute(self, target_node: str, name: str, value: Any, attr_type: str,
provider: str, confidence: float = 0.8,
metadata: Optional[Dict[str, Any]] = None) -> None:
"""Helper method to add an attribute to the result."""
self.attributes.append(StandardAttribute(
target_node=target_node,
name=name,
value=value,
type=attr_type,
provider=provider,
confidence=confidence,
metadata=metadata or {}
))
def add_relationship(self, source_node: str, target_node: str, relationship_type: str,
provider: str, confidence: float = 0.8,
raw_data: Optional[Dict[str, Any]] = None) -> None:
"""Helper method to add a relationship to the result."""
self.relationships.append(Relationship(
source_node=source_node,
target_node=target_node,
relationship_type=relationship_type,
confidence=confidence,
provider=provider,
raw_data=raw_data or {}
))
def get_discovered_nodes(self) -> set:
"""Get all unique node identifiers discovered in this result."""
nodes = set()
# Add nodes from relationships
for rel in self.relationships:
nodes.add(rel.source_node)
nodes.add(rel.target_node)
# Add nodes from attributes
for attr in self.attributes:
nodes.add(attr.target_node)
return nodes
def get_relationship_count(self) -> int:
"""Get the total number of relationships in this result."""
return len(self.relationships)
def get_attribute_count(self) -> int:
"""Get the total number of attributes in this result."""
return len(self.attributes)
##TODO
#def is_large_entity(self, threshold: int) -> bool:
# """Check if this result qualifies as a large entity based on relationship count."""
# return self.get_relationship_count() > threshold

1
core/rate_limiter.py
View File

@ -1,7 +1,6 @@
# dnsrecon-reduced/core/rate_limiter.py # dnsrecon-reduced/core/rate_limiter.py
import time import time
import redis
class GlobalRateLimiter: class GlobalRateLimiter:
def __init__(self, redis_client): def __init__(self, redis_client):

899
core/scanner.py
View File

File diff suppressed because it is too large Load Diff

88
core/session_manager.py
View File

@ -5,18 +5,15 @@ import time
import uuid import uuid
import redis import redis
import pickle import pickle
from typing import Dict, Optional, Any, List from typing import Dict, Optional, Any
from core.scanner import Scanner from core.scanner import Scanner
from config import config from config import config
# WARNING: Using pickle can be a security risk if the data source is not trusted.
# In this case, we are only serializing/deserializing our own trusted Scanner objects,
# which is generally safe. Do not unpickle data from untrusted sources.
class SessionManager: class SessionManager:
""" """
Manages multiple scanner instances for concurrent user sessions using Redis. FIXED: Manages multiple scanner instances for concurrent user sessions using Redis.
Now more conservative about session creation to preserve API keys and configuration.
""" """
def __init__(self, session_timeout_minutes: int = 0): def __init__(self, session_timeout_minutes: int = 0):
@ -28,7 +25,10 @@ class SessionManager:
self.redis_client = redis.StrictRedis(db=0, decode_responses=False) self.redis_client = redis.StrictRedis(db=0, decode_responses=False)
self.session_timeout = session_timeout_minutes * 60 # Convert to seconds self.session_timeout = session_timeout_minutes * 60 # Convert to seconds
self.lock = threading.Lock() # Lock for local operations, Redis handles atomic ops self.lock = threading.Lock()
# FIXED: Add a creation lock to prevent race conditions
self.creation_lock = threading.Lock()
# Start cleanup thread # Start cleanup thread
self.cleanup_thread = threading.Thread(target=self._cleanup_loop, daemon=True) self.cleanup_thread = threading.Thread(target=self._cleanup_loop, daemon=True)
@ -40,7 +40,7 @@ class SessionManager:
"""Prepare SessionManager for pickling.""" """Prepare SessionManager for pickling."""
state = self.__dict__.copy() state = self.__dict__.copy()
# Exclude unpickleable attributes - Redis client and threading objects # Exclude unpickleable attributes - Redis client and threading objects
unpicklable_attrs = ['lock', 'cleanup_thread', 'redis_client'] unpicklable_attrs = ['lock', 'cleanup_thread', 'redis_client', 'creation_lock']
for attr in unpicklable_attrs: for attr in unpicklable_attrs:
if attr in state: if attr in state:
del state[attr] del state[attr]
@ -50,9 +50,9 @@ class SessionManager:
"""Restore SessionManager after unpickling.""" """Restore SessionManager after unpickling."""
self.__dict__.update(state) self.__dict__.update(state)
# Re-initialize unpickleable attributes # Re-initialize unpickleable attributes
import redis
self.redis_client = redis.StrictRedis(db=0, decode_responses=False) self.redis_client = redis.StrictRedis(db=0, decode_responses=False)
self.lock = threading.Lock() self.lock = threading.Lock()
self.creation_lock = threading.Lock()
self.cleanup_thread = threading.Thread(target=self._cleanup_loop, daemon=True) self.cleanup_thread = threading.Thread(target=self._cleanup_loop, daemon=True)
self.cleanup_thread.start() self.cleanup_thread.start()
@ -66,44 +66,47 @@ class SessionManager:
def create_session(self) -> str: def create_session(self) -> str:
""" """
Create a new user session and store it in Redis. FIXED: Create a new user session with thread-safe creation to prevent duplicates.
""" """
session_id = str(uuid.uuid4()) # FIXED: Use creation lock to prevent race conditions
print(f"=== CREATING SESSION {session_id} IN REDIS ===") with self.creation_lock:
session_id = str(uuid.uuid4())
print(f"=== CREATING SESSION {session_id} IN REDIS ===")
try: try:
from core.session_config import create_session_config from core.session_config import create_session_config
session_config = create_session_config() session_config = create_session_config()
scanner_instance = Scanner(session_config=session_config) scanner_instance = Scanner(session_config=session_config)
# Set the session ID on the scanner for cross-process stop signal management # Set the session ID on the scanner for cross-process stop signal management
scanner_instance.session_id = session_id scanner_instance.session_id = session_id
session_data = { session_data = {
'scanner': scanner_instance, 'scanner': scanner_instance,
'config': session_config, 'config': session_config,
'created_at': time.time(), 'created_at': time.time(),
'last_activity': time.time(), 'last_activity': time.time(),
'status': 'active' 'status': 'active'
} }
# Serialize the entire session data dictionary using pickle # Serialize the entire session data dictionary using pickle
serialized_data = pickle.dumps(session_data) serialized_data = pickle.dumps(session_data)
# Store in Redis # Store in Redis
session_key = self._get_session_key(session_id) session_key = self._get_session_key(session_id)
self.redis_client.setex(session_key, self.session_timeout, serialized_data) self.redis_client.setex(session_key, self.session_timeout, serialized_data)
# Initialize stop signal as False # Initialize stop signal as False
stop_key = self._get_stop_signal_key(session_id) stop_key = self._get_stop_signal_key(session_id)
self.redis_client.setex(stop_key, self.session_timeout, b'0') self.redis_client.setex(stop_key, self.session_timeout, b'0')
print(f"Session {session_id} stored in Redis with stop signal initialized") print(f"Session {session_id} stored in Redis with stop signal initialized")
return session_id print(f"Session has {len(scanner_instance.providers)} providers: {[p.get_name() for p in scanner_instance.providers]}")
return session_id
except Exception as e: except Exception as e:
print(f"ERROR: Failed to create session {session_id}: {e}") print(f"ERROR: Failed to create session {session_id}: {e}")
raise raise
def set_stop_signal(self, session_id: str) -> bool: def set_stop_signal(self, session_id: str) -> bool:
""" """
@ -212,7 +215,14 @@ class SessionManager:
# Immediately save to Redis for GUI updates # Immediately save to Redis for GUI updates
success = self._save_session_data(session_id, session_data) success = self._save_session_data(session_id, session_data)
if success: if success:
print(f"Scanner state updated for session {session_id} (status: {scanner.status})") # Only log occasionally to reduce noise
if hasattr(self, '_last_update_log'):
if time.time() - self._last_update_log > 5: # Log every 5 seconds max
#print(f"Scanner state updated for session {session_id} (status: {scanner.status})")
self._last_update_log = time.time()
else:
#print(f"Scanner state updated for session {session_id} (status: {scanner.status})")
self._last_update_log = time.time()
else: else:
print(f"WARNING: Failed to save scanner state for session {session_id}") print(f"WARNING: Failed to save scanner state for session {session_id}")
return success return success

25
providers/base_provider.py
View File

@ -4,16 +4,17 @@ import time
import requests import requests
import threading import threading
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
from typing import List, Dict, Any, Optional, Tuple from typing import Dict, Any, Optional
from core.logger import get_forensic_logger from core.logger import get_forensic_logger
from core.rate_limiter import GlobalRateLimiter from core.rate_limiter import GlobalRateLimiter
from core.provider_result import ProviderResult
class BaseProvider(ABC): class BaseProvider(ABC):
""" """
Abstract base class for all DNSRecon data providers. Abstract base class for all DNSRecon data providers.
Now supports session-specific configuration. Now supports session-specific configuration and returns standardized ProviderResult objects.
""" """
def __init__(self, name: str, rate_limit: int = 60, timeout: int = 30, session_config=None): def __init__(self, name: str, rate_limit: int = 60, timeout: int = 30, session_config=None):
@ -101,7 +102,7 @@ class BaseProvider(ABC):
pass pass
@abstractmethod @abstractmethod
def query_domain(self, domain: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]: def query_domain(self, domain: str) -> ProviderResult:
""" """
Query the provider for information about a domain. Query the provider for information about a domain.
@ -109,12 +110,12 @@ class BaseProvider(ABC):
domain: Domain to investigate domain: Domain to investigate
Returns: Returns:
List of tuples: (source_node, target_node, relationship_type, confidence, raw_data) ProviderResult containing standardized attributes and relationships
""" """
pass pass
@abstractmethod @abstractmethod
def query_ip(self, ip: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]: def query_ip(self, ip: str) -> ProviderResult:
""" """
Query the provider for information about an IP address. Query the provider for information about an IP address.
@ -122,7 +123,7 @@ class BaseProvider(ABC):
ip: IP address to investigate ip: IP address to investigate
Returns: Returns:
List of tuples: (source_node, target_node, relationship_type, confidence, raw_data) ProviderResult containing standardized attributes and relationships
""" """
pass pass
@ -132,6 +133,8 @@ class BaseProvider(ABC):
target_indicator: str = "") -> Optional[requests.Response]: target_indicator: str = "") -> Optional[requests.Response]:
""" """
Make a rate-limited HTTP request. Make a rate-limited HTTP request.
FIXED: Returns response without automatically raising HTTPError exceptions.
Individual providers should handle status codes appropriately.
""" """
if self._is_stop_requested(): if self._is_stop_requested():
print(f"Request cancelled before start: {url}") print(f"Request cancelled before start: {url}")
@ -168,8 +171,14 @@ class BaseProvider(ABC):
raise ValueError(f"Unsupported HTTP method: {method}") raise ValueError(f"Unsupported HTTP method: {method}")
print(f"Response status: {response.status_code}") print(f"Response status: {response.status_code}")
response.raise_for_status()
self.successful_requests += 1 # FIXED: Don't automatically raise for HTTP error status codes
# Let individual providers handle status codes appropriately
# Only count 2xx responses as successful
if 200 <= response.status_code < 300:
self.successful_requests += 1
else:
self.failed_requests += 1
duration_ms = (time.time() - start_time) * 1000 duration_ms = (time.time() - start_time) * 1000
self.logger.log_api_request( self.logger.log_api_request(

905
providers/crtsh_provider.py
View File

File diff suppressed because it is too large Load Diff

202
providers/dns_provider.py
View File

@ -1,15 +1,16 @@
# dnsrecon/providers/dns_provider.py # dnsrecon/providers/dns_provider.py
from dns import resolver, reversename from dns import resolver, reversename
from typing import List, Dict, Any, Tuple from typing import Dict
from .base_provider import BaseProvider from .base_provider import BaseProvider
from utils.helpers import _is_valid_ip, _is_valid_domain from core.provider_result import ProviderResult
from utils.helpers import _is_valid_ip, _is_valid_domain, get_ip_version
class DNSProvider(BaseProvider): class DNSProvider(BaseProvider):
""" """
Provider for standard DNS resolution and reverse DNS lookups. Provider for standard DNS resolution and reverse DNS lookups.
Now uses session-specific configuration. Now returns standardized ProviderResult objects with IPv4 and IPv6 support.
""" """
def __init__(self, name=None, session_config=None): def __init__(self, name=None, session_config=None):
@ -25,7 +26,6 @@ class DNSProvider(BaseProvider):
self.resolver = resolver.Resolver() self.resolver = resolver.Resolver()
self.resolver.timeout = 5 self.resolver.timeout = 5
self.resolver.lifetime = 10 self.resolver.lifetime = 10
#self.resolver.nameservers = ['127.0.0.1']
def get_name(self) -> str: def get_name(self) -> str:
"""Return the provider name.""" """Return the provider name."""
@ -47,80 +47,118 @@ class DNSProvider(BaseProvider):
"""DNS is always available - no API key required.""" """DNS is always available - no API key required."""
return True return True
def query_domain(self, domain: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]: def query_domain(self, domain: str) -> ProviderResult:
""" """
Query DNS records for the domain to discover relationships. Query DNS records for the domain to discover relationships and attributes.
... FIXED: Now creates separate attributes for each DNS record type.
Args:
domain: Domain to investigate
Returns:
ProviderResult containing discovered relationships and attributes
""" """
if not _is_valid_domain(domain): if not _is_valid_domain(domain):
return [] return ProviderResult()
relationships = [] result = ProviderResult()
# Query all record types # Query all record types - each gets its own attribute
for record_type in ['A', 'AAAA', 'CNAME', 'MX', 'NS', 'SOA', 'TXT', 'SRV', 'CAA']: for record_type in ['A', 'AAAA', 'CNAME', 'MX', 'NS', 'SOA', 'TXT', 'SRV', 'CAA']:
try: try:
relationships.extend(self._query_record(domain, record_type)) self._query_record(domain, record_type, result)
except resolver.NoAnswer: #except resolver.NoAnswer:
# This is not an error, just a confirmation that the record doesn't exist. # This is not an error, just a confirmation that the record doesn't exist.
self.logger.logger.debug(f"No {record_type} record found for {domain}") #self.logger.logger.debug(f"No {record_type} record found for {domain}")
except Exception as e: except Exception as e:
self.failed_requests += 1 self.failed_requests += 1
self.logger.logger.debug(f"{record_type} record query failed for {domain}: {e}") self.logger.logger.debug(f"{record_type} record query failed for {domain}: {e}")
# Optionally, you might want to re-raise other, more serious exceptions.
return relationships return result
def query_ip(self, ip: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]: def query_ip(self, ip: str) -> ProviderResult:
""" """
Query reverse DNS for the IP address. Query reverse DNS for the IP address (supports both IPv4 and IPv6).
Args: Args:
ip: IP address to investigate ip: IP address to investigate (IPv4 or IPv6)
Returns: Returns:
List of relationships discovered from reverse DNS ProviderResult containing discovered relationships and attributes
""" """
if not _is_valid_ip(ip): if not _is_valid_ip(ip):
return [] return ProviderResult()
relationships = [] result = ProviderResult()
ip_version = get_ip_version(ip)
try: try:
# Perform reverse DNS lookup # Perform reverse DNS lookup (works for both IPv4 and IPv6)
self.total_requests += 1 self.total_requests += 1
reverse_name = reversename.from_address(ip) reverse_name = reversename.from_address(ip)
response = self.resolver.resolve(reverse_name, 'PTR') response = self.resolver.resolve(reverse_name, 'PTR')
self.successful_requests += 1 self.successful_requests += 1
ptr_records = []
for ptr_record in response: for ptr_record in response:
hostname = str(ptr_record).rstrip('.') hostname = str(ptr_record).rstrip('.')
if _is_valid_domain(hostname): if _is_valid_domain(hostname):
raw_data = { # Determine appropriate forward relationship type based on IP version
'query_type': 'PTR', if ip_version == 6:
'ip_address': ip, relationship_type = 'dns_aaaa_record'
'hostname': hostname, record_prefix = 'AAAA'
'ttl': response.ttl else:
} relationship_type = 'dns_a_record'
record_prefix = 'A'
relationships.append(( # Add the relationship
ip, result.add_relationship(
hostname, source_node=ip,
'ptr_record', target_node=hostname,
0.8, relationship_type='dns_ptr_record',
raw_data provider=self.name,
)) confidence=0.8,
raw_data={
'query_type': 'PTR',
'ip_address': ip,
'ip_version': ip_version,
'hostname': hostname,
'ttl': response.ttl
}
)
# Add to PTR records list
ptr_records.append(f"PTR: {hostname}")
# Log the relationship discovery
self.log_relationship_discovery( self.log_relationship_discovery(
source_node=ip, source_node=ip,
target_node=hostname, target_node=hostname,
relationship_type='ptr_record', relationship_type='dns_ptr_record',
confidence_score=0.8, confidence_score=0.8,
raw_data=raw_data, raw_data={
discovery_method="reverse_dns_lookup" 'query_type': 'PTR',
'ip_address': ip,
'ip_version': ip_version,
'hostname': hostname,
'ttl': response.ttl
},
discovery_method=f"reverse_dns_lookup_ipv{ip_version}"
) )
# Add PTR records as separate attribute
if ptr_records:
result.add_attribute(
target_node=ip,
name='ptr_records', # Specific name for PTR records
value=ptr_records,
attr_type='dns_record',
provider=self.name,
confidence=0.8,
metadata={'ttl': response.ttl, 'ip_version': ip_version}
)
except resolver.NXDOMAIN: except resolver.NXDOMAIN:
self.failed_requests += 1 self.failed_requests += 1
self.logger.logger.debug(f"Reverse DNS lookup failed for {ip}: NXDOMAIN") self.logger.logger.debug(f"Reverse DNS lookup failed for {ip}: NXDOMAIN")
@ -130,22 +168,28 @@ class DNSProvider(BaseProvider):
# Re-raise the exception so the scanner can handle the failure # Re-raise the exception so the scanner can handle the failure
raise e raise e
return relationships return result
def _query_record(self, domain: str, record_type: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]: def _query_record(self, domain: str, record_type: str, result: ProviderResult) -> None:
""" """
Query a specific type of DNS record for the domain. FIXED: Query DNS records with unique attribute names for each record type.
Enhanced to better handle IPv6 AAAA records.
""" """
relationships = []
try: try:
self.total_requests += 1 self.total_requests += 1
response = self.resolver.resolve(domain, record_type) response = self.resolver.resolve(domain, record_type)
self.successful_requests += 1 self.successful_requests += 1
dns_records = []
for record in response: for record in response:
target = "" target = ""
if record_type in ['A', 'AAAA']: if record_type in ['A', 'AAAA']:
target = str(record) target = str(record)
# Validate that the IP address is properly formed
if not _is_valid_ip(target):
self.logger.logger.debug(f"Invalid IP address in {record_type} record: {target}")
continue
elif record_type in ['CNAME', 'NS', 'PTR']: elif record_type in ['CNAME', 'NS', 'PTR']:
target = str(record.target).rstrip('.') target = str(record.target).rstrip('.')
elif record_type == 'MX': elif record_type == 'MX':
@ -153,32 +197,56 @@ class DNSProvider(BaseProvider):
elif record_type == 'SOA': elif record_type == 'SOA':
target = str(record.mname).rstrip('.') target = str(record.mname).rstrip('.')
elif record_type in ['TXT']: elif record_type in ['TXT']:
# TXT records are treated as metadata, not relationships. # Keep raw TXT record value
txt_value = str(record).strip('"')
dns_records.append(txt_value) # Just the value for TXT
continue continue
elif record_type == 'SRV': elif record_type == 'SRV':
target = str(record.target).rstrip('.') target = str(record.target).rstrip('.')
elif record_type == 'CAA': elif record_type == 'CAA':
target = f"{record.flags} {record.tag.decode('utf-8')} \"{record.value.decode('utf-8')}\"" # Keep raw CAA record format
caa_value = f"{record.flags} {record.tag.decode('utf-8')} \"{record.value.decode('utf-8')}\""
dns_records.append(caa_value) # Just the value for CAA
continue
else: else:
target = str(record) target = str(record)
if target: if target:
# Determine IP version for metadata if this is an IP record
ip_version = None
if record_type in ['A', 'AAAA'] and _is_valid_ip(target):
ip_version = get_ip_version(target)
raw_data = { raw_data = {
'query_type': record_type, 'query_type': record_type,
'domain': domain, 'domain': domain,
'value': target, 'value': target,
'ttl': response.ttl 'ttl': response.ttl
} }
relationship_type = f"{record_type.lower()}_record"
confidence = 0.8 # Default confidence for DNS records
relationships.append(( if ip_version:
domain, raw_data['ip_version'] = ip_version
target,
relationship_type, relationship_type = f"dns_{record_type.lower()}_record"
confidence, confidence = 0.8
raw_data
)) # Add relationship
result.add_relationship(
source_node=domain,
target_node=target,
relationship_type=relationship_type,
provider=self.name,
confidence=confidence,
raw_data=raw_data
)
# Add target to records list
dns_records.append(target)
# Log relationship discovery with IP version info
discovery_method = f"dns_{record_type.lower()}_record"
if ip_version:
discovery_method += f"_ipv{ip_version}"
self.log_relationship_discovery( self.log_relationship_discovery(
source_node=domain, source_node=domain,
@ -186,13 +254,33 @@ class DNSProvider(BaseProvider):
relationship_type=relationship_type, relationship_type=relationship_type,
confidence_score=confidence, confidence_score=confidence,
raw_data=raw_data, raw_data=raw_data,
discovery_method=f"dns_{record_type.lower()}_record" discovery_method=discovery_method
) )
# FIXED: Create attribute with specific name for each record type
if dns_records:
# Use record type specific attribute name (e.g., 'a_records', 'mx_records', etc.)
attribute_name = f"{record_type.lower()}_records"
metadata = {'record_type': record_type, 'ttl': response.ttl}
# Add IP version info for A/AAAA records
if record_type in ['A', 'AAAA'] and dns_records:
first_ip_version = get_ip_version(dns_records[0])
if first_ip_version:
metadata['ip_version'] = first_ip_version
result.add_attribute(
target_node=domain,
name=attribute_name, # UNIQUE name for each record type!
value=dns_records,
attr_type='dns_record_list',
provider=self.name,
confidence=0.8,
metadata=metadata
)
except Exception as e: except Exception as e:
self.failed_requests += 1 self.failed_requests += 1
self.logger.logger.debug(f"{record_type} record query failed for {domain}: {e}") self.logger.logger.debug(f"{record_type} record query failed for {domain}: {e}")
# Re-raise the exception so the scanner can handle it
raise e raise e
return relationships

302
providers/shodan_provider.py
View File

@ -1,20 +1,20 @@
# dnsrecon/providers/shodan_provider.py # dnsrecon/providers/shodan_provider.py
import json import json
import os
from pathlib import Path from pathlib import Path
from typing import List, Dict, Any, Tuple from typing import Dict, Any
from datetime import datetime, timezone from datetime import datetime, timezone
import requests import requests
from .base_provider import BaseProvider from .base_provider import BaseProvider
from utils.helpers import _is_valid_ip, _is_valid_domain from core.provider_result import ProviderResult
from utils.helpers import _is_valid_ip, _is_valid_domain, get_ip_version, normalize_ip
class ShodanProvider(BaseProvider): class ShodanProvider(BaseProvider):
""" """
Provider for querying Shodan API for IP address information. Provider for querying Shodan API for IP address information.
Now uses session-specific API keys, is limited to IP-only queries, and includes caching. Now returns standardized ProviderResult objects with caching support for IPv4 and IPv6.
""" """
def __init__(self, name=None, session_config=None): def __init__(self, name=None, session_config=None):
@ -53,8 +53,19 @@ class ShodanProvider(BaseProvider):
return {'domains': False, 'ips': True} return {'domains': False, 'ips': True}
def _get_cache_file_path(self, ip: str) -> Path: def _get_cache_file_path(self, ip: str) -> Path:
"""Generate cache file path for an IP address.""" """
safe_ip = ip.replace('.', '_').replace(':', '_') Generate cache file path for an IP address (IPv4 or IPv6).
IPv6 addresses contain colons which are replaced with underscores for filesystem safety.
"""
# Normalize the IP address first to ensure consistent caching
normalized_ip = normalize_ip(ip)
if not normalized_ip:
# Fallback for invalid IPs
safe_ip = ip.replace('.', '_').replace(':', '_')
else:
# Replace problematic characters for both IPv4 and IPv6
safe_ip = normalized_ip.replace('.', '_').replace(':', '_')
return self.cache_dir / f"{safe_ip}.json" return self.cache_dir / f"{safe_ip}.json"
def _get_cache_status(self, cache_file_path: Path) -> str: def _get_cache_status(self, cache_file_path: Path) -> str:
@ -85,115 +96,254 @@ class ShodanProvider(BaseProvider):
except (json.JSONDecodeError, ValueError, KeyError): except (json.JSONDecodeError, ValueError, KeyError):
return "stale" return "stale"
def query_domain(self, domain: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]: def query_domain(self, domain: str) -> ProviderResult:
""" """
Domain queries are no longer supported for the Shodan provider. Domain queries are no longer supported for the Shodan provider.
"""
return []
def query_ip(self, ip: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]: Args:
domain: Domain to investigate
Returns:
Empty ProviderResult
""" """
Query Shodan for information about an IP address, with caching of processed relationships. return ProviderResult()
def query_ip(self, ip: str) -> ProviderResult:
"""
Query Shodan for information about an IP address (IPv4 or IPv6), with caching of processed data.
Args:
ip: IP address to investigate (IPv4 or IPv6)
Returns:
ProviderResult containing discovered relationships and attributes
""" """
if not _is_valid_ip(ip) or not self.is_available(): if not _is_valid_ip(ip) or not self.is_available():
return [] return ProviderResult()
cache_file = self._get_cache_file_path(ip) # Normalize IP address for consistent processing
normalized_ip = normalize_ip(ip)
if not normalized_ip:
return ProviderResult()
cache_file = self._get_cache_file_path(normalized_ip)
cache_status = self._get_cache_status(cache_file) cache_status = self._get_cache_status(cache_file)
relationships = [] result = ProviderResult()
try: try:
if cache_status == "fresh": if cache_status == "fresh":
relationships = self._load_from_cache(cache_file) result = self._load_from_cache(cache_file)
self.logger.logger.info(f"Using cached Shodan relationships for {ip}") self.logger.logger.info(f"Using cached Shodan data for {normalized_ip}")
else: # "stale" or "not_found" else: # "stale" or "not_found"
url = f"{self.base_url}/shodan/host/{ip}" url = f"{self.base_url}/shodan/host/{normalized_ip}"
params = {'key': self.api_key} params = {'key': self.api_key}
response = self.make_request(url, method="GET", params=params, target_indicator=ip) response = self.make_request(url, method="GET", params=params, target_indicator=normalized_ip)
if response and response.status_code == 200: if response and response.status_code == 200:
data = response.json() data = response.json()
# Process the data into relationships BEFORE caching # Process the data into ProviderResult BEFORE caching
relationships = self._process_shodan_data(ip, data) result = self._process_shodan_data(normalized_ip, data)
self._save_to_cache(cache_file, relationships) # Save the processed relationships self._save_to_cache(cache_file, result, data) # Save both result and raw data
elif response and response.status_code == 404:
# Handle 404 "No information available" as successful empty result
try:
error_data = response.json()
if "No information available" in error_data.get('error', ''):
# This is a successful query - Shodan just has no data
self.logger.logger.debug(f"Shodan has no information for {normalized_ip}")
result = ProviderResult() # Empty but successful result
# Cache the empty result to avoid repeated queries
self._save_to_cache(cache_file, result, {'error': 'No information available'})
else:
# Some other 404 error - treat as failure
raise requests.exceptions.RequestException(f"Shodan API returned 404: {error_data}")
except (ValueError, KeyError):
# Could not parse JSON response - treat as failure
raise requests.exceptions.RequestException(f"Shodan API returned 404 with unparseable response")
elif cache_status == "stale": elif cache_status == "stale":
# If API fails on a stale cache, use the old data # If API fails on a stale cache, use the old data
relationships = self._load_from_cache(cache_file) result = self._load_from_cache(cache_file)
else:
# Other HTTP error codes should be treated as failures
status_code = response.status_code if response else "No response"
raise requests.exceptions.RequestException(f"Shodan API returned HTTP {status_code}")
except requests.exceptions.RequestException as e: except requests.exceptions.RequestException as e:
self.logger.logger.error(f"Shodan API query failed for {ip}: {e}") self.logger.logger.info(f"Shodan API query returned no info for {normalized_ip}: {e}")
if cache_status == "stale": if cache_status == "stale":
relationships = self._load_from_cache(cache_file) result = self._load_from_cache(cache_file)
else:
# Re-raise for retry scheduling - but only for actual failures
raise e
return relationships return result
def _load_from_cache(self, cache_file_path: Path) -> List[Tuple[str, str, str, float, Dict[str, Any]]]: def _load_from_cache(self, cache_file_path: Path) -> ProviderResult:
"""Load processed Shodan relationships from a cache file.""" """Load processed Shodan data from a cache file."""
try: try:
with open(cache_file_path, 'r') as f: with open(cache_file_path, 'r') as f:
cache_content = json.load(f) cache_content = json.load(f)
# The entire file content is the list of relationships
return cache_content.get("relationships", [])
except (json.JSONDecodeError, FileNotFoundError, KeyError):
return []
def _save_to_cache(self, cache_file_path: Path, relationships: List[Tuple[str, str, str, float, Dict[str, Any]]]) -> None: result = ProviderResult()
"""Save processed Shodan relationships to a cache file."""
# Reconstruct relationships
for rel_data in cache_content.get("relationships", []):
result.add_relationship(
source_node=rel_data["source_node"],
target_node=rel_data["target_node"],
relationship_type=rel_data["relationship_type"],
provider=rel_data["provider"],
confidence=rel_data["confidence"],
raw_data=rel_data.get("raw_data", {})
)
# Reconstruct attributes
for attr_data in cache_content.get("attributes", []):
result.add_attribute(
target_node=attr_data["target_node"],
name=attr_data["name"],
value=attr_data["value"],
attr_type=attr_data["type"],
provider=attr_data["provider"],
confidence=attr_data["confidence"],
metadata=attr_data.get("metadata", {})
)
return result
except (json.JSONDecodeError, FileNotFoundError, KeyError):
return ProviderResult()
def _save_to_cache(self, cache_file_path: Path, result: ProviderResult, raw_data: Dict[str, Any]) -> None:
"""Save processed Shodan data to a cache file."""
try: try:
cache_data = { cache_data = {
"last_upstream_query": datetime.now(timezone.utc).isoformat(), "last_upstream_query": datetime.now(timezone.utc).isoformat(),
"relationships": relationships "raw_data": raw_data, # Preserve original for forensic purposes
"relationships": [
{
"source_node": rel.source_node,
"target_node": rel.target_node,
"relationship_type": rel.relationship_type,
"confidence": rel.confidence,
"provider": rel.provider,
"raw_data": rel.raw_data
} for rel in result.relationships
],
"attributes": [
{
"target_node": attr.target_node,
"name": attr.name,
"value": attr.value,
"type": attr.type,
"provider": attr.provider,
"confidence": attr.confidence,
"metadata": attr.metadata
} for attr in result.attributes
]
} }
with open(cache_file_path, 'w') as f: with open(cache_file_path, 'w') as f:
json.dump(cache_data, f, separators=(',', ':')) json.dump(cache_data, f, separators=(',', ':'), default=str)
except Exception as e: except Exception as e:
self.logger.logger.warning(f"Failed to save Shodan cache for {cache_file_path.name}: {e}") self.logger.logger.warning(f"Failed to save Shodan cache for {cache_file_path.name}: {e}")
def _process_shodan_data(self, ip: str, data: Dict[str, Any]) -> List[Tuple[str, str, str, float, Dict[str, Any]]]: def _process_shodan_data(self, ip: str, data: Dict[str, Any]) -> ProviderResult:
""" """
Process Shodan data to extract relationships. VERIFIED: Process Shodan data creating ISP nodes with ASN attributes and proper relationships.
Enhanced to include IP version information for IPv6 addresses.
""" """
relationships = [] result = ProviderResult()
# Extract hostname relationships # Determine IP version for metadata
hostnames = data.get('hostnames', []) ip_version = get_ip_version(ip)
for hostname in hostnames:
if _is_valid_domain(hostname):
relationships.append((
ip,
hostname,
'a_record',
0.8,
data
))
self.log_relationship_discovery(
source_node=ip,
target_node=hostname,
relationship_type='a_record',
confidence_score=0.8,
raw_data=data,
discovery_method="shodan_host_lookup"
)
# Extract ASN relationship # VERIFIED: Extract ISP information and create proper ISP node with ASN
asn = data.get('asn') isp_name = data.get('org')
if asn: asn_value = data.get('asn')
asn_name = f"AS{asn[2:]}" if isinstance(asn, str) and asn.startswith('AS') else f"AS{asn}"
relationships.append(( if isp_name and asn_value:
ip, # Create relationship from IP to ISP
asn_name, result.add_relationship(
'asn_membership',
0.7,
data
))
self.log_relationship_discovery(
source_node=ip, source_node=ip,
target_node=asn_name, target_node=isp_name,
relationship_type='asn_membership', relationship_type='shodan_isp',
confidence_score=0.7, provider=self.name,
raw_data=data, confidence=0.9,
discovery_method="shodan_asn_lookup" raw_data={'asn': asn_value, 'shodan_org': isp_name, 'ip_version': ip_version}
) )
return relationships # Add ASN as attribute to the ISP node
result.add_attribute(
target_node=isp_name,
name='asn',
value=asn_value,
attr_type='isp_info',
provider=self.name,
confidence=0.9,
metadata={'description': 'Autonomous System Number from Shodan', 'ip_version': ip_version}
)
# Also add organization name as attribute to ISP node for completeness
result.add_attribute(
target_node=isp_name,
name='organization_name',
value=isp_name,
attr_type='isp_info',
provider=self.name,
confidence=0.9,
metadata={'description': 'Organization name from Shodan', 'ip_version': ip_version}
)
# Process hostnames (reverse DNS)
for key, value in data.items():
if key == 'hostnames':
for hostname in value:
if _is_valid_domain(hostname):
# Use appropriate relationship type based on IP version
if ip_version == 6:
relationship_type = 'shodan_aaaa_record'
else:
relationship_type = 'shodan_a_record'
result.add_relationship(
source_node=ip,
target_node=hostname,
relationship_type=relationship_type,
provider=self.name,
confidence=0.8,
raw_data={**data, 'ip_version': ip_version}
)
self.log_relationship_discovery(
source_node=ip,
target_node=hostname,
relationship_type=relationship_type,
confidence_score=0.8,
raw_data={**data, 'ip_version': ip_version},
discovery_method=f"shodan_host_lookup_ipv{ip_version}"
)
elif key == 'ports':
# Add open ports as attributes to the IP
for port in value:
result.add_attribute(
target_node=ip,
name='shodan_open_port',
value=port,
attr_type='shodan_network_info',
provider=self.name,
confidence=0.9,
metadata={'ip_version': ip_version}
)
elif isinstance(value, (str, int, float, bool)) and value is not None:
# Add other Shodan fields as IP attributes (keep raw field names)
result.add_attribute(
target_node=ip,
name=key, # Raw field name from Shodan API
value=value,
attr_type='shodan_info',
provider=self.name,
confidence=0.9,
metadata={'ip_version': ip_version}
)
return result

14
requirements.txt
View File

@ -1,10 +1,10 @@
Flask>=2.3.3 Flask
networkx>=3.1 networkx
requests>=2.31.0 requests
python-dateutil>=2.8.2 python-dateutil
Werkzeug>=2.3.7 Werkzeug
urllib3>=2.0.0 urllib3
dnspython>=2.4.2 dnspython
gunicorn gunicorn
redis redis
python-dotenv python-dotenv

2217
static/css/main.css
View File

File diff suppressed because it is too large Load Diff

210
static/js/graph.js
View File

@ -1,6 +1,7 @@
/** /**
* Graph visualization module for DNSRecon * Graph visualization module for DNSRecon
* Handles network graph rendering using vis.js with proper large entity node hiding * Handles network graph rendering using vis.js with proper large entity node hiding
* UPDATED: Now compatible with a strictly flat, unified data model for attributes.
*/ */
const contextMenuCSS = ` const contextMenuCSS = `
.graph-context-menu { .graph-context-menu {
@ -213,7 +214,6 @@ class GraphManager {
}); });
document.body.appendChild(this.contextMenu); document.body.appendChild(this.contextMenu);
console.log('Context menu created and added to body');
} }
/** /**
@ -290,7 +290,6 @@ class GraphManager {
// FIXED: Right-click context menu // FIXED: Right-click context menu
this.container.addEventListener('contextmenu', (event) => { this.container.addEventListener('contextmenu', (event) => {
event.preventDefault(); event.preventDefault();
console.log('Right-click detected at:', event.offsetX, event.offsetY);
// Get coordinates relative to the canvas // Get coordinates relative to the canvas
const pointer = { const pointer = {
@ -299,7 +298,6 @@ class GraphManager {
}; };
const nodeId = this.network.getNodeAt(pointer); const nodeId = this.network.getNodeAt(pointer);
console.log('Node at pointer:', nodeId);
if (nodeId) { if (nodeId) {
// Pass the original client event for positioning // Pass the original client event for positioning
@ -340,19 +338,12 @@ class GraphManager {
// Stabilization events with progress // Stabilization events with progress
this.network.on('stabilizationProgress', (params) => { this.network.on('stabilizationProgress', (params) => {
const progress = params.iterations / params.total; const progress = params.iterations / params.total;
this.updateStabilizationProgress(progress);
}); });
this.network.on('stabilizationIterationsDone', () => { this.network.on('stabilizationIterationsDone', () => {
this.onStabilizationComplete(); this.onStabilizationComplete();
}); });
// Selection events
this.network.on('select', (params) => {
console.log('Selected nodes:', params.nodes);
console.log('Selected edges:', params.edges);
});
// Click away to hide context menu // Click away to hide context menu
document.addEventListener('click', (e) => { document.addEventListener('click', (e) => {
if (!this.contextMenu.contains(e.target)) { if (!this.contextMenu.contains(e.target)) {
@ -376,28 +367,62 @@ class GraphManager {
this.initialize(); this.initialize();
} }
this.initialTargetIds = new Set(graphData.initial_targets || []);
// Check if we have actual data to display
const hasData = graphData.nodes.length > 0 || graphData.edges.length > 0;
// Handle placeholder visibility
const placeholder = this.container.querySelector('.graph-placeholder');
if (placeholder) {
if (hasData) {
placeholder.style.display = 'none';
} else {
placeholder.style.display = 'flex';
// Early return if no data to process
return;
}
}
this.largeEntityMembers.clear(); this.largeEntityMembers.clear();
const largeEntityMap = new Map(); const largeEntityMap = new Map();
graphData.nodes.forEach(node => { graphData.nodes.forEach(node => {
if (node.type === 'large_entity' && node.attributes && Array.isArray(node.attributes.nodes)) { if (node.type === 'large_entity' && node.attributes) {
node.attributes.nodes.forEach(nodeId => { const nodesAttribute = this.findAttributeByName(node.attributes, 'nodes');
largeEntityMap.set(nodeId, node.id); if (nodesAttribute && Array.isArray(nodesAttribute.value)) {
this.largeEntityMembers.add(nodeId); nodesAttribute.value.forEach(nodeId => {
}); largeEntityMap.set(nodeId, node.id);
this.largeEntityMembers.add(nodeId);
});
}
} }
}); });
const filteredNodes = graphData.nodes.filter(node => { const filteredNodes = graphData.nodes.filter(node => {
// Only include nodes that are NOT members of large entities, but always include the container itself
return !this.largeEntityMembers.has(node.id) || node.type === 'large_entity'; return !this.largeEntityMembers.has(node.id) || node.type === 'large_entity';
}); });
console.log(`Filtered ${graphData.nodes.length - filteredNodes.length} large entity member nodes from visualization`); console.log(`Filtered ${graphData.nodes.length - filteredNodes.length} large entity member nodes from visualization`);
// Process only the filtered nodes // Process nodes with proper certificate coloring
const processedNodes = filteredNodes.map(node => { const processedNodes = filteredNodes.map(node => {
return this.processNode(node); const processed = this.processNode(node);
// Apply certificate-based coloring here in frontend
if (node.type === 'domain' && Array.isArray(node.attributes)) {
const certInfo = this.analyzeCertificateInfo(node.attributes);
if (certInfo.hasExpiredOnly) {
// Red for domains with only expired/invalid certificates
processed.color = { background: '#ff6b6b', border: '#cc5555' };
} else if (!certInfo.hasCertificates) {
// Grey for domains with no certificates
processed.color = { background: '#c7c7c7', border: '#999999' };
}
// Valid certificates use default green (handled by processNode)
}
return processed;
}); });
const mergedEdges = {}; const mergedEdges = {};
@ -434,24 +459,19 @@ class GraphManager {
const existingNodeIds = this.nodes.getIds(); const existingNodeIds = this.nodes.getIds();
const existingEdgeIds = this.edges.getIds(); const existingEdgeIds = this.edges.getIds();
// Add new nodes with fade-in animation
const newNodes = processedNodes.filter(node => !existingNodeIds.includes(node.id)); const newNodes = processedNodes.filter(node => !existingNodeIds.includes(node.id));
const newEdges = processedEdges.filter(edge => !existingEdgeIds.includes(edge.id)); const newEdges = processedEdges.filter(edge => !existingEdgeIds.includes(edge.id));
// Update existing data
this.nodes.update(processedNodes); this.nodes.update(processedNodes);
this.edges.update(processedEdges); this.edges.update(processedEdges);
// After data is loaded, apply filters
this.updateFilterControls(); this.updateFilterControls();
this.applyAllFilters(); this.applyAllFilters();
// Highlight new additions briefly
if (newNodes.length > 0 || newEdges.length > 0) { if (newNodes.length > 0 || newEdges.length > 0) {
setTimeout(() => this.highlightNewElements(newNodes, newEdges), 100); setTimeout(() => this.highlightNewElements(newNodes, newEdges), 100);
} }
// Auto-fit view for small graphs or first update
if (processedNodes.length <= 10 || existingNodeIds.length === 0) { if (processedNodes.length <= 10 || existingNodeIds.length === 0) {
setTimeout(() => this.fitView(), 800); setTimeout(() => this.fitView(), 800);
} }
@ -465,9 +485,62 @@ class GraphManager {
} }
} }
analyzeCertificateInfo(attributes) {
let hasCertificates = false;
let hasValidCertificates = false;
let hasExpiredCertificates = false;
for (const attr of attributes) {
const attrName = (attr.name || '').toLowerCase();
const attrProvider = (attr.provider || '').toLowerCase();
const attrValue = attr.value;
// Look for certificate attributes from crtsh provider
if (attrProvider === 'crtsh' || attrName.startsWith('cert_')) {
hasCertificates = true;
// Check certificate validity using raw attribute names
if (attrName === 'cert_is_currently_valid') {
if (attrValue === true) {
hasValidCertificates = true;
} else if (attrValue === false) {
hasExpiredCertificates = true;
}
}
// Check for expiry indicators
else if (attrName === 'cert_expires_soon' && attrValue === true) {
hasExpiredCertificates = true;
}
else if (attrName.includes('expired') && attrValue === true) {
hasExpiredCertificates = true;
}
}
}
return {
hasCertificates,
hasValidCertificates,
hasExpiredCertificates,
hasExpiredOnly: hasExpiredCertificates && !hasValidCertificates
};
}
/** /**
* Process node data with styling and metadata * UPDATED: Helper method to find an attribute by name in the standardized attributes list
* @param {Object} node - Raw node data * @param {Array} attributes - List of StandardAttribute objects
* @param {string} name - Attribute name to find
* @returns {Object|null} The attribute object if found, null otherwise
*/
findAttributeByName(attributes, name) {
if (!Array.isArray(attributes)) {
return null;
}
return attributes.find(attr => attr.name === name) || null;
}
/**
* UPDATED: Process node data with styling and metadata for the flat data model
* @param {Object} node - Raw node data with standardized attributes
* @returns {Object} Processed node data * @returns {Object} Processed node data
*/ */
processNode(node) { processNode(node) {
@ -478,7 +551,7 @@ class GraphManager {
size: this.getNodeSize(node.type), size: this.getNodeSize(node.type),
borderColor: this.getNodeBorderColor(node.type), borderColor: this.getNodeBorderColor(node.type),
shape: this.getNodeShape(node.type), shape: this.getNodeShape(node.type),
attributes: node.attributes || {}, attributes: node.attributes || [],
description: node.description || '', description: node.description || '',
metadata: node.metadata || {}, metadata: node.metadata || {},
type: node.type, type: node.type,
@ -491,26 +564,33 @@ class GraphManager {
processedNode.borderWidth = Math.max(2, Math.floor(node.confidence * 5)); processedNode.borderWidth = Math.max(2, Math.floor(node.confidence * 5));
} }
// Style based on certificate validity // FIXED: Certificate-based domain coloring
if (node.type === 'domain') { if (node.type === 'domain' && Array.isArray(node.attributes)) {
if (node.attributes && node.attributes.certificates && node.attributes.certificates.has_valid_cert === false) { const certInfo = this.analyzeCertificateInfo(node.attributes);
processedNode.color = { background: '#888888', border: '#666666' };
if (certInfo.hasExpiredOnly) {
// Red for domains with only expired/invalid certificates
processedNode.color = '#ff6b6b';
processedNode.borderColor = '#cc5555';
} else if (!certInfo.hasCertificates) {
// Grey for domains with no certificates
processedNode.color = '#c7c7c7';
processedNode.borderColor = '#999999';
} }
// Green for valid certificates (default color)
} }
// Handle merged correlation objects (similar to large entities) // Handle merged correlation objects
if (node.type === 'correlation_object') { if (node.type === 'correlation_object') {
const metadata = node.metadata || {}; const metadata = node.metadata || {};
const values = metadata.values || []; const values = metadata.values || [];
const mergeCount = metadata.merge_count || 1; const mergeCount = metadata.merge_count || 1;
if (mergeCount > 1) { if (mergeCount > 1) {
// Display as merged correlation container
processedNode.label = `Correlations (${mergeCount})`; processedNode.label = `Correlations (${mergeCount})`;
processedNode.title = `Merged correlation container with ${mergeCount} values: ${values.slice(0, 3).join(', ')}${values.length > 3 ? '...' : ''}`; processedNode.title = `Merged correlation container with ${mergeCount} values: ${values.slice(0, 3).join(', ')}${values.length > 3 ? '...' : ''}`;
processedNode.borderWidth = 3; // Thicker border for merged nodes processedNode.borderWidth = 3;
} else { } else {
// Single correlation value
const value = Array.isArray(values) && values.length > 0 ? values[0] : (metadata.value || 'Unknown'); const value = Array.isArray(values) && values.length > 0 ? values[0] : (metadata.value || 'Unknown');
const displayValue = typeof value === 'string' && value.length > 20 ? value.substring(0, 17) + '...' : value; const displayValue = typeof value === 'string' && value.length > 20 ? value.substring(0, 17) + '...' : value;
processedNode.label = `${displayValue}`; processedNode.label = `${displayValue}`;
@ -521,6 +601,7 @@ class GraphManager {
return processedNode; return processedNode;
} }
/** /**
* Process edge data with styling and metadata * Process edge data with styling and metadata
* @param {Object} edge - Raw edge data * @param {Object} edge - Raw edge data
@ -584,7 +665,8 @@ class GraphManager {
const colors = { const colors = {
'domain': '#00ff41', // Green 'domain': '#00ff41', // Green
'ip': '#ff9900', // Amber 'ip': '#ff9900', // Amber
'asn': '#00aaff', // Blue 'isp': '#00aaff', // Blue
'ca': '#ff6b6b', // Red
'large_entity': '#ff6b6b', // Red for large entities 'large_entity': '#ff6b6b', // Red for large entities
'correlation_object': '#9620c0ff' 'correlation_object': '#9620c0ff'
}; };
@ -600,7 +682,8 @@ class GraphManager {
const borderColors = { const borderColors = {
'domain': '#00aa2e', 'domain': '#00aa2e',
'ip': '#cc7700', 'ip': '#cc7700',
'asn': '#0088cc', 'isp': '#0088cc',
'ca': '#cc5555',
'correlation_object': '#c235c9ff' 'correlation_object': '#c235c9ff'
}; };
return borderColors[nodeType] || '#666666'; return borderColors[nodeType] || '#666666';
@ -615,9 +698,10 @@ class GraphManager {
const sizes = { const sizes = {
'domain': 12, 'domain': 12,
'ip': 14, 'ip': 14,
'asn': 16, 'isp': 16,
'ca': 16,
'correlation_object': 8, 'correlation_object': 8,
'large_entity': 5 'large_entity': 25
}; };
return sizes[nodeType] || 12; return sizes[nodeType] || 12;
} }
@ -631,9 +715,10 @@ class GraphManager {
const shapes = { const shapes = {
'domain': 'dot', 'domain': 'dot',
'ip': 'square', 'ip': 'square',
'asn': 'triangle', 'isp': 'triangle',
'ca': 'diamond',
'correlation_object': 'hexagon', 'correlation_object': 'hexagon',
'large_entity': 'database' 'large_entity': 'dot'
}; };
return shapes[nodeType] || 'dot'; return shapes[nodeType] || 'dot';
} }
@ -889,15 +974,6 @@ class GraphManager {
}, 2000); }, 2000);
} }
/**
* Update stabilization progress
* @param {number} progress - Progress value (0-1)
*/
updateStabilizationProgress(progress) {
// Could show a progress indicator if needed
console.log(`Graph stabilization: ${(progress * 100).toFixed(1)}%`);
}
/** /**
* Handle stabilization completion * Handle stabilization completion
*/ */
@ -982,7 +1058,7 @@ class GraphManager {
this.edges.clear(); this.edges.clear();
this.history = []; this.history = [];
this.largeEntityMembers.clear(); // Clear large entity tracking this.largeEntityMembers.clear(); // Clear large entity tracking
this.clearInitialTargets(); this.initialTargetIds.clear();
// Show placeholder // Show placeholder
const placeholder = this.container.querySelector('.graph-placeholder'); const placeholder = this.container.querySelector('.graph-placeholder');
@ -1085,11 +1161,11 @@ class GraphManager {
adjacencyList adjacencyList
); );
console.log(`Reachability analysis complete:`, { /*console.log(`Reachability analysis complete:`, {
reachable: analysis.reachableNodes.size, reachable: analysis.reachableNodes.size,
unreachable: analysis.unreachableNodes.size, unreachable: analysis.unreachableNodes.size,
clusters: analysis.isolatedClusters.length clusters: analysis.isolatedClusters.length
}); });*/
return analysis; return analysis;
} }
@ -1157,16 +1233,6 @@ class GraphManager {
}; };
} }
addInitialTarget(targetId) {
this.initialTargetIds.add(targetId);
console.log("Initial targets:", this.initialTargetIds);
}
clearInitialTargets() {
this.initialTargetIds.clear();
console.log("Initial targets cleared.");
}
updateFilterControls() { updateFilterControls() {
if (!this.filterPanel) return; if (!this.filterPanel) return;
const nodeTypes = new Set(this.nodes.get().map(n => n.type)); const nodeTypes = new Set(this.nodes.get().map(n => n.type));
@ -1204,7 +1270,6 @@ class GraphManager {
* Replaces the existing applyAllFilters() method * Replaces the existing applyAllFilters() method
*/ */
applyAllFilters() { applyAllFilters() {
console.log("Applying filters with enhanced reachability analysis...");
if (this.nodes.length === 0) return; if (this.nodes.length === 0) return;
// Get filter criteria from UI // Get filter criteria from UI
@ -1261,22 +1326,10 @@ class GraphManager {
timestamp: Date.now() timestamp: Date.now()
}; };
// Apply hiding with forensic documentation const updates = nodesToHide.map(id => ({ id: id, hidden: true }));
const updates = nodesToHide.map(id => ({
id: id,
hidden: true,
forensicNote: `Hidden due to reachability analysis from ${nodeId}`
}));
this.nodes.update(updates); this.nodes.update(updates);
this.addToHistory('hide', historyData); this.addToHistory('hide', historyData);
console.log(`Forensic hide operation: ${nodesToHide.length} nodes hidden`, {
originalTarget: nodeId,
cascadeNodes: nodesToHide.length - 1,
isolatedClusters: analysis.isolatedClusters.length
});
return { return {
hiddenNodes: nodesToHide, hiddenNodes: nodesToHide,
isolatedClusters: analysis.isolatedClusters isolatedClusters: analysis.isolatedClusters
@ -1360,8 +1413,6 @@ class GraphManager {
// Handle operation results // Handle operation results
if (!operationFailed) { if (!operationFailed) {
this.addToHistory('delete', historyData); this.addToHistory('delete', historyData);
console.log(`Forensic delete operation completed:`, historyData.forensicAnalysis);
return { return {
success: true, success: true,
deletedNodes: nodesToDelete, deletedNodes: nodesToDelete,
@ -1452,7 +1503,6 @@ class GraphManager {
e.stopPropagation(); e.stopPropagation();
const action = e.currentTarget.dataset.action; const action = e.currentTarget.dataset.action;
const nodeId = e.currentTarget.dataset.nodeId; const nodeId = e.currentTarget.dataset.nodeId;
console.log('Context menu action:', action, 'for node:', nodeId);
this.performContextMenuAction(action, nodeId); this.performContextMenuAction(action, nodeId);
this.hideContextMenu(); this.hideContextMenu();
}); });
@ -1473,8 +1523,6 @@ class GraphManager {
* Updates the existing performContextMenuAction() method * Updates the existing performContextMenuAction() method
*/ */
performContextMenuAction(action, nodeId) { performContextMenuAction(action, nodeId) {
console.log('Performing enhanced action:', action, 'on node:', nodeId);
switch (action) { switch (action) {
case 'focus': case 'focus':
this.focusOnNode(nodeId); this.focusOnNode(nodeId);

1797
static/js/main.js
View File

File diff suppressed because it is too large Load Diff

196
templates/index.html
View File

@ -1,5 +1,6 @@
<!DOCTYPE html> <!DOCTYPE html>
<html lang="en"> <html lang="en">
<head> <head>
<meta charset="UTF-8"> <meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0"> <meta name="viewport" content="width=device-width, initial-scale=1.0">
@ -7,8 +8,11 @@
<link rel="stylesheet" href="{{ url_for('static', filename='css/main.css') }}"> <link rel="stylesheet" href="{{ url_for('static', filename='css/main.css') }}">
<script src="https://cdnjs.cloudflare.com/ajax/libs/vis/4.21.0/vis.min.js"></script> <script src="https://cdnjs.cloudflare.com/ajax/libs/vis/4.21.0/vis.min.js"></script>
<link href="https://cdnjs.cloudflare.com/ajax/libs/vis/4.21.0/vis.min.css" rel="stylesheet" type="text/css"> <link href="https://cdnjs.cloudflare.com/ajax/libs/vis/4.21.0/vis.min.css" rel="stylesheet" type="text/css">
<link href="https://fonts.googleapis.com/css2?family=Roboto+Mono:wght@300;400;500;700&family=Special+Elite&display=swap" rel="stylesheet"> <link
href="https://fonts.googleapis.com/css2?family=Roboto+Mono:wght@300;400;500;700&family=Special+Elite&display=swap"
rel="stylesheet">
</head> </head>
<body> <body>
<div class="container"> <div class="container">
<header class="header"> <header class="header">
@ -49,9 +53,9 @@
<span class="btn-icon">[STOP]</span> <span class="btn-icon">[STOP]</span>
<span>Terminate Scan</span> <span>Terminate Scan</span>
</button> </button>
<button id="export-results" class="btn btn-secondary"> <button id="export-options" class="btn btn-secondary">
<span class="btn-icon">[EXPORT]</span> <span class="btn-icon">[EXPORT]</span>
<span>Download Results</span> <span>Export Options</span>
</button> </button>
<button id="configure-settings" class="btn btn-secondary"> <button id="configure-settings" class="btn btn-secondary">
<span class="btn-icon">[API]</span> <span class="btn-icon">[API]</span>
@ -95,11 +99,13 @@
</div> </div>
<div class="progress-placeholder"> <div class="progress-placeholder">
<span class="status-label"> <span class="status-label">
⚠️ <strong>Important:</strong> Scanning large public services (e.g., Google, Cloudflare, AWS) is ⚠️ <strong>Important:</strong> Scanning large public services (e.g., Google, Cloudflare,
AWS) is
<strong>discouraged</strong> due to rate limits (e.g., crt.sh). <strong>discouraged</strong> due to rate limits (e.g., crt.sh).
<br><br> <br><br>
Our task scheduler operates on a <strong>priority-based queue</strong>: Our task scheduler operates on a <strong>priority-based queue</strong>:
Short, targeted tasks like DNS are processed first, while resource-intensive requests (e.g., crt.sh) Short, targeted tasks like DNS are processed first, while resource-intensive requests (e.g.,
crt.sh)
are <strong>automatically deprioritized</strong> and may be processed later. are <strong>automatically deprioritized</strong> and may be processed later.
</span> </span>
</div> </div>
@ -114,9 +120,10 @@
<div id="network-graph" class="graph-container"> <div id="network-graph" class="graph-container">
<div class="graph-placeholder"> <div class="graph-placeholder">
<div class="placeholder-content"> <div class="placeholder-content">
<div class="placeholder-icon">[○]</div> <div class="placeholder-icon">[◯]</div>
<div class="placeholder-text">Infrastructure map will appear here</div> <div class="placeholder-text">Infrastructure map will appear here</div>
<div class="placeholder-subtext">Start a reconnaissance scan to visualize relationships</div> <div class="placeholder-subtext">Start a reconnaissance scan to visualize relationships
</div>
</div> </div>
</div> </div>
</div> </div>
@ -126,29 +133,30 @@
<div class="legend-color" style="background-color: #00ff41;"></div> <div class="legend-color" style="background-color: #00ff41;"></div>
<span>Domains</span> <span>Domains</span>
</div> </div>
<div class="legend-item">
<div class="legend-color" style="background-color: #c92f2f;"></div>
<span>Domain (no valid cert)</span>
</div>
<div class="legend-item">
<div class="legend-color" style="background-color: #c7c7c7;"></div>
<span>Domain (never had cert)</span>
</div>
<div class="legend-item"> <div class="legend-item">
<div class="legend-color" style="background-color: #ff9900;"></div> <div class="legend-color" style="background-color: #ff9900;"></div>
<span>IP Addresses</span> <span>IP Addresses</span>
</div> </div>
<div class="legend-item"> <div class="legend-item">
<div class="legend-color" style="background-color: #c7c7c7;"></div> <div class="legend-color" style="background-color: #00aaff;"></div>
<span>Domain (invalid cert)</span> <span>ISPs</span>
</div>
<div class="legend-item">
<div class="legend-color" style="background-color: #9d4edd;"></div>
<span>Correlation Objects</span>
</div>
<div class="legend-item">
<div class="legend-edge high-confidence"></div>
<span>High Confidence</span>
</div>
<div class="legend-item">
<div class="legend-edge medium-confidence"></div>
<span>Medium Confidence</span>
</div> </div>
<div class="legend-item"> <div class="legend-item">
<div class="legend-color" style="background-color: #ff6b6b;"></div> <div class="legend-color" style="background-color: #ff6b6b;"></div>
<span>Large Entity</span> <span>Certificate Authorities</span>
</div>
<div class="legend-item">
<div class="legend-color" style="background-color: #9d4edd;"></div>
<span>Correlation Objects</span>
</div> </div>
</div> </div>
</section> </section>
@ -159,7 +167,7 @@
</div> </div>
<div id="provider-list" class="provider-list"> <div id="provider-list" class="provider-list">
</div> </div>
</section> </section>
</main> </main>
@ -181,61 +189,127 @@
</div> </div>
<div class="modal-body"> <div class="modal-body">
<div id="modal-details"> <div id="modal-details">
</div> </div>
</div> </div>
</div> </div>
</div> </div>
<!-- Settings Modal -->
<div id="settings-modal" class="modal"> <div id="settings-modal" class="modal">
<div class="modal-content"> <div class="modal-content">
<div class="modal-header"> <div class="modal-header">
<h3>Settings</h3> <h3>Scanner Configuration</h3>
<button id="settings-modal-close" class="modal-close">[×]</button> <button id="settings-modal-close" class="modal-close">[×]</button>
</div> </div>
<div class="modal-body"> <div class="modal-body">
<p class="modal-description"> <div class="modal-details">
Configure scan settings and API keys. Keys are stored in memory for the current session only. <!-- Scan Settings Section -->
Only provide API-keys you dont use for anything else. Don´t enter an API-key if you don´t trust me (best practice would that you don´t). <section class="modal-section">
</p> <details open>
<br> <summary>
<div class="input-group"> <span>⚙️ Scan Settings</span>
<label for="max-depth">Recursion Depth</label> </summary>
<select id="max-depth"> <div class="modal-section-content">
<option value="1">Depth 1 - Direct relationships</option> <div class="input-group">
<option value="2" selected>Depth 2 - Recommended</option> <label for="max-depth">Recursion Depth</label>
<option value="3">Depth 3 - Extended analysis</option> <select id="max-depth">
<option value="4">Depth 4 - Deep reconnaissance</option> <option value="1">Depth 1 - Direct relationships</option>
<option value="5">Depth 5 - Maximum depth</option> <option value="2" selected>Depth 2 - Recommended</option>
</select> <option value="3">Depth 3 - Extended analysis</option>
</div> <option value="4">Depth 4 - Deep reconnaissance</option>
<div id="api-key-inputs"> <option value="5">Depth 5 - Maximum depth</option>
</select>
</div>
</div>
</details>
</section>
<!-- Provider Configuration Section -->
<section class="modal-section">
<details open>
<summary>
<span>🔧 Provider Configuration</span>
<span class="merge-badge" id="provider-count">0</span>
</summary>
<div class="modal-section-content">
<div id="provider-config-list">
<!-- Dynamically populated -->
</div>
</div>
</details>
</section>
<!-- API Keys Section -->
<section class="modal-section">
<details>
<summary>
<span>🔑 API Keys</span>
<span class="merge-badge" id="api-key-count">0</span>
</summary>
<div class="modal-section-content">
<p class="placeholder-subtext" style="margin-bottom: 1rem;">
⚠️ API keys are stored in memory for the current session only.
Only provide API keys you don't use for anything else.
</p>
<div id="api-key-inputs">
<!-- Dynamically populated -->
</div>
</div>
</details>
</section>
<!-- Action Buttons -->
<div class="button-group" style="margin-top: 1.5rem;">
<button id="save-settings" class="btn btn-primary">
<span class="btn-icon">[SAVE]</span>
<span>Save Configuration</span>
</button>
<button id="reset-settings" class="btn btn-secondary">
<span class="btn-icon">[RESET]</span>
<span>Reset to Defaults</span>
</button>
</div> </div>
<div class="button-group" style="flex-direction: row; justify-content: flex-end;"> </div>
<button id="reset-api-keys" class="btn btn-secondary"> </div>
<span>Reset</span> </div>
</button> </div>
<button id="save-api-keys" class="btn btn-primary">
<span>Save API-Keys</span> <!-- Export Modal -->
</button> <div id="export-modal" class="modal">
<div class="modal-content">
<div class="modal-header">
<h3>Export Options</h3>
<button id="export-modal-close" class="modal-close">[×]</button>
</div>
<div class="modal-body">
<div class="modal-details">
<section class="modal-section">
<details open>
<summary>
<span>📊 Available Exports</span>
</summary>
<div class="modal-section-content">
<div class="button-group" style="margin-top: 1rem;">
<button id="export-graph-json" class="btn btn-primary">
<span class="btn-icon">[JSON]</span>
<span>Export Graph Data</span>
</button>
<div class="status-row" style="margin-top: 0.5rem;">
<span class="status-label">Complete graph data with forensic audit trail,
provider statistics, and scan metadata in JSON format for analysis and
archival.</span>
</div>
</div>
</div>
</details>
</section>
</div> </div>
</div> </div>
</div> </div>
</div> </div>
</div> </div>
<script>
function copyToClipboard(elementId) {
const element = document.getElementById(elementId);
const textToCopy = element.innerText;
navigator.clipboard.writeText(textToCopy).then(() => {
// Optional: Show a success message
console.log('Copied to clipboard');
}).catch(err => {
console.error('Failed to copy: ', err);
});
}
</script>
<script src="{{ url_for('static', filename='js/graph.js') }}"></script> <script src="{{ url_for('static', filename='js/graph.js') }}"></script>
<script src="{{ url_for('static', filename='js/main.js') }}"></script> <script src="{{ url_for('static', filename='js/main.js') }}"></script>
</body> </body>
</html> </html>

58
utils/helpers.py
View File

@ -1,3 +1,8 @@
# dnsrecon-reduced/utils/helpers.py
import ipaddress
from typing import Union
def _is_valid_domain(domain: str) -> bool: def _is_valid_domain(domain: str) -> bool:
""" """
Basic domain validation. Basic domain validation.
@ -26,32 +31,27 @@ def _is_valid_domain(domain: str) -> bool:
def _is_valid_ip(ip: str) -> bool: def _is_valid_ip(ip: str) -> bool:
""" """
Basic IP address validation. IP address validation supporting both IPv4 and IPv6.
Args: Args:
ip: IP address string to validate ip: IP address string to validate
Returns: Returns:
True if IP appears valid True if IP appears valid (IPv4 or IPv6)
""" """
if not ip:
return False
try: try:
parts = ip.split('.') # This handles both IPv4 and IPv6 validation
if len(parts) != 4: ipaddress.ip_address(ip.strip())
return False
for part in parts:
num = int(part)
if not 0 <= num <= 255:
return False
return True return True
except (ValueError, AttributeError): except (ValueError, AttributeError):
return False return False
def is_valid_target(target: str) -> bool: def is_valid_target(target: str) -> bool:
""" """
Checks if the target is a valid domain or IP address. Checks if the target is a valid domain or IP address (IPv4/IPv6).
Args: Args:
target: The target string to validate. target: The target string to validate.
@ -60,3 +60,35 @@ def is_valid_target(target: str) -> bool:
True if the target is a valid domain or IP, False otherwise. True if the target is a valid domain or IP, False otherwise.
""" """
return _is_valid_domain(target) or _is_valid_ip(target) return _is_valid_domain(target) or _is_valid_ip(target)
def get_ip_version(ip: str) -> Union[int, None]:
"""
Get the IP version (4 or 6) of a valid IP address.
Args:
ip: IP address string
Returns:
4 for IPv4, 6 for IPv6, None if invalid
"""
try:
addr = ipaddress.ip_address(ip.strip())
return addr.version
except (ValueError, AttributeError):
return None
def normalize_ip(ip: str) -> Union[str, None]:
"""
Normalize an IP address to its canonical form.
Args:
ip: IP address string
Returns:
Normalized IP address string, None if invalid
"""
try:
addr = ipaddress.ip_address(ip.strip())
return str(addr)
except (ValueError, AttributeError):
return None