393 lines
17 KiB
Python
393 lines
17 KiB
Python
# File: src/web_app.py
|
|
"""Flask web application for forensic reconnaissance tool."""
|
|
|
|
from flask import Flask, render_template, request, jsonify, send_from_directory
|
|
import threading
|
|
import time
|
|
import logging
|
|
from .config import Config
|
|
from .reconnaissance import ForensicReconnaissanceEngine
|
|
from .report_generator import ForensicReportGenerator
|
|
from .data_structures import ForensicReconData
|
|
|
|
# Set up logging for this module
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# Global variables for tracking ongoing scans
|
|
active_scans = {}
|
|
scan_lock = threading.Lock()
|
|
|
|
def create_app(config: Config):
|
|
"""Create Flask application."""
|
|
app = Flask(__name__,
|
|
template_folder='../templates',
|
|
static_folder='../static')
|
|
|
|
app.config['SECRET_KEY'] = 'forensic-recon-tool-secret-key'
|
|
|
|
# Set up logging for web app
|
|
config.setup_logging(cli_mode=False)
|
|
logger.info("🌐 Forensic web application initialized")
|
|
|
|
@app.route('/')
|
|
def index():
|
|
"""Main page."""
|
|
return render_template('index.html')
|
|
|
|
@app.route('/api/scan', methods=['POST'])
|
|
def start_scan():
|
|
"""Start a new forensic reconnaissance scan."""
|
|
try:
|
|
data = request.get_json()
|
|
target = data.get('target')
|
|
scan_config = Config.from_args(
|
|
shodan_key=data.get('shodan_key'),
|
|
virustotal_key=data.get('virustotal_key'),
|
|
max_depth=data.get('max_depth', 2)
|
|
)
|
|
|
|
if not target:
|
|
logger.warning("⚠️ Scan request missing target")
|
|
return jsonify({'error': 'Target is required'}), 400
|
|
|
|
# Generate scan ID
|
|
scan_id = f"{target}_{int(time.time())}"
|
|
logger.info(f"🚀 Starting new forensic scan: {scan_id} for target: {target}")
|
|
|
|
# Create shared ForensicReconData object for live updates
|
|
shared_data = ForensicReconData()
|
|
shared_data.scan_config = {
|
|
'target': target,
|
|
'max_depth': scan_config.max_depth,
|
|
'shodan_enabled': scan_config.shodan_key is not None,
|
|
'virustotal_enabled': scan_config.virustotal_key is not None
|
|
}
|
|
|
|
# Initialize scan data with the shared forensic data object
|
|
with scan_lock:
|
|
active_scans[scan_id] = {
|
|
'status': 'starting',
|
|
'progress': 0,
|
|
'message': 'Initializing forensic scan...',
|
|
'data': shared_data, # Share the forensic data object from the start
|
|
'error': None,
|
|
'live_stats': shared_data.get_stats(), # Use forensic stats
|
|
'latest_discoveries': []
|
|
}
|
|
|
|
# Start forensic reconnaissance in background thread
|
|
thread = threading.Thread(
|
|
target=run_forensic_reconnaissance_background,
|
|
args=(scan_id, target, scan_config, shared_data)
|
|
)
|
|
thread.daemon = True
|
|
thread.start()
|
|
|
|
return jsonify({'scan_id': scan_id})
|
|
|
|
except Exception as e:
|
|
logger.error(f"❌ Error starting scan: {e}", exc_info=True)
|
|
return jsonify({'error': str(e)}), 500
|
|
|
|
@app.route('/api/scan/<scan_id>/status')
|
|
def get_scan_status(scan_id):
|
|
"""Get scan status and progress with live forensic discoveries."""
|
|
with scan_lock:
|
|
if scan_id not in active_scans:
|
|
return jsonify({'error': 'Scan not found'}), 404
|
|
|
|
scan_data = active_scans[scan_id].copy()
|
|
|
|
# Update live stats from forensic data if available
|
|
if scan_data.get('data') and hasattr(scan_data['data'], 'get_stats'):
|
|
scan_data['live_stats'] = scan_data['data'].get_stats()
|
|
|
|
# Don't include the full forensic data object in status (too large)
|
|
if 'data' in scan_data:
|
|
del scan_data['data']
|
|
|
|
return jsonify(scan_data)
|
|
|
|
@app.route('/api/scan/<scan_id>/report')
|
|
def get_scan_report(scan_id):
|
|
"""Get forensic scan report."""
|
|
with scan_lock:
|
|
if scan_id not in active_scans:
|
|
return jsonify({'error': 'Scan not found'}), 404
|
|
|
|
scan_data = active_scans[scan_id]
|
|
|
|
if scan_data['status'] != 'completed' or not scan_data['data']:
|
|
return jsonify({'error': 'Scan not completed'}), 400
|
|
|
|
try:
|
|
# Generate forensic report
|
|
report_gen = ForensicReportGenerator(scan_data['data'])
|
|
|
|
return jsonify({
|
|
'json_report': scan_data['data'].to_json(),
|
|
'text_report': report_gen.generate_text_report()
|
|
})
|
|
except Exception as e:
|
|
logger.error(f"❌ Error generating report for {scan_id}: {e}", exc_info=True)
|
|
return jsonify({'error': f'Failed to generate report: {str(e)}'}), 500
|
|
|
|
|
|
@app.route('/api/scan/<scan_id>/graph')
|
|
def get_scan_graph(scan_id):
|
|
"""Get graph data for visualization."""
|
|
with scan_lock:
|
|
if scan_id not in active_scans:
|
|
return jsonify({'error': 'Scan not found'}), 404
|
|
|
|
scan_data = active_scans[scan_id]
|
|
|
|
if scan_data['status'] != 'completed' or not scan_data['data']:
|
|
return jsonify({'error': 'Scan not completed'}), 400
|
|
|
|
try:
|
|
forensic_data = scan_data['data']
|
|
|
|
# Extract nodes for graph
|
|
nodes = []
|
|
for hostname, node in forensic_data.nodes.items():
|
|
# Determine node color based on depth
|
|
color_map = {
|
|
0: '#00ff41', # Green for root
|
|
1: '#ff9900', # Orange for depth 1
|
|
2: '#ff6b6b', # Red for depth 2
|
|
3: '#4ecdc4', # Teal for depth 3
|
|
4: '#45b7d1', # Blue for depth 4+
|
|
}
|
|
color = color_map.get(node.depth, '#666666')
|
|
|
|
# Calculate node size based on number of connections and data
|
|
connections = len(forensic_data.get_children(hostname)) + len(forensic_data.get_parents(hostname))
|
|
dns_records = len(node.get_all_dns_records())
|
|
certificates = len(node.certificates)
|
|
|
|
# Size based on importance (connections + data)
|
|
size = max(8, min(20, 8 + connections * 2 + dns_records // 3 + certificates))
|
|
|
|
nodes.append({
|
|
'id': hostname,
|
|
'label': hostname,
|
|
'depth': node.depth,
|
|
'color': color,
|
|
'size': size,
|
|
'dns_records': dns_records,
|
|
'certificates': certificates,
|
|
'ip_addresses': list(node.resolved_ips),
|
|
'discovery_methods': [method.value for method in node.discovery_methods],
|
|
'first_seen': node.first_seen.isoformat() if node.first_seen else None
|
|
})
|
|
|
|
# Extract edges for graph
|
|
edges = []
|
|
for edge in forensic_data.edges:
|
|
# Skip synthetic TLD expansion edges for cleaner visualization
|
|
if edge.source_hostname.startswith('tld_expansion:'):
|
|
continue
|
|
|
|
# Color edges by discovery method
|
|
method_colors = {
|
|
'initial_target': '#00ff41',
|
|
'tld_expansion': '#ff9900',
|
|
'dns_record_value': '#4ecdc4',
|
|
'certificate_subject': '#ff6b6b',
|
|
'dns_subdomain_extraction': '#45b7d1'
|
|
}
|
|
|
|
edges.append({
|
|
'source': edge.source_hostname,
|
|
'target': edge.target_hostname,
|
|
'method': edge.discovery_method.value,
|
|
'color': method_colors.get(edge.discovery_method.value, '#666666'),
|
|
'operation_id': edge.operation_id,
|
|
'timestamp': edge.timestamp.isoformat() if edge.timestamp else None
|
|
})
|
|
|
|
# Graph statistics
|
|
stats = {
|
|
'node_count': len(nodes),
|
|
'edge_count': len(edges),
|
|
'max_depth': max([node['depth'] for node in nodes]) if nodes else 0,
|
|
'discovery_methods': list(set([edge['method'] for edge in edges])),
|
|
'root_nodes': [node['id'] for node in nodes if node['depth'] == 0]
|
|
}
|
|
|
|
return jsonify({
|
|
'nodes': nodes,
|
|
'edges': edges,
|
|
'stats': stats
|
|
})
|
|
|
|
except Exception as e:
|
|
logger.error(f"⚠️ Error generating graph data for {scan_id}: {e}", exc_info=True)
|
|
return jsonify({'error': f'Failed to generate graph: {str(e)}'}), 500
|
|
|
|
@app.route('/api/scan/<scan_id>/live-data')
|
|
def get_live_scan_data(scan_id):
|
|
"""Get current forensic reconnaissance data (for real-time updates)."""
|
|
with scan_lock:
|
|
if scan_id not in active_scans:
|
|
return jsonify({'error': 'Scan not found'}), 404
|
|
|
|
scan_data = active_scans[scan_id]
|
|
forensic_data = scan_data['data']
|
|
|
|
if not forensic_data:
|
|
return jsonify({
|
|
'hostnames': [],
|
|
'ip_addresses': [],
|
|
'stats': {
|
|
'hostnames': 0,
|
|
'ip_addresses': 0,
|
|
'discovery_edges': 0,
|
|
'operations_performed': 0,
|
|
'dns_records': 0,
|
|
'certificates_total': 0,
|
|
'certificates_current': 0,
|
|
'certificates_expired': 0,
|
|
'shodan_results': 0,
|
|
'virustotal_results': 0
|
|
},
|
|
'latest_discoveries': []
|
|
})
|
|
|
|
# Extract data from forensic structure for frontend
|
|
try:
|
|
hostnames = sorted(list(forensic_data.nodes.keys()))
|
|
ip_addresses = sorted(list(forensic_data.ip_addresses))
|
|
stats = forensic_data.get_stats()
|
|
|
|
# Generate activity log from recent operations
|
|
latest_discoveries = []
|
|
recent_operations = forensic_data.operation_timeline[-10:] # Last 10 operations
|
|
|
|
for op_id in recent_operations:
|
|
if op_id in forensic_data.operations:
|
|
operation = forensic_data.operations[op_id]
|
|
activity_entry = {
|
|
'timestamp': operation.timestamp.timestamp(),
|
|
'message': f"{operation.operation_type.value}: {operation.target}"
|
|
}
|
|
|
|
# Add result summary
|
|
if operation.discovered_hostnames:
|
|
activity_entry['message'] += f" → {len(operation.discovered_hostnames)} hostnames"
|
|
if operation.discovered_ips:
|
|
activity_entry['message'] += f" → {len(operation.discovered_ips)} IPs"
|
|
|
|
latest_discoveries.append(activity_entry)
|
|
|
|
# Update scan data with latest discoveries
|
|
scan_data['latest_discoveries'] = latest_discoveries
|
|
|
|
return jsonify({
|
|
'hostnames': hostnames,
|
|
'ip_addresses': ip_addresses,
|
|
'stats': stats,
|
|
'latest_discoveries': latest_discoveries
|
|
})
|
|
|
|
except Exception as e:
|
|
logger.error(f"❌ Error extracting live data for {scan_id}: {e}", exc_info=True)
|
|
# Return minimal data structure
|
|
return jsonify({
|
|
'hostnames': [],
|
|
'ip_addresses': [],
|
|
'stats': {
|
|
'hostnames': len(forensic_data.nodes) if forensic_data.nodes else 0,
|
|
'ip_addresses': len(forensic_data.ip_addresses) if forensic_data.ip_addresses else 0,
|
|
'discovery_edges': 0,
|
|
'operations_performed': 0,
|
|
'dns_records': 0,
|
|
'certificates_total': 0,
|
|
'certificates_current': 0,
|
|
'certificates_expired': 0,
|
|
'shodan_results': 0,
|
|
'virustotal_results': 0
|
|
},
|
|
'latest_discoveries': []
|
|
})
|
|
|
|
return app
|
|
|
|
def run_forensic_reconnaissance_background(scan_id: str, target: str, config: Config, shared_data: ForensicReconData):
|
|
"""Run forensic reconnaissance in background thread with shared forensic data object."""
|
|
|
|
def update_progress(message: str, percentage: int = None):
|
|
"""Update scan progress and live forensic statistics."""
|
|
with scan_lock:
|
|
if scan_id in active_scans:
|
|
active_scans[scan_id]['message'] = message
|
|
if percentage is not None:
|
|
active_scans[scan_id]['progress'] = percentage
|
|
|
|
# Update live stats from the shared forensic data object
|
|
if shared_data:
|
|
active_scans[scan_id]['live_stats'] = shared_data.get_stats()
|
|
|
|
# Add to latest discoveries (keep last 10)
|
|
if 'latest_discoveries' not in active_scans[scan_id]:
|
|
active_scans[scan_id]['latest_discoveries'] = []
|
|
|
|
# Create activity entry
|
|
activity_entry = {
|
|
'timestamp': time.time(),
|
|
'message': message
|
|
}
|
|
|
|
active_scans[scan_id]['latest_discoveries'].append(activity_entry)
|
|
|
|
# Keep only last 10 discoveries
|
|
active_scans[scan_id]['latest_discoveries'] = \
|
|
active_scans[scan_id]['latest_discoveries'][-10:]
|
|
|
|
logger.info(f"[{scan_id}] {message} ({percentage}%)" if percentage else f"[{scan_id}] {message}")
|
|
|
|
try:
|
|
logger.info(f"🔧 Initializing forensic reconnaissance engine for scan: {scan_id}")
|
|
|
|
# Initialize forensic engine
|
|
engine = ForensicReconnaissanceEngine(config)
|
|
engine.set_progress_callback(update_progress)
|
|
|
|
# IMPORTANT: Pass the shared forensic data object to the engine
|
|
engine.set_shared_data(shared_data)
|
|
|
|
# Update status
|
|
with scan_lock:
|
|
active_scans[scan_id]['status'] = 'running'
|
|
|
|
logger.info(f"🚀 Starting forensic reconnaissance for: {target}")
|
|
|
|
# Run forensic reconnaissance - this will populate the shared_data object incrementally
|
|
final_data = engine.run_reconnaissance(target)
|
|
|
|
logger.info(f"✅ Forensic reconnaissance completed for scan: {scan_id}")
|
|
|
|
# Update with final results (the shared_data should already be populated)
|
|
with scan_lock:
|
|
active_scans[scan_id]['status'] = 'completed'
|
|
active_scans[scan_id]['progress'] = 100
|
|
active_scans[scan_id]['message'] = 'Forensic reconnaissance completed'
|
|
active_scans[scan_id]['data'] = final_data # This should be the same as shared_data
|
|
active_scans[scan_id]['live_stats'] = final_data.get_stats()
|
|
|
|
# Log final forensic statistics
|
|
final_stats = final_data.get_stats()
|
|
logger.info(f"📊 Final forensic stats for {scan_id}: {final_stats}")
|
|
|
|
# Log discovery graph analysis
|
|
graph_analysis = final_data._generate_graph_analysis()
|
|
logger.info(f"🌐 Discovery graph: {len(final_data.nodes)} nodes, {len(final_data.edges)} edges, max depth: {graph_analysis['max_depth']}")
|
|
|
|
except Exception as e:
|
|
logger.error(f"❌ Error in forensic reconnaissance for {scan_id}: {e}", exc_info=True)
|
|
# Handle errors
|
|
with scan_lock:
|
|
active_scans[scan_id]['status'] = 'error'
|
|
active_scans[scan_id]['error'] = str(e)
|
|
active_scans[scan_id]['message'] = f'Error: {str(e)}' |