This commit is contained in:
overcuriousity 2025-05-10 21:07:26 +02:00
parent 7dd2cca9a6
commit afa0454f74

View File

@ -1,302 +1,259 @@
#!/bin/bash
# Forensic Website Collection Script
# This script creates a forensically sound copy of a website
# with all resources, proper documentation, and integrity verification
# Forensic Collector
# Version 0.1
# Chain of custody and integrity verification
# Set consistent locale for date processing and UTC timezone
export LC_ALL=C
export TZ=UTC
#Copyright [yyyy] [name of copyright owner]
#Licensed under the Apache License, Version 2.0 (the "License");
#you may not use this file except in compliance with the License.
#You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
#Unless required by applicable law or agreed to in writing, software
#distributed under the License is distributed on an "AS IS" BASIS,
#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#See the License for the specific language governing permissions and
#limitations under the License.
# Color codes for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m' # No Color
# Function to print colored messages
print_message() {
local color=$1
local message=$2
echo -e "${color}${message}${NC}"
}
# header
echo -e "${GREEN}=========================================="
echo " FORENSIC COLLECTOR"
echo
echo
echo
echo "\"Experten, die keine KI verwenden, werden aussterben."
echo " Ja, und Experten wie meine Frau, die KI verwenden,"
echo " werden die anderen Experten ersetzen.\""
echo -e "${YELLOW} - Dirk Labudde, 29.10.2024${NC}"
echo -e "==========================================${NC}"
echo
# Function to check if required tools are installed
check_dependencies() {
local deps=("wget" "tar" "sha256sum" "date")
local missing=()
START_TIME=$(date -u +%Y-%m-%dT%H:%M:%SZ)
HOSTNAME=$(hostname)
OS_INFO=$(uname -a)
WGET_VERSION=$(wget --version | head -n1)
USERNAME=$(whoami)
for dep in "${deps[@]}"; do
if ! command -v "$dep" &> /dev/null; then
missing+=("$dep")
fi
done
echo -e "${YELLOW}Öffentliche IP wird abgerufen...${NC}"
EXTERNAL_IP=$(curl -s https://api.ipify.org)
if [ -z "$EXTERNAL_IP" ]; then
EXTERNAL_IP="Fehler beim Abruf der öffentlichen IP-Adresse."
echo -e "${RED}WARNUNG: Die öffentliche IP-Adresse konnte nicht abgerufen werden.${NC}"
echo -e "&{RED}Prüfen Sie die Netzwerkverbindung.${NC}"
else
echo -e "${GREEN}Öffentliche IP: $EXTERNAL_IP${NC}"
echo -e "${RED}Hinweis:${NC}"
echo -e "${YELLOW}Diese IP-Adresse wird bei der Sicherung der Webseite im Webserver-${NC}"
echo -e "${YELLOW}oder Firewall-Log des Ziels auffallen.${NC}"
echo -e "${NC}Die Nutzung von VPN oder TOR ist zu empfehlen.${NC}"
fi
if [ ${#missing[@]} -gt 0 ]; then
print_message "$RED" "Error: Missing dependencies: ${missing[*]}"
print_message "$YELLOW" "Please install the missing tools and try again."
echo -e "\n${YELLOW}Eingabe der Ziel-URI:${NC}"
read -p "URI: " TARGET_URL
# Validation
if [[ ! $TARGET_URL =~ ^https?:// ]]; then
echo -e "${RED}FEHLER: Geben Sie eine valide URI ein, beginnend mit http:// oder https://${NC}"
exit 1
fi
}
fi
# Function to validate URL
validate_url() {
local url=$1
if [[ ! $url =~ ^https?:// ]]; then
print_message "$RED" "Error: URL must start with http:// or https://"
return 1
fi
return 0
}
echo -e "\n${YELLOW}Angabe des Geschäftszeichens (optional):${NC}"
read -p "Geschäftszeichen: " CASE_NUMBER
# Function to create working directory
create_working_directory() {
local timestamp=$(date -u +%Y%m%d_%H%M%S_UTC)
local dir_name="forensic_collection_${timestamp}"
echo -e "\n${YELLOW}MAximale Rekursion eingeben (Enter=default=unlimitiert):${NC}"
read -p "Rekursion (default: unlimitiert): " MAX_DEPTH
mkdir -p "$dir_name"
cd "$dir_name" || exit 1
if [ -z "$MAX_DEPTH" ]; then
MAX_DEPTH="unlimitiert"
DEPTH_PARAM=""
else
DEPTH_PARAM="--level=$MAX_DEPTH"
fi
print_message "$GREEN" "Created working directory: $dir_name"
echo "$PWD"
}
HOSTNAME_FROM_URL=$(echo "$TARGET_URL" | sed -E 's/^https?:\/\///' | sed 's/\/.*$//' | sed 's/:.*$//')
# Function to create metadata file
create_metadata() {
local url=$1
local collector_name=$2
local collector_email=$3
local case_number=$4
local timestamp=$(date -u '+%Y-%m-%d %H:%M:%S UTC')
DATE_STR=$(date +%Y%m%d_%H%M%S)
if [ -z "$CASE_NUMBER" ]; then
OUTPUT_DIR="${DATE_STR}_${HOSTNAME_FROM_URL}"
else
OUTPUT_DIR="${DATE_STR}_${CASE_NUMBER}_${HOSTNAME_FROM_URL}"
fi
cat > collection_metadata.txt << EOF
=== FORENSIC WEB COLLECTION METADATA ===
Collection Date/Time: $timestamp
Source URL: $url
Collector Name: $collector_name
Collector Email: $collector_email
Case/Reference Number: ${case_number:-N/A}
Collection Method: wget with WARC output
Tool Versions:
- wget: $(wget --version | head -n1)
- bash: $BASH_VERSION
- OS: $(uname -a)
mkdir -p "$OUTPUT_DIR"
cd "$OUTPUT_DIR" || exit 1
Collection Parameters:
- Recursive: Yes
- Page requisites: Yes (CSS, JS, images)
- Maximum file size per WARC: 1GB
- Rate limit: 200kb/s
- Random delays: 1-2 seconds
- User agent: Modified browser string
- SSL verification: Disabled (for self-signed certificates)
WEBSITE_DIR="website"
mkdir -p "$WEBSITE_DIR"
Documentation:
- WARC files: Contains full HTTP transactions
- Mirror directory: Local file system copy
- Checksums: SHA-256 for all collected files
- This metadata file: collection_metadata.txt
EOF
# Log file for wget output
WGET_LOG="wget.log"
print_message "$GREEN" "Created metadata file"
}
# Initialize report
REPORT_FILE="forensic_report.txt"
cat > "$REPORT_FILE" << EOF
FORENSIC COLLECTOR
===================================
# Function to perform the actual wget collection
perform_collection() {
local url=$1
local domain=$(echo "$url" | sed -E 's|^https?://||' | sed -E 's|/.*$||' | sed -E 's|:.*$||')
local timestamp=$(date -u +%Y%m%d_%H%M%S_UTC)
local warc_file="${domain}_${timestamp}"
CASE INFORMATION
-----------------
CASE NUMBER: ${CASE_NUMBER:-N/A}
PRESERVATION TARGET: $TARGET_URL
PRESERVATION TIMESTAMP (UTC): $START_TIME
OUTPUT FOLDER: $OUTPUT_DIR
print_message "$YELLOW" "Starting collection of: $url"
print_message "$YELLOW" "Domain: $domain"
print_message "$YELLOW" "WARC file: ${warc_file}.warc"
# Extract collector info for WARC headers
local collector_info=$(grep "Collector Name:\|Collector Email:" collection_metadata.txt | tr '\n' ' ')
wget \
--mirror \
--convert-links \
--adjust-extension \
--page-requisites \
--no-parent \
--recursive \
--level=0 \
--no-clobber \
--continue \
--timestamping \
--no-check-certificate \
--user-agent="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36" \
--wait=1 \
--random-wait \
--limit-rate=200k \
--header="Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8" \
--header="Accept-Language: en-US,en;q=0.5" \
--warc-file="$warc_file" \
--warc-header="operator: $collector_info" \
--warc-header="description: Forensic collection of $url" \
--warc-max-size=1G \
"$url"
if [ $? -eq 0 ]; then
print_message "$GREEN" "Website collection completed successfully"
echo "$warc_file.warc"
else
print_message "$RED" "Website collection failed"
return 1
fi
}
# Function to generate checksums
generate_checksums() {
print_message "$YELLOW" "Generating SHA-256 checksums for all collected files..."
# Find all files and generate checksums
find . -type f -exec sha256sum {} \; > all_files_checksums.sha256
# Generate checksum for the WARC file specifically
if ls *.warc &> /dev/null; then
sha256sum *.warc > warc_checksum.sha256
fi
# Generate checksum for the metadata file
sha256sum collection_metadata.txt > metadata_checksum.sha256
print_message "$GREEN" "Checksums generated successfully"
}
# Function to create final package
create_final_package() {
local timestamp=$(date -u +%Y%m%d_%H%M%S_UTC)
local package_name="forensic_web_evidence_${timestamp}.tar.gz"
print_message "$YELLOW" "Creating final evidence package..."
# Create the tar.gz package
tar czf "../$package_name" ./*
if [ $? -eq 0 ]; then
# Generate checksum for the final package
cd ..
sha256sum "$package_name" > "${package_name}.sha256"
print_message "$GREEN" "Evidence package created: $package_name"
print_message "$GREEN" "Package checksum: ${package_name}.sha256"
# Display package contents
print_message "$YELLOW" "\nPackage contents:"
tar tzf "$package_name" | head -20
if [ $(tar tzf "$package_name" | wc -l) -gt 20 ]; then
echo "... (truncated, showing first 20 items)"
fi
# Display final checksums
print_message "$YELLOW" "\nFinal integrity checksums:"
cat "${package_name}.sha256"
else
print_message "$RED" "Failed to create evidence package"
return 1
fi
}
# Function to create collection report
create_collection_report() {
local url=$1
local start_time=$2
local end_time=$(date -u '+%Y-%m-%d %H:%M:%S UTC')
# Calculate duration more safely, using Unix timestamps
local start_timestamp=$3
local end_timestamp=$(date +%s)
local duration=$((end_timestamp - start_timestamp))
cat > ../collection_report.txt << EOF
=== FORENSIC WEB COLLECTION REPORT ===
Collection Summary:
HOST SYSTEM INFORMATION
-------------------
Target URL: $url
Start Time: $start_time
End Time: $end_time
Duration: $duration seconds
USERNAME: $USERNAME
HOSTNAME: $HOSTNAME
OPERATING SYSTEM: $OS_INFO
wget-version: $WGET_VERSION
PUBLIC IP ADDRESS: $EXTERNAL_IP
Results:
--------
WARC Files: $(ls *.warc 2>/dev/null | wc -l)
Total Files Collected: $(find . -type f | wc -l)
Total Size: $(du -sh . | cut -f1)
Package Information:
--------------------
Final Package: $(ls ../forensic_web_evidence_*.tar.gz 2>/dev/null)
Package Size: $(du -sh ../forensic_web_evidence_*.tar.gz 2>/dev/null | cut -f1)
Verification:
-------------
All files have been hashed with SHA-256
Final package integrity verified
Chain of custody maintained
Collection performed by: $(grep "Collector Name:" collection_metadata.txt | cut -d: -f2-)
Case Reference: $(grep "Case/Reference Number:" collection_metadata.txt | cut -d: -f2-)
PARAMETERS
-----------------
MAX RECURSION: $MAX_DEPTH
EXTRACTED TARGET DOMAIN: $HOSTNAME_FROM_URL
EOF
print_message "$GREEN" "Collection report created"
}
WGET_CMD="wget --recursive --page-requisites --html-extension --convert-links \
--restrict-file-names=windows --domains=$HOSTNAME_FROM_URL \
--user-agent='Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36' \
--wait=1 --random-wait --timeout=30 --tries=3 \
--no-parent --timestamping --backup-converted \
$DEPTH_PARAM --directory-prefix='$WEBSITE_DIR' '$TARGET_URL' 2>&1 | tee '$WGET_LOG'"
# Main script execution
main() {
clear
print_message "$GREEN" "=== FORENSIC WEBSITE COLLECTION SCRIPT ==="
print_message "$GREEN" "========================================\n"
# wget-command documentation
echo "wget-method:" >> "$REPORT_FILE"
echo "----------------------------------------" >> "$REPORT_FILE"
echo "$WGET_CMD" >> "$REPORT_FILE"
echo "----------------------------------------" >> "$REPORT_FILE"
echo >> "$REPORT_FILE"
# Check dependencies
check_dependencies
echo -e "\n${YELLOW}Beginne Sicherungsmaßnahme...${NC}"
echo -e "${GREEN}Methode:${NC} $WGET_CMD"
echo
# Get user input
read -p "Enter the website URL to collect: " URL
validate_url "$URL" || exit 1
# Execute wget (eval is used to properly handle the command with variables)
eval "$WGET_CMD"
WGET_EXIT_CODE=$?
if [ $WGET_EXIT_CODE -ne 0 ]; then
echo -e "${RED}WARNUNG: wget mit Fehlerstatus (exit code: $WGET_EXIT_CODE). Prüfen Sie $WGET_LOG für Details.${NC}"
echo "ERROR NOTICE: wget execution produced runtime errors. Exit code: $WGET_EXIT_CODE" >> "$REPORT_FILE"
echo "wget.log contains runtime logs." >> "$REPORT_FILE"
fi
read -p "Enter your name: " COLLECTOR_NAME
read -p "Enter your email: " COLLECTOR_EMAIL
read -p "Enter case/reference number (optional): " CASE_NUMBER
echo -e "\n${YELLOW}Kalkulation der Sicherungsstatistiken...${NC}"
TOTAL_FILES=$(find "$WEBSITE_DIR" -type f | wc -l)
TOTAL_SIZE=$(du -sh "$WEBSITE_DIR" | cut -f1)
FILE_TYPES=$(find "$WEBSITE_DIR" -type f -name "*.*" | sed 's/.*\.//' | sort | uniq -c | sort -rn)
# Record start time (both human-readable and timestamp)
START_TIME=$(date -u '+%Y-%m-%d %H:%M:%S UTC')
START_TIMESTAMP=$(date +%s)
echo "Preservation Statistics:" >> "$REPORT_FILE"
echo "-------------------" >> "$REPORT_FILE"
echo "Total File Count: $TOTAL_FILES" >> "$REPORT_FILE"
echo "Total Filesize: $TOTAL_SIZE" >> "$REPORT_FILE"
echo "Filetype-Distribution:" >> "$REPORT_FILE"
echo "$FILE_TYPES" >> "$REPORT_FILE"
echo >> "$REPORT_FILE"
# Create working directory
WORK_DIR=$(create_working_directory)
if [ -s "$WGET_LOG" ]; then
echo "wget Error Log:" >> "$REPORT_FILE"
echo "----------------" >> "$REPORT_FILE"
cat "$WGET_LOG" >> "$REPORT_FILE"
echo >> "$REPORT_FILE"
fi
# Create metadata
create_metadata "$URL" "$COLLECTOR_NAME" "$COLLECTOR_EMAIL" "$CASE_NUMBER"
# Generate hash list
echo -e "\n${YELLOW}Generiere Hashwerte...${NC}"
HASH_FILE="file_hashes.sha256"
echo "File Hash List (SHA-256)" > "$HASH_FILE"
echo "========================" >> "$HASH_FILE"
echo "Generated on: $(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "$HASH_FILE"
echo >> "$HASH_FILE"
# Perform the collection
if ! perform_collection "$URL"; then
print_message "$RED" "Collection failed. Exiting."
exit 1
fi
find "$WEBSITE_DIR" -type f -print0 | while IFS= read -r -d '' file; do
sha256sum "$file" >> "$HASH_FILE"
done
# Generate checksums
generate_checksums
# Hash the report file itself
sha256sum "$REPORT_FILE" >> "$HASH_FILE"
sha256sum "$HASH_FILE" >> "$HASH_FILE"
# Create collection report
create_collection_report "$URL" "$START_TIME" "$START_TIMESTAMP"
echo "Hash Verification:" >> "$REPORT_FILE"
echo "------------------" >> "$REPORT_FILE"
echo "Hash Algorithm: SHA-256" >> "$REPORT_FILE"
echo "Hash File: $HASH_FILE" >> "$HASH_FILE"
echo "Report File Hash: $(sha256sum "$REPORT_FILE" | cut -d' ' -f1)" >> "$REPORT_FILE"
echo >> "$REPORT_FILE"
# Create final package
create_final_package
# Create final archive
echo -e "\n${YELLOW}Erstelle Archiv...${NC}"
ARCHIVE_NAME="../${OUTPUT_DIR}.tar.gz"
tar -czf "$ARCHIVE_NAME" .
# Final summary
print_message "$GREEN" "\n=== COLLECTION COMPLETE ==="
print_message "$GREEN" "Working directory: $WORK_DIR"
print_message "$GREEN" "Final package and checksums are one directory above"
print_message "$YELLOW" "\nNext steps:"
print_message "$YELLOW" "1. Verify the package checksum"
print_message "$YELLOW" "2. Store the package in secure evidence storage"
print_message "$YELLOW" "3. Document the storage location in your case management system"
print_message "$YELLOW" "4. Consider creating a backup copy"
}
ARCHIVE_HASH=$(sha256sum "$ARCHIVE_NAME" | cut -d' ' -f1)
# Run the main function
main
END_TIME=$(date -u +%Y-%m-%dT%H:%M:%SZ)
echo "Preservation Completion:" >> "$REPORT_FILE"
echo "------------------------" >> "$REPORT_FILE"
echo "End Time (UTC): $END_TIME" >> "$REPORT_FILE"
echo "Archive Name: ${OUTPUT_DIR}.tar.gz" >> "$REPORT_FILE"
echo "Archive Hash (SHA-256): $ARCHIVE_HASH" >> "$REPORT_FILE"
SUMMARY_FILE="../${OUTPUT_DIR}_summary.txt"
cat > "$SUMMARY_FILE" << EOF
FORENSISCHE SICHERUNG - ZUSAMMENFASSUNG
=============================
Archiv: ${OUTPUT_DIR}.tar.gz
Archivhash (SHA-256): $ARCHIVE_HASH
Geschäftszeichen: ${CASE_NUMBER:-N/A}
Ziel-URL: $TARGET_URL
Beginn der Sicherung: $START_TIME
Ende der Sicherung: $END_TIME
Anzahl der Dateien: $TOTAL_FILES
Gesamtvolumen: $TOTAL_SIZE
Das Archiv enthält:
1. Komplettsicherung im Ordner 'website/'
2. Dokumentation der technischen Sicherung (forensic_report.txt)
3. SHA-256-Hashwerte aller Dateien
4. wget-Log
Zur Verifizierung der Integrität: sha256sum ${OUTPUT_DIR}.tar.gz
Hashwert: $ARCHIVE_HASH
EOF
cd ..
rm -rf "$OUTPUT_DIR"
# Display completion information
echo -e "\n${GREEN}=========================================="
echo " SICHERUNG ABGESCHLOSSEN"
echo -e "==========================================${NC}"
echo
echo -e "${GREEN}Archiv erstellt:${NC} ${OUTPUT_DIR}.tar.gz"
echo -e "${GREEN}Archivhash:${NC} $ARCHIVE_HASH"
echo -e "${GREEN}Zusammenfassung:${NC} ${OUTPUT_DIR}_summary.txt"
echo
echo -e "${YELLOW}Zugriff auf die Webseitensicherung:${NC}"
echo "1. Extraktion des Archivs: tar -xzf ${OUTPUT_DIR}.tar.gz"
echo "2. Navigieren Sie zu: ${OUTPUT_DIR}/website/"
echo "3. Öffnen Sie index.html in einem Browser"
echo
echo -e "${RED}ACHTUNG!${NC}"
echo -e "${RED}Das Öffnen der Offlinesicherung schließt nicht aus, dass Javascript aus dem Internet nachgeladen wird!${NC}"
echo -e "${RED}Es wird dringend empfohlen, dies nur in einem Airgapped-System zu versuchen.${NC}"
echo
echo -e "${YELLOW}Verifizierung der Integrität:${NC}"
echo "sha256sum ${OUTPUT_DIR}.tar.gz"
echo "Erwartungswert: $ARCHIVE_HASH"
echo