373 lines
16 KiB
Python
373 lines
16 KiB
Python
import sys
|
|
import os
|
|
import logging
|
|
import shutil
|
|
import multiprocessing
|
|
import logline_leviathan.gui.versionvars as versionvars
|
|
from PyQt5.QtWidgets import QApplication, QWidget, QMessageBox, QLabel
|
|
from PyQt5.QtCore import QTimer
|
|
from logline_leviathan.file_processor.file_processor_thread import FileProcessorThread
|
|
from logline_leviathan.database.database_manager import EntityTypesTable, EntitiesTable, session_scope
|
|
from logline_leviathan.database.database_utility import DatabaseUtility
|
|
from logline_leviathan.database.database_operations import DatabaseOperations
|
|
from logline_leviathan.gui.checkbox_panel import *
|
|
from logline_leviathan.gui.initui_mainwindow import initialize_main_window
|
|
from logline_leviathan.gui.generate_report import GenerateReportWindow
|
|
from logline_leviathan.gui.generate_wordlist import GenerateWordlistWindow
|
|
from logline_leviathan.gui.ui_helper import UIHelper, format_time
|
|
from logline_leviathan.gui.settings_gui import FileSettingsWindow, AnalysisSettingsWindow
|
|
from logline_leviathan.gui.query_window import ResultsWindow
|
|
from logline_leviathan.database.query import DatabaseGUIQuery
|
|
from sqlalchemy import func
|
|
from datetime import datetime
|
|
|
|
|
|
|
|
class MainWindow(QWidget):
|
|
def __init__(self, app, db_init_func, directory=""):
|
|
super().__init__()
|
|
logging_level = getattr(logging, versionvars.loglevel, None)
|
|
if isinstance(logging_level, int):
|
|
logging.basicConfig(level=logging_level)
|
|
else:
|
|
logging.warning(f"Invalid log level: {versionvars.loglevel}")
|
|
|
|
self.app = app
|
|
self.ui_helper = UIHelper(self)
|
|
self.db_init_func = db_init_func
|
|
db_init_func()
|
|
self.database_operations = DatabaseOperations(self, db_init_func)
|
|
self.current_db_path = 'entities.db' # Default database path
|
|
self.directory = directory
|
|
self.filePaths = []
|
|
self.log_dir = os.path.join(os.getcwd(), 'output', 'entities_export', 'log')
|
|
os.makedirs(self.log_dir, exist_ok=True)
|
|
|
|
self.external_db_path = None
|
|
self.processing_thread = None
|
|
self.generate_report_window = None
|
|
self.databaseTree = DatabasePanel()
|
|
|
|
self.db_query_instance = DatabaseGUIQuery()
|
|
self.results_window = ResultsWindow(self.db_query_instance, parent=self)
|
|
self.generate_wordlist_window = GenerateWordlistWindow(self.db_query_instance)
|
|
self.generate_report_window = GenerateReportWindow(self.app)
|
|
self.analysis_settings_window = AnalysisSettingsWindow(self)
|
|
self.analysis_settings_window.parsersUpdated.connect(self.refreshApplicationState)
|
|
self.file_selection_window = FileSettingsWindow(self.filePaths, self)
|
|
|
|
self.database_operations.ensureDatabaseExists()
|
|
|
|
self.initUI()
|
|
|
|
self.ui_helper = UIHelper(self)
|
|
self.database_utility = DatabaseUtility(self)
|
|
|
|
yaml_data = self.database_operations.loadRegexFromYAML()
|
|
self.database_operations.populate_and_update_entities_from_yaml(yaml_data)
|
|
# Load data and update checkboxes
|
|
self.refreshApplicationState()
|
|
|
|
self.database_operations.checkScriptPresence()
|
|
|
|
# Load files from the directory if specified
|
|
if self.directory and os.path.isdir(self.directory):
|
|
self.loadFilesFromDirectory(self.directory)
|
|
|
|
self.ui_update_interval = 500
|
|
self.needs_tree_update = False
|
|
self.needs_checkbox_update = False
|
|
self.update_timer = QTimer(self)
|
|
self.update_timer.timeout.connect(self.performPeriodicUpdate)
|
|
self.update_timer.start(self.ui_update_interval)
|
|
|
|
|
|
def loadFilesFromDirectory(self, directory):
|
|
for root, dirs, files in os.walk(directory):
|
|
for filename in files:
|
|
file_path = os.path.join(root, filename)
|
|
self.filePaths.append(file_path)
|
|
self.updateFileCountLabel()
|
|
|
|
|
|
def initUI(self):
|
|
initialize_main_window(self, self.app)
|
|
|
|
def openFileNameDialog(self):
|
|
self.ui_helper.openFileNameDialog()
|
|
self.file_selection_window.populateTable()
|
|
self.updateFileCountLabel()
|
|
|
|
def openDirNameDialog(self):
|
|
self.ui_helper.openDirNameDialog()
|
|
self.file_selection_window.populateTable()
|
|
self.updateFileCountLabel()
|
|
|
|
def clearFileSelection(self):
|
|
self.ui_helper.clearFileSelection()
|
|
self.file_selection_window.populateTable()
|
|
self.updateFileCountLabel()
|
|
|
|
def removeSingleFile(self, file):
|
|
self.ui_helper.removeSingleFile(file)
|
|
self.file_selection_window.populateTable()
|
|
self.updateFileCountLabel()
|
|
|
|
def refreshApplicationState(self):
|
|
#yaml_data = self.database_operations.loadRegexFromYAML()
|
|
#self.database_operations.populate_and_update_entities_from_yaml(yaml_data)
|
|
self.processing_thread = FileProcessorThread(self.filePaths)
|
|
self.processing_thread.update_checkboxes_signal.connect(self.generate_report_window.updateCheckboxes)
|
|
self.processing_thread.update_checkboxes_signal.connect(self.generate_wordlist_window.updateCheckboxes)
|
|
self.generate_report_window.updateCheckboxes()
|
|
self.generate_wordlist_window.updateCheckboxes()
|
|
self.updateDatabaseStatusLabel()
|
|
self.updateTree()
|
|
self.updateFileCountLabel()
|
|
|
|
def updateFileCountLabel(self):
|
|
file_count = len(self.filePaths)
|
|
file_count_label = f" {file_count} Dateien selektiert"
|
|
readable_size = self.ui_helper.calculate_total_size(self.filePaths)
|
|
self.fileCountLabel.setText(file_count_label + f' // {readable_size}')
|
|
|
|
|
|
def updateTree(self):
|
|
with session_scope() as session:
|
|
self.databaseTree.updateTree(session)
|
|
|
|
def updateDatabaseStatusLabel(self):
|
|
with session_scope() as session:
|
|
entity_count = session.query(EntitiesTable).count()
|
|
|
|
db_file_path = self.current_db_path # Replace with your actual database file path
|
|
db_file_size = os.path.getsize(db_file_path)
|
|
db_file_size_mb = db_file_size / (1024 * 1024) # Convert size to MB
|
|
|
|
status_text = f"Anzahl Entitäten: {entity_count}\nDatenbank-Größe: {db_file_size_mb:.2f} MB"
|
|
self.databaseStatusLabel.setText(status_text)
|
|
|
|
def onTreeUpdateSignalReceived(self):
|
|
self.needs_tree_update = True
|
|
|
|
def onCheckboxUpdateSignalReceived(self):
|
|
self.needs_checkbox_update = True
|
|
|
|
def performPeriodicUpdate(self):
|
|
if self.needs_tree_update:
|
|
self.updateTree()
|
|
self.needs_tree_update = False
|
|
|
|
if self.needs_checkbox_update:
|
|
self.generate_report_window.updateCheckboxes()
|
|
self.generate_wordlist_window.updateCheckboxes()
|
|
self.updateTree()
|
|
self.needs_checkbox_update = False
|
|
|
|
def execute_query_wrapper(self, query_text):
|
|
self.results_window.show()
|
|
self.results_window.set_query_and_execute(query_text)
|
|
|
|
|
|
def quickStartWorkflow(self):
|
|
self.clearFileSelection()
|
|
self.purgeDatabase()
|
|
yaml_data = self.database_operations.loadRegexFromYAML()
|
|
self.database_operations.populate_and_update_entities_from_yaml(yaml_data)
|
|
self.openDirNameDialog()
|
|
self.processFiles()
|
|
|
|
def purgeDatabase(self):
|
|
self.database_utility.purgeDatabase()
|
|
|
|
def importDatabase(self):
|
|
self.database_utility.importDatabase()
|
|
|
|
def exportDatabase(self):
|
|
self.database_utility.exportDatabase()
|
|
|
|
|
|
def processFiles(self):
|
|
try:
|
|
fileCount = len(self.filePaths)
|
|
if fileCount > 0:
|
|
self.progressBar.setMaximum(fileCount)
|
|
self.db_init_func()
|
|
self.processing_thread = FileProcessorThread(self.filePaths) # Assign the thread to processing_thread
|
|
self.processing_thread.finished.connect(self.onProcessingFinished)
|
|
self.processing_thread.update_progress.connect(self.progressBar.setValue)
|
|
self.processing_thread.update_status.connect(self.statusLabel.setText)
|
|
self.processing_thread.update_rate.connect(self.updateEntityRate)
|
|
#self.processing_thread.update_tree_signal.connect(self.updateTree)
|
|
#self.processing_thread.update_checkboxes_signal.connect(self.generate_report_window.updateCheckboxes)
|
|
#self.processing_thread.update_checkboxes_signal.connect(self.generate_wordlist_window.updateCheckboxes)
|
|
self.processing_thread.update_tree_signal.connect(self.onTreeUpdateSignalReceived)
|
|
self.processing_thread.update_checkboxes_signal.connect(self.onCheckboxUpdateSignalReceived)
|
|
self.processing_thread.start()
|
|
logging.debug(f"Thread started, isRunning: {self.processing_thread.isRunning()}")
|
|
else:
|
|
self.message("Information", "Keine Dateien Selektiert. Selektion vornehmen.")
|
|
except Exception as e:
|
|
logging.error(f"Error processing files: {e}")
|
|
|
|
def abortAnalysis(self):
|
|
if self.processing_thread and self.isProcessing():
|
|
logging.debug(f"Abort Analysis initiated.")
|
|
self.processing_thread.abort()
|
|
self.processing_thread.wait()
|
|
#self.processing_thread = None
|
|
self.statusLabel.setText(" Verarbeitung durch User unterbrochen.")
|
|
logging.info(f"Analysis aborted manually.")
|
|
self.refreshApplicationState()
|
|
|
|
def isProcessing(self):
|
|
if self.processing_thread is not None:
|
|
return self.processing_thread.isRunning()
|
|
return False
|
|
|
|
|
|
def onProcessingFinished(self):
|
|
if self.processing_thread:
|
|
summary = self.getProcessingSummary()
|
|
unsupported_files_count = self.processing_thread.getUnsupportedFilesCount()
|
|
|
|
# Generate CSV files for unprocessed and processed files
|
|
current_timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
|
unprocessed_files_log = os.path.join(self.log_dir, f"{current_timestamp}_unprocessed_files_log.csv")
|
|
processed_files_log = os.path.join(self.log_dir, f"{current_timestamp}_processed_files_log.csv")
|
|
|
|
self.ui_helper.generate_files_log(unprocessed_files_log, self.processing_thread.all_unsupported_files)
|
|
processed_files = set(self.processing_thread.file_paths) - set(self.processing_thread.all_unsupported_files)
|
|
self.ui_helper.generate_files_log(processed_files_log, list(processed_files))
|
|
|
|
if unsupported_files_count > 0:
|
|
summary += f"\n{unsupported_files_count} nicht unterstützte Dateien übersprungen."
|
|
link_label = QLabel(f'<a href="#">Open list of all unsupported files...</a>')
|
|
link_label.linkActivated.connect(lambda: self.ui_helper.openFile(unprocessed_files_log))
|
|
self.message("Analyse-Zusammenfassung", summary, link_label)
|
|
else:
|
|
self.message("Analyse-Zusammenfassung", summary)
|
|
|
|
if self.external_db_path:
|
|
try:
|
|
shutil.copy('entities.db', self.external_db_path)
|
|
self.statusLabel.setText(f" Datenbank gespeichert unter: {self.external_db_path}")
|
|
except Exception as e:
|
|
logging.error(f"Error exporting database: {e}")
|
|
self.statusLabel.setText(f" Fehler beim Exportieren der Datenbank: {e}")
|
|
|
|
self.refreshApplicationState()
|
|
self.processing_thread = None
|
|
|
|
def openLogDir(self):
|
|
self.ui_helper.openFile(self.log_dir)
|
|
|
|
def getProcessingSummary(self):
|
|
with session_scope() as session:
|
|
entity_counts = session.query(EntityTypesTable.gui_name, func.count(EntitiesTable.entities_id)) \
|
|
.join(EntityTypesTable, EntitiesTable.entity_types_id == EntityTypesTable.entity_type_id) \
|
|
.group_by(EntityTypesTable.gui_name) \
|
|
.all()
|
|
|
|
summary = "Analyse-Zusammenfassung:\n\n"
|
|
for gui_name, count in entity_counts:
|
|
summary += f"{gui_name}: {count} gefunden\n"
|
|
|
|
return summary
|
|
|
|
def getUnsupportedFilesCount(self):
|
|
if self.processing_thread:
|
|
return self.processing_thread.getUnsupportedFilesCount()
|
|
return 0
|
|
|
|
|
|
|
|
def showProcessingWarning(self):
|
|
self.message("Operation unmöglich", "Diese Operation kann nicht durchgeführt werden, während Dateien analysiert werden. Warten oder abbrechen.")
|
|
|
|
def updateEntityRate(self, entity_rate, total_entities, file_rate, total_files_processed, estimated_time, data_rate_kibs):
|
|
formatted_time = format_time(estimated_time)
|
|
total_cpu_cores = multiprocessing.cpu_count()
|
|
rate_text = (f"{entity_rate:.2f} entities/second, Total: {total_entities} // "
|
|
f"{file_rate:.2f} files/second, Total: {total_files_processed} // "
|
|
f"{data_rate_kibs:.2f} KiB/s // "
|
|
f"ETC: {formatted_time} // "
|
|
f"CPU Cores: {total_cpu_cores}")
|
|
self.entityRateLabel.setText(rate_text)
|
|
|
|
|
|
def openGenerateReportWindow(self):
|
|
if self.isProcessing():
|
|
self.showProcessingWarning()
|
|
return
|
|
if not self.generate_report_window:
|
|
self.generate_report_window = GenerateReportWindow(self.app)
|
|
self.generate_report_window.show()
|
|
|
|
def openGenerateWordlistWindow(self):
|
|
if self.isProcessing():
|
|
self.showProcessingWarning()
|
|
return
|
|
if not self.generate_wordlist_window:
|
|
self.generate_wordlist_window = GenerateWordlistWindow(self.app)
|
|
self.generate_wordlist_window.show()
|
|
|
|
def openFileSettingsWindow(self):
|
|
if self.isProcessing():
|
|
self.showProcessingWarning()
|
|
return
|
|
if not self.file_selection_window:
|
|
self.file_selection_window = FileSettingsWindow(self.filePaths, self)
|
|
self.file_selection_window.show()
|
|
|
|
def openAnalysisSettingsWindow(self):
|
|
if self.isProcessing():
|
|
self.showProcessingWarning()
|
|
return
|
|
if not self.analysis_settings_window: # Use self.analysis_settings_window
|
|
self.analysis_settings_window = AnalysisSettingsWindow(self) # Use self.analysis_settings_window
|
|
self.analysis_settings_window.show() # Use self.analysis_settings_window
|
|
|
|
|
|
|
|
def message(self, title, text, extra_widget=None):
|
|
msgBox = QMessageBox()
|
|
msgBox.setStyleSheet("""
|
|
QMessageBox {
|
|
background-color: #282C34; /* Dark grey background */
|
|
}
|
|
QLabel {
|
|
color: white; /* White text */
|
|
}
|
|
QPushButton {
|
|
color: white; /* White text for buttons */
|
|
background-color: #4B5563; /* Dark grey background for buttons */
|
|
border-style: solid;
|
|
border-width: 2px;
|
|
border-radius: 5px;
|
|
border-color: #4A4A4A;
|
|
padding: 6px;
|
|
min-width: 80px;
|
|
min-height: 30px;
|
|
}
|
|
""")
|
|
msgBox.setIcon(QMessageBox.Warning)
|
|
msgBox.setWindowTitle(title)
|
|
msgBox.setText(text)
|
|
if extra_widget:
|
|
msgBox.setInformativeText('')
|
|
msgBox.layout().addWidget(extra_widget, 1, 1)
|
|
msgBox.exec_()
|
|
|
|
|
|
|
|
def main():
|
|
app = QApplication(sys.argv)
|
|
ex = MainWindow()
|
|
ex.show()
|
|
sys.exit(app.exec_())
|
|
|
|
if __name__ == '__main__':
|
|
main()
|
|
|