From c86d707d89d5fd2ec7bb157f00647989bb81c161 Mon Sep 17 00:00:00 2001 From: alexjacks92 Date: Wed, 2 Apr 2014 16:17:40 -0400 Subject: [PATCH 01/15] Added code to move diffs to a folder specified in a config file. Will be used by Jenkins to locate email attachments. --- test/script/regression.py | 45 ++++++++++++++++++++++++++++++++------- 1 file changed, 37 insertions(+), 8 deletions(-) diff --git a/test/script/regression.py b/test/script/regression.py index 5e8025fb5d..ab78332386 100755 --- a/test/script/regression.py +++ b/test/script/regression.py @@ -43,7 +43,8 @@ import zlib import Emailer import srcupdater from regression_utils import * - +import shutil +import ntpath # # Please read me... # @@ -152,8 +153,9 @@ class TestRunner(object): if (len(logres)>0): for lm in logres: for ln in lm: - Errors.add_email_msg(ln) - + #EMAIL HERE + Errors.add_email_msg(ln) +#LOTS OF EMAIL HERE # TODO: possibly worth putting this in a sub method if all([ test_data.overall_passed for test_data in test_data_list ]): Errors.add_email_msg("All images passed.\n") @@ -166,8 +168,9 @@ class TestRunner(object): html = open(test_config.html_log) Errors.add_email_attachment(html.name) html.close() - +#EMAIL HERE if test_config.email_enabled: + setupAttachments(Errors.email_attachs, test_config) Emailer.send_email(test_config.mail_to, test_config.mail_server, test_config.mail_subject, Errors.email_body, Errors.email_attachs) @@ -234,6 +237,7 @@ class TestRunner(object): diffFiles = [ f for f in os.listdir(test_data.output_path) if os.path.isfile(os.path.join(test_data.output_path,f)) ] for f in diffFiles: if f.endswith("Diff.txt"): + #EMAIL HERE Errors.add_email_attachment(os.path.join(test_data.output_path, f)) Errors.add_email_attachment(test_data.common_log_path) return logres @@ -303,6 +307,7 @@ class TestRunner(object): shutil.copy(test_data.sorted_log, error_pth) except IOError as e: Errors.print_error(str(e)) + #EMAIL HERE Errors.add_email_message("Not rebuilt properly") print(str(e)) print(traceback.format_exc()) @@ -365,6 +370,7 @@ class TestRunner(object): test_data.ant.append("-Dgold_path=" + test_config.gold) test_data.ant.append("-Dout_path=" + make_local_path(test_data.output_path)) + test_data.ant.append("-Ddiff_dir="+ test_config.diff_dir) test_data.ant.append("-Dignore_unalloc=" + "%s" % test_config.args.unallocated) test_data.ant.append("-Dtest.timeout=" + str(test_config.timeout)) @@ -609,6 +615,7 @@ class TestConfiguration(object): timer = 0 self.images = [] # Email info + # EMAIL HERE self.email_enabled = args.email_enabled self.mail_server = "" self.mail_to = "" @@ -650,7 +657,10 @@ class TestConfiguration(object): if parsed_config.getElementsByTagName("golddir"): self.gold = parsed_config.getElementsByTagName("golddir")[0].getAttribute("value").encode().decode("utf_8") self.img_gold = make_path(self.gold, 'tmp') - + if parsed_config.getElementsByTagName("diffdir"): + self.diff_dir = parsed_config.getElementsByTagName("diffdir")[0].getAttribute("value").encode().decode("utf_8") + print("660 self.diff_dir is " + str(self.diff_dir)) +#EMAIL HERE self._init_imgs(parsed_config) self._init_build_info(parsed_config) self._init_email_info(parsed_config) @@ -658,6 +668,7 @@ class TestConfiguration(object): except IOError as e: msg = "There was an error loading the configuration file.\n" msg += "\t" + str(e) + # EMAIL HERE Errors.add_email_msg(msg) logging.critical(traceback.format_exc()) print(traceback.format_exc()) @@ -691,6 +702,7 @@ class TestConfiguration(object): else: msg = "File: " + value + " doesn't exist" Errors.print_error(msg) + #EMAIL HERE Errors.add_email_msg(msg) image_count = len(self.images) @@ -704,7 +716,7 @@ class TestConfiguration(object): print("******Alert: There are more input images than gold standards, some images will not be properly tested.\n") elif (image_count < gold_count): print("******Alert: There are more gold standards than input images, this will not check all gold Standards.\n") - +#EMAIL HERE def _init_email_info(self, parsed_config): """Initializes email information dictionary""" email_elements = parsed_config.getElementsByTagName("email") @@ -803,6 +815,7 @@ class TestResultsDiffer(object): diff_file = codecs.open(diff_path, "wb", "utf_8") dffcmdlst = ["diff", output_file, gold_file] subprocess.call(dffcmdlst, stdout = diff_file) + #EMAIL HERE Errors.add_email_attachment(diff_path) msg = "There was a difference in " msg += os.path.basename(output_file) + ".\n" @@ -1472,6 +1485,7 @@ class Errors: email_attchs: a listof_pathto_File, the files to be attached to the report email """ + #EMAIL HERE printout = [] printerror = [] email_body = "" @@ -1484,6 +1498,7 @@ class Errors: Args: image_name: a String, representing the current image being tested """ + #EMAIL HERE Errors.email_msg_prefix = image_name def print_out(msg): @@ -1508,7 +1523,7 @@ class Errors: """Reset the image-specific attributes of the Errors class.""" Errors.printout = [] Errors.printerror = [] - +#EMAIL HERE def add_email_msg(msg): """Add the given message to the body of the report email. @@ -1523,6 +1538,7 @@ class Errors: Args: file: a pathto_File, the file to add """ + #EMAIL HERE Errors.email_attachs.append(path) @@ -1605,6 +1621,7 @@ class Args(object): self.exception = False self.exception_string = "" self.fr = False + #EMAIL HERE self.email_enabled = False def parse(self): @@ -1666,6 +1683,7 @@ class Args(object): print("Not downloading new images") self.fr = True elif arg == "--email": + #EMAIL HERE self.email_enabled = True else: print(usage()) @@ -1875,11 +1893,22 @@ def find_file_in_dir(dir, name, ext): except: raise DirNotFoundException(dir) +def setupAttachments(attachments, test_config): + for file in attachments: + print(str(file)) + print(str(test_config.diff_dir)) + print("I need to figure out how to move these things into that folder.") + filename = ntpath.basename(file) + destination = os.path.join(test_config.diff_dir, filename) + call = ['cp', file, destination] + print("call is " + str(call)) + subprocess.call(call) + #shutil.copy2(filename, os.path.join(test_config.diff_dir, filename)) + class OS: LINUX, MAC, WIN, CYGWIN = range(4) - if __name__ == "__main__": global SYS if _platform == "linux" or _platform == "linux2": From 0b5032918ce475a5769b3e77f2ee74f60bce9482 Mon Sep 17 00:00:00 2001 From: alexjacks92 Date: Thu, 3 Apr 2014 09:47:48 -0400 Subject: [PATCH 02/15] Removing some unneeded logging. --- test/script/regression.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/test/script/regression.py b/test/script/regression.py index ab78332386..ff0c9a44dc 100755 --- a/test/script/regression.py +++ b/test/script/regression.py @@ -1895,13 +1895,10 @@ def find_file_in_dir(dir, name, ext): def setupAttachments(attachments, test_config): for file in attachments: - print(str(file)) - print(str(test_config.diff_dir)) - print("I need to figure out how to move these things into that folder.") filename = ntpath.basename(file) destination = os.path.join(test_config.diff_dir, filename) call = ['cp', file, destination] - print("call is " + str(call)) + print("about to copy " + file + " to " + destination) subprocess.call(call) #shutil.copy2(filename, os.path.join(test_config.diff_dir, filename)) From c980facdab0d64a9681d6152670b19b2e8cdde66 Mon Sep 17 00:00:00 2001 From: alexjacks92 Date: Thu, 3 Apr 2014 15:41:04 -0400 Subject: [PATCH 03/15] Removed mail code, added code to work with jenkins. --- test/script/regression.py | 90 ++++++++++----------------------------- 1 file changed, 23 insertions(+), 67 deletions(-) diff --git a/test/script/regression.py b/test/script/regression.py index ff0c9a44dc..6126293cff 100755 --- a/test/script/regression.py +++ b/test/script/regression.py @@ -34,13 +34,12 @@ import xml from time import localtime, strftime from xml.dom.minidom import parse, parseString import smtplib -from email.mime.image import MIMEImage -from email.mime.multipart import MIMEMultipart -from email.mime.text import MIMEText +#from email.mime.image import MIMEImage +#from email.mime.multipart import MIMEMultipart +#from email.mime.text import MIMEText import re import zipfile import zlib -import Emailer import srcupdater from regression_utils import * import shutil @@ -131,7 +130,6 @@ class TestRunner(object): logres =[] for test_data in test_data_list: Errors.clear_print_logs() - Errors.set_testing_phase(test_data.image) if not (test_config.args.rebuild or os.path.exists(test_data.gold_archive)): msg = "Gold standard doesn't exist, skipping image:" Errors.print_error(msg) @@ -154,25 +152,18 @@ class TestRunner(object): for lm in logres: for ln in lm: #EMAIL HERE - Errors.add_email_msg(ln) + print("I should probably do something with " + str(ln) + " on line 156...") #LOTS OF EMAIL HERE # TODO: possibly worth putting this in a sub method if all([ test_data.overall_passed for test_data in test_data_list ]): - Errors.add_email_msg("All images passed.\n") + print("something needs to be done here") else: - msg = "The following images failed:\n" - for test_data in test_data_list: - if not test_data.overall_passed: - msg += "\t" + test_data.image + "\n" - Errors.add_email_msg(msg) html = open(test_config.html_log) Errors.add_email_attachment(html.name) html.close() #EMAIL HERE - if test_config.email_enabled: - setupAttachments(Errors.email_attachs, test_config) - Emailer.send_email(test_config.mail_to, test_config.mail_server, - test_config.mail_subject, Errors.email_body, Errors.email_attachs) + if test_config.jenkins: + setupAttachments(Errors.email_attachs, test_config) def _run_autopsy_ingest(test_data): """Run Autopsy ingest for the image in the given TestData. @@ -588,6 +579,7 @@ class TestConfiguration(object): images: a listof_Image, the images to be tested timeout: a Nat, the amount of time before killing the test ant: a listof_String, the ant command to run the tests + jenkins: a boolean, is this test running through a Jenkins job? """ def __init__(self, args): @@ -620,6 +612,7 @@ class TestConfiguration(object): self.mail_server = "" self.mail_to = "" self.mail_subject = "" + self.jenkins = False # Set the timeout to something huge # The entire tester should not timeout before this number in ms # However it only seems to take about half this time @@ -657,13 +650,14 @@ class TestConfiguration(object): if parsed_config.getElementsByTagName("golddir"): self.gold = parsed_config.getElementsByTagName("golddir")[0].getAttribute("value").encode().decode("utf_8") self.img_gold = make_path(self.gold, 'tmp') - if parsed_config.getElementsByTagName("diffdir"): - self.diff_dir = parsed_config.getElementsByTagName("diffdir")[0].getAttribute("value").encode().decode("utf_8") - print("660 self.diff_dir is " + str(self.diff_dir)) -#EMAIL HERE + if parsed_config.getElementsByTagName("jenkins"): + self.jenkins = True + if parsed_config.getElementsByTagName("diffdir"): + self.diff_dir = parsed_config.getElementsByTagName("diffdir")[0].getAttribute("value").encode().decode("utf_8") + else: + self.jenkins = False self._init_imgs(parsed_config) self._init_build_info(parsed_config) - self._init_email_info(parsed_config) except IOError as e: msg = "There was an error loading the configuration file.\n" @@ -716,27 +710,6 @@ class TestConfiguration(object): print("******Alert: There are more input images than gold standards, some images will not be properly tested.\n") elif (image_count < gold_count): print("******Alert: There are more gold standards than input images, this will not check all gold Standards.\n") -#EMAIL HERE - def _init_email_info(self, parsed_config): - """Initializes email information dictionary""" - email_elements = parsed_config.getElementsByTagName("email") - if email_elements: - mail_to = email_elements[0] - self.mail_to = mail_to.getAttribute("value").encode().decode("utf_8") - mail_server_elements = parsed_config.getElementsByTagName("mail_server") - if mail_server_elements: - mail_from = mail_server_elements[0] - self.mail_server = mail_from.getAttribute("value").encode().decode("utf_8") - subject_elements = parsed_config.getElementsByTagName("subject") - if subject_elements: - subject = subject_elements[0] - self.mail_subject = subject.getAttribute("value").encode().decode("utf_8") - if self.mail_server and self.mail_to and self.args.email_enabled: - self.email_enabled = True - print("Email will be sent to ", self.mail_to) - else: - self.email_enabled = False - print("No email will be sent.") #-------------------------------------------------# @@ -817,10 +790,6 @@ class TestResultsDiffer(object): subprocess.call(dffcmdlst, stdout = diff_file) #EMAIL HERE Errors.add_email_attachment(diff_path) - msg = "There was a difference in " - msg += os.path.basename(output_file) + ".\n" - Errors.add_email_msg(msg) - Errors.print_error(msg) return False else: return True @@ -1480,27 +1449,14 @@ class Errors: Attributes: printout: a listof_String, the non-error messages that were printed printerror: a listof_String, the error messages that were printed - email_body: a String, the body of the report email - email_msg_prefix: a String, the prefix for lines added to the email email_attchs: a listof_pathto_File, the files to be attached to the report email """ #EMAIL HERE printout = [] printerror = [] - email_body = "" - email_msg_prefix = "Configuration" email_attachs = [] - def set_testing_phase(image_name): - """Change the email message prefix to be the given testing phase. - - Args: - image_name: a String, representing the current image being tested - """ - #EMAIL HERE - Errors.email_msg_prefix = image_name - def print_out(msg): """Print out an informational message. @@ -1523,14 +1479,6 @@ class Errors: """Reset the image-specific attributes of the Errors class.""" Errors.printout = [] Errors.printerror = [] -#EMAIL HERE - def add_email_msg(msg): - """Add the given message to the body of the report email. - - Args: - msg: a String, the message to be added to the email - """ - Errors.email_body += Errors.email_msg_prefix + ":" + msg def add_email_attachment(path): """Add the given file to be an attachment for the report email @@ -1894,6 +1842,14 @@ def find_file_in_dir(dir, name, ext): raise DirNotFoundException(dir) def setupAttachments(attachments, test_config): + """Move email attachments to the location specified in the config file. + Used for Jenkins build. + + Args: + attachments: a listof_String, the files to be moved + test_config: TestConfiguration, used to determine where to move the files to + """ + for file in attachments: filename = ntpath.basename(file) destination = os.path.join(test_config.diff_dir, filename) From 13a176f413e3582f5b217b3298771639da16b11b Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Fri, 4 Apr 2014 09:43:28 -0400 Subject: [PATCH 04/15] Removed sleeps from IngestManager.stoAll(), added more cancel checks --- .../autopsy/ingest/IngestManager.java | 76 +++++++++---------- 1 file changed, 37 insertions(+), 39 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java index 64159e19f8..833575169a 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java @@ -24,6 +24,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.concurrent.CancellationException; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.logging.Level; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.coreutils.Logger; @@ -362,15 +363,9 @@ public class IngestManager { // First get the task scheduling worker to stop adding new tasks. if (taskSchedulingWorker != null) { taskSchedulingWorker.cancel(true); - while (!taskSchedulingWorker.isDone()) { - try { - Thread.sleep(1000); - } catch (InterruptedException ex) { - } - } taskSchedulingWorker = null; - } - + } + // Now mark all of the ingest jobs as cancelled. This way the ingest // modules will know they are being shut down due to cancellation when // the cancelled ingest workers release their pipelines. @@ -378,16 +373,15 @@ public class IngestManager { job.cancel(); } + // Jettision the remaining tasks. This will dispose of any tasks that + // the scheduling worker queued up before it was cancelled. + scheduler.getFileScheduler().empty(); + scheduler.getDataSourceScheduler().empty(); + // Cancel the data source task worker. It will release its pipelines // in its done() method and the pipelines will shut down their modules. if (dataSourceTaskWorker != null) { dataSourceTaskWorker.cancel(true); - while (!dataSourceTaskWorker.isDone()) { - try { - Thread.sleep(1000); - } catch (InterruptedException ex) { - } - } dataSourceTaskWorker = null; } @@ -397,26 +391,20 @@ public class IngestManager { for (FileTaskWorker worker : fileTaskWorkers) { if (worker != null) { worker.cancel(true); - while (!worker.isDone()) { - try { - Thread.sleep(1000); - } catch (InterruptedException ex) { - } - } worker = null; } } - // Jettision the remaining tasks. This will dispose of any tasks that - // the scheduling worker queued up before it was cancelled. + // Jettision the remaining tasks again to try to dispose of any tasks + // queued up task workers before they were cancelled. scheduler.getFileScheduler().empty(); scheduler.getDataSourceScheduler().empty(); } /** - * Test if any ingest modules are running + * Test if any ingest jobs are in progress. * - * @return true if any module is running, false otherwise + * @return True if any ingest jobs are in progress, false otherwise */ public synchronized boolean isIngestRunning() { // TODO: There is a race condition here with SwingWorker.isDone(). @@ -474,6 +462,7 @@ public class IngestManager { private final List moduleTemplates; private final boolean processUnallocatedSpace; private ProgressHandle progress; + private volatile boolean finished = false; TaskSchedulingWorker(List dataSources, List moduleTemplates, boolean processUnallocatedSpace) { this.dataSources = dataSources; @@ -483,13 +472,10 @@ public class IngestManager { @Override protected Object doInBackground() throws Exception { - // Set up a progress bar that can be used to cancel all of the - // ingest jobs currently being performed. final String displayName = "Queueing ingest tasks"; progress = ProgressHandleFactory.createHandle(displayName, new Cancellable() { @Override public boolean cancel() { - logger.log(Level.INFO, "Queueing ingest cancelled by user."); if (progress != null) { progress.setDisplayName(displayName + " (Cancelling...)"); } @@ -502,10 +488,8 @@ public class IngestManager { int processed = 0; for (Content dataSource : dataSources) { if (isCancelled()) { - logger.log(Level.INFO, "Task scheduling thread cancelled"); return null; } - final String inputName = dataSource.getName(); IngestJob ingestJob = new IngestJob(IngestManager.this.getNextDataSourceTaskId(), dataSource, moduleTemplates, processUnallocatedSpace); @@ -521,7 +505,7 @@ public class IngestManager { failedModules.append(","); } } - MessageNotifyUtil.Message.error( + MessageNotifyUtil.Message.error( // RJCTODO: Fix this "Failed to start the following ingest modules: " + failedModules.toString() + " .\n\n" + "No ingest modules will be run. Please disable the module " + "or fix the error and restart ingest by right clicking on " @@ -552,7 +536,6 @@ public class IngestManager { try { super.get(); } catch (CancellationException | InterruptedException ex) { - // IngestManager.stopAll() will dispose of all tasks. } catch (Exception ex) { logger.log(Level.SEVERE, "Error while scheduling ingest jobs", ex); MessageNotifyUtil.Message.error("An error occurred while starting ingest. Results may only be partial"); @@ -561,8 +544,13 @@ public class IngestManager { startAll(); } progress.finish(); + finished = true; } } + + boolean isFinished() { + return finished; + } } /** @@ -572,6 +560,7 @@ public class IngestManager { class DataSourceTaskWorker extends SwingWorker { private final long id; + private volatile boolean finished = false; DataSourceTaskWorker(long threadId) { this.id = threadId; @@ -579,18 +568,18 @@ public class IngestManager { @Override protected Void doInBackground() throws Exception { - logger.log(Level.INFO, "Data source ingest thread (id={0}) started", this.id); IngestScheduler.DataSourceScheduler scheduler = IngestScheduler.getInstance().getDataSourceScheduler(); while (scheduler.hasNext()) { if (isCancelled()) { - logger.log(Level.INFO, "Data source ingest thread (id={0}) cancelled", this.id); return null; } IngestJob job = scheduler.next(); DataSourceIngestPipeline pipeline = job.getDataSourceIngestPipelineForThread(this.id); pipeline.process(this, job.getDataSourceTaskProgressBar()); + if (isCancelled()) { + return null; + } } - logger.log(Level.INFO, "Data source ingest thread (id={0}) completed", this.id); return null; } @@ -599,14 +588,18 @@ public class IngestManager { try { super.get(); } catch (CancellationException | InterruptedException e) { - logger.log(Level.INFO, "Data source ingest thread (id={0}) cancelled", this.id); } catch (Exception ex) { String message = String.format("Data source ingest thread (id=%d) experienced a fatal error", this.id); logger.log(Level.SEVERE, message, ex); } finally { IngestManager.getInstance().reportThreadDone(this.id); + finished = true; } } + + boolean isFinished() { + return finished; + } } /** @@ -616,6 +609,7 @@ public class IngestManager { class FileTaskWorker extends SwingWorker { private final long id; + private volatile boolean finished = false; FileTaskWorker(long threadId) { this.id = threadId; @@ -623,11 +617,9 @@ public class IngestManager { @Override protected Object doInBackground() throws Exception { - logger.log(Level.INFO, "File ingest thread (id={0}) started", this.id); IngestScheduler.FileScheduler fileScheduler = IngestScheduler.getInstance().getFileScheduler(); while (fileScheduler.hasNext()) { if (isCancelled()) { - logger.log(Level.INFO, "File ingest thread (id={0}) cancelled", this.id); return null; } IngestScheduler.FileScheduler.FileTask task = fileScheduler.next(); @@ -635,8 +627,10 @@ public class IngestManager { FileIngestPipeline pipeline = job.getFileIngestPipelineForThread(this.id); job.handleFileTaskStarted(task); pipeline.process(task.getFile()); + if (isCancelled()) { + return null; + } } - logger.log(Level.INFO, "File ingest thread (id={0}) completed", this.id); return null; } @@ -645,13 +639,17 @@ public class IngestManager { try { super.get(); } catch (CancellationException | InterruptedException e) { - logger.log(Level.INFO, "File ingest thread (id={0}) cancelled", this.id); } catch (Exception ex) { String message = String.format("File ingest thread {0} experienced a fatal error", this.id); logger.log(Level.SEVERE, message, ex); } finally { IngestManager.getInstance().reportThreadDone(this.id); + finished = true; } } + + boolean isFinished() { + return finished; + } } } From 48025c2e15186c69e290e91c595581746bdc9f90 Mon Sep 17 00:00:00 2001 From: alexjacks92 Date: Fri, 4 Apr 2014 15:50:18 -0400 Subject: [PATCH 05/15] Removed unneeded logging, removed email code, refactored variable names from email-centric to jenkins- and error-centric names. --- test/script/regression.py | 53 ++++++++------------------------------- 1 file changed, 10 insertions(+), 43 deletions(-) diff --git a/test/script/regression.py b/test/script/regression.py index 6126293cff..a80e301383 100755 --- a/test/script/regression.py +++ b/test/script/regression.py @@ -34,9 +34,6 @@ import xml from time import localtime, strftime from xml.dom.minidom import parse, parseString import smtplib -#from email.mime.image import MIMEImage -#from email.mime.multipart import MIMEMultipart -#from email.mime.text import MIMEText import re import zipfile import zlib @@ -147,23 +144,16 @@ class TestRunner(object): time.sleep(10) Reports.write_html_foot(test_config.html_log) - # TODO: move this elsewhere - if (len(logres)>0): - for lm in logres: - for ln in lm: - #EMAIL HERE - print("I should probably do something with " + str(ln) + " on line 156...") -#LOTS OF EMAIL HERE # TODO: possibly worth putting this in a sub method if all([ test_data.overall_passed for test_data in test_data_list ]): - print("something needs to be done here") + pass else: html = open(test_config.html_log) - Errors.add_email_attachment(html.name) + Errors.add_errors_out(html.name) html.close() -#EMAIL HERE + if test_config.jenkins: - setupAttachments(Errors.email_attachs, test_config) + setupAttachments(Errors.errors_out, test_config) def _run_autopsy_ingest(test_data): """Run Autopsy ingest for the image in the given TestData. @@ -228,9 +218,8 @@ class TestRunner(object): diffFiles = [ f for f in os.listdir(test_data.output_path) if os.path.isfile(os.path.join(test_data.output_path,f)) ] for f in diffFiles: if f.endswith("Diff.txt"): - #EMAIL HERE - Errors.add_email_attachment(os.path.join(test_data.output_path, f)) - Errors.add_email_attachment(test_data.common_log_path) + Errors.add_errors_out(os.path.join(test_data.output_path, f)) + Errors.add_errors_out(test_data.common_log_path) return logres def _extract_gold(test_data): @@ -298,8 +287,6 @@ class TestRunner(object): shutil.copy(test_data.sorted_log, error_pth) except IOError as e: Errors.print_error(str(e)) - #EMAIL HERE - Errors.add_email_message("Not rebuilt properly") print(str(e)) print(traceback.format_exc()) # Rebuild the HTML report @@ -606,12 +593,6 @@ class TestConfiguration(object): # Infinite Testing info timer = 0 self.images = [] - # Email info - # EMAIL HERE - self.email_enabled = args.email_enabled - self.mail_server = "" - self.mail_to = "" - self.mail_subject = "" self.jenkins = False # Set the timeout to something huge # The entire tester should not timeout before this number in ms @@ -662,8 +643,6 @@ class TestConfiguration(object): except IOError as e: msg = "There was an error loading the configuration file.\n" msg += "\t" + str(e) - # EMAIL HERE - Errors.add_email_msg(msg) logging.critical(traceback.format_exc()) print(traceback.format_exc()) @@ -696,8 +675,6 @@ class TestConfiguration(object): else: msg = "File: " + value + " doesn't exist" Errors.print_error(msg) - #EMAIL HERE - Errors.add_email_msg(msg) image_count = len(self.images) # Sanity check to see if there are obvious gold images that we are not testing @@ -788,8 +765,7 @@ class TestResultsDiffer(object): diff_file = codecs.open(diff_path, "wb", "utf_8") dffcmdlst = ["diff", output_file, gold_file] subprocess.call(dffcmdlst, stdout = diff_file) - #EMAIL HERE - Errors.add_email_attachment(diff_path) + Errors.add_errors_out(diff_path) return False else: return True @@ -1452,10 +1428,9 @@ class Errors: email_attchs: a listof_pathto_File, the files to be attached to the report email """ - #EMAIL HERE printout = [] printerror = [] - email_attachs = [] + errors_out = [] def print_out(msg): """Print out an informational message. @@ -1480,14 +1455,13 @@ class Errors: Errors.printout = [] Errors.printerror = [] - def add_email_attachment(path): + def add_errors_out(path): """Add the given file to be an attachment for the report email Args: file: a pathto_File, the file to add """ - #EMAIL HERE - Errors.email_attachs.append(path) + Errors.errors_out.append(path) class DiffResults(object): @@ -1569,8 +1543,6 @@ class Args(object): self.exception = False self.exception_string = "" self.fr = False - #EMAIL HERE - self.email_enabled = False def parse(self): """Get the command line arguments and parse them.""" @@ -1630,9 +1602,6 @@ class Args(object): elif arg == "-fr" or arg == "--forcerun": print("Not downloading new images") self.fr = True - elif arg == "--email": - #EMAIL HERE - self.email_enabled = True else: print(usage()) return False @@ -1854,9 +1823,7 @@ def setupAttachments(attachments, test_config): filename = ntpath.basename(file) destination = os.path.join(test_config.diff_dir, filename) call = ['cp', file, destination] - print("about to copy " + file + " to " + destination) subprocess.call(call) - #shutil.copy2(filename, os.path.join(test_config.diff_dir, filename)) class OS: From e5e1dbe07508d40320360b2ece7f927c8586668e Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Mon, 7 Apr 2014 09:50:45 -0400 Subject: [PATCH 06/15] Experimented with ingest job cancellation improvements --- .../DataSourceIngestModuleStatusHelper.java | 9 +- .../ingest/DataSourceIngestPipeline.java | 4 +- .../sleuthkit/autopsy/ingest/IngestJob.java | 4 +- .../autopsy/ingest/IngestJobContext.java | 2 +- .../autopsy/ingest/IngestJobLauncher.java | 2 +- .../autopsy/ingest/IngestManager.java | 587 ++++++++---------- .../ingest/IngestMessageTopComponent.java | 2 +- .../autopsy/ingest/IngestMonitor.java | 6 +- 8 files changed, 260 insertions(+), 356 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleStatusHelper.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleStatusHelper.java index 689d8e7652..4779f59c63 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleStatusHelper.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleStatusHelper.java @@ -20,7 +20,6 @@ package org.sleuthkit.autopsy.ingest; import javax.swing.SwingWorker; import org.netbeans.api.progress.ProgressHandle; -import org.sleuthkit.datamodel.Content; /** * Used by data source ingest modules to report progress and detect data source @@ -28,15 +27,11 @@ import org.sleuthkit.datamodel.Content; */ public class DataSourceIngestModuleStatusHelper { - private final SwingWorker worker; private final ProgressHandle progress; - private final Content dataSource; private final String moduleDisplayName; - DataSourceIngestModuleStatusHelper(SwingWorker worker, ProgressHandle progress, Content dataSource, String moduleDisplayName) { - this.worker = worker; + DataSourceIngestModuleStatusHelper(ProgressHandle progress, String moduleDisplayName) { this.progress = progress; - this.dataSource = dataSource; this.moduleDisplayName = moduleDisplayName; } @@ -48,7 +43,7 @@ public class DataSourceIngestModuleStatusHelper { * @return True if the task has been canceled, false otherwise. */ public boolean isIngestJobCancelled() { - return worker.isCancelled(); + return (Thread.currentThread().isInterrupted()); // RJCTODO: This is not right? Appears to be right... } /** diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java index 3b919391bc..8a6f07786d 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java @@ -81,13 +81,13 @@ final class DataSourceIngestPipeline { return errors; } - List process(SwingWorker worker, ProgressHandle progress) { + List process(ProgressHandle progress) { List errors = new ArrayList<>(); Content dataSource = this.job.getDataSource(); logger.log(Level.INFO, "Processing data source {0}", dataSource.getName()); for (DataSourceIngestModuleDecorator module : this.modules) { try { - module.process(dataSource, new DataSourceIngestModuleStatusHelper(worker, progress, dataSource, module.getDisplayName())); + module.process(dataSource, new DataSourceIngestModuleStatusHelper(progress, module.getDisplayName())); } catch (Exception ex) { errors.add(new IngestModuleError(module.getDisplayName(), ex)); } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java index 0a65bafcbb..80f75818d6 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java @@ -85,7 +85,7 @@ final class IngestJob { "IngestJob.progress.cancelling", displayName)); } - IngestManager.getInstance().stopAll(); + IngestManager.getInstance().cancelIngestTasks(); return true; } }); @@ -104,7 +104,7 @@ final class IngestJob { NbBundle.getMessage(this.getClass(), "IngestJob.progress.cancelling", displayName)); } - IngestManager.getInstance().stopAll(); + IngestManager.getInstance().cancelIngestTasks(); return true; } }); diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java index 7d0301ce5c..dc9fb58e3f 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java @@ -60,7 +60,7 @@ public final class IngestJobContext { */ public void addFiles(List files) { for (AbstractFile file : files) { - IngestManager.getInstance().scheduleFile(ingestJob.getId(), file); + IngestManager.getInstance().addFileToIngestJob(ingestJob.getId(), file); } } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobLauncher.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobLauncher.java index fbd9fb0db3..22125647d7 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobLauncher.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobLauncher.java @@ -232,7 +232,7 @@ public final class IngestJobLauncher { } if ((!enabledModuleTemplates.isEmpty()) && (dataSources != null) && (!dataSources.isEmpty())) { - IngestManager.getInstance().scheduleDataSourceTasks(dataSources, enabledModuleTemplates, ingestConfigPanel.getProcessUnallocSpace()); + IngestManager.getInstance().startIngestJobs(dataSources, enabledModuleTemplates, ingestConfigPanel.getProcessUnallocSpace()); } } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java index 833575169a..44286f87dc 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java @@ -23,12 +23,14 @@ import java.beans.PropertyChangeSupport; import java.util.ArrayList; import java.util.HashMap; import java.util.List; -import java.util.concurrent.CancellationException; -import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; import java.util.logging.Level; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.coreutils.Logger; -import javax.swing.SwingWorker; import org.netbeans.api.progress.ProgressHandle; import org.netbeans.api.progress.ProgressHandleFactory; import org.openide.util.Cancellable; @@ -49,18 +51,73 @@ public class IngestManager { private static final int DEFAULT_NUMBER_OF_FILE_INGEST_THREADS = 2; private static final Logger logger = Logger.getLogger(IngestManager.class.getName()); private static final PropertyChangeSupport pcs = new PropertyChangeSupport(IngestManager.class); + private static final Preferences userPreferences = NbPreferences.forModule(IngestManager.class); private static IngestManager instance; private final IngestScheduler scheduler = IngestScheduler.getInstance(); private final IngestMonitor ingestMonitor = new IngestMonitor(); - private final Preferences userPreferences = NbPreferences.forModule(this.getClass()); - private final HashMap ingestJobs = new HashMap<>(); - private TaskSchedulingWorker taskSchedulingWorker = null; - private DataSourceTaskWorker dataSourceTaskWorker = null; - private final List fileTaskWorkers = new ArrayList<>(); - private long nextDataSourceTaskId = 0; - private long nextThreadId = 0; + private ExecutorService startIngestJobsExecutor = null; + private ExecutorService dataSourceIngestTasksExecutor = null; + private ExecutorService fileIngestTasksExecutor = null; + private AtomicLong ingestJobId = new AtomicLong(0L); + private AtomicLong ingestTaskId = new AtomicLong(0L); + private final HashMap ingestJobs = new HashMap<>(); // Maps job ids to jobs + private final HashMap> ingestTasks = new HashMap<>(); // Maps task ids to Runnable tasks +// private TaskSchedulingWorker taskSchedulingWorker = null; private volatile IngestUI ingestMessageBox; + /** + * Gets the IngestManager singleton, creating it if necessary. + * + * @returns The IngestManager singleton. + */ + public synchronized static IngestManager getInstance() { + if (instance == null) { + instance = new IngestManager(); + } + return instance; + } + + private IngestManager() { + createThreadPools(); + } + + private synchronized void createThreadPools() { + if (startIngestJobsExecutor == null) { + startIngestJobsExecutor = Executors.newSingleThreadExecutor(); + } + if (dataSourceIngestTasksExecutor == null) { + dataSourceIngestTasksExecutor = Executors.newSingleThreadExecutor(); + } + if (fileIngestTasksExecutor == null) { + fileIngestTasksExecutor = Executors.newFixedThreadPool(MAX_NUMBER_OF_FILE_INGEST_THREADS); + } + } + + public synchronized static int getNumberOfFileIngestThreads() { + return userPreferences.getInt(NUMBER_OF_FILE_INGEST_THREADS_KEY, DEFAULT_NUMBER_OF_FILE_INGEST_THREADS); + } + + public static void setNumberOfFileIngestThreads(int numberOfThreads) { + if (numberOfThreads < MIN_NUMBER_OF_FILE_INGEST_THREADS + || numberOfThreads > MAX_NUMBER_OF_FILE_INGEST_THREADS) { + numberOfThreads = DEFAULT_NUMBER_OF_FILE_INGEST_THREADS; + } + + synchronized (IngestManager.class) { + userPreferences.putInt(NUMBER_OF_FILE_INGEST_THREADS_KEY, numberOfThreads); + } + } + + /** + * Finds the ingest messages in box TopComponent. Called by the custom + * installer for this package once the window system is initialized. + */ + void initIngestMessageInbox() { + if (this.ingestMessageBox == null) { + this.ingestMessageBox = IngestMessageTopComponent.findInstance(); + } + } + /** * Ingest events. */ @@ -110,65 +167,20 @@ public class IngestManager { FILE_DONE, }; - private IngestManager() { - } - /** - * Returns reference to singleton instance. - * - * @returns Instance of class. - */ - synchronized public static IngestManager getInstance() { - if (instance == null) { - instance = new IngestManager(); - } - return instance; - } - - /** - * called by Installer in AWT thread once the Window System is ready - */ - void initIngestMessageInbox() { - if (this.ingestMessageBox == null) { - this.ingestMessageBox = IngestMessageTopComponent.findInstance(); - } - } - - synchronized private long getNextDataSourceTaskId() { - return ++this.nextDataSourceTaskId; - } - - synchronized private long getNextThreadId() { - return ++this.nextThreadId; - } - - public synchronized int getNumberOfFileIngestThreads() { - return userPreferences.getInt(NUMBER_OF_FILE_INGEST_THREADS_KEY, DEFAULT_NUMBER_OF_FILE_INGEST_THREADS); - } - - public synchronized void setNumberOfFileIngestThreads(int numberOfThreads) { - if (numberOfThreads < MIN_NUMBER_OF_FILE_INGEST_THREADS - || numberOfThreads > MAX_NUMBER_OF_FILE_INGEST_THREADS) { - numberOfThreads = DEFAULT_NUMBER_OF_FILE_INGEST_THREADS; - } - userPreferences.putInt(NUMBER_OF_FILE_INGEST_THREADS_KEY, numberOfThreads); - } - - /** - * Add property change listener to listen to ingest events as defined in - * IngestModuleEvent. + * Add property change listener to listen to ingest events. * * @param listener PropertyChangeListener to register */ - public static synchronized void addPropertyChangeListener(final PropertyChangeListener listener) { + synchronized public static void addPropertyChangeListener(final PropertyChangeListener listener) { pcs.addPropertyChangeListener(listener); } - public static synchronized void removePropertyChangeListener(final PropertyChangeListener listener) { + synchronized public static void removePropertyChangeListener(final PropertyChangeListener listener) { pcs.removePropertyChangeListener(listener); } - static synchronized void fireModuleEvent(String eventType, String moduleName) { + synchronized static void fireModuleEvent(String eventType, String moduleName) { try { pcs.firePropertyChange(eventType, moduleName, null); } catch (Exception e) { @@ -184,7 +196,7 @@ public class IngestManager { * * @param objId ID of file that is done */ - static synchronized void fireFileDone(long objId) { + synchronized static void fireFileDone(long objId) { try { pcs.firePropertyChange(IngestEvent.FILE_DONE.toString(), objId, null); } catch (Exception e) { @@ -201,7 +213,7 @@ public class IngestManager { * * @param moduleDataEvent */ - static synchronized void fireModuleDataEvent(ModuleDataEvent moduleDataEvent) { + synchronized static void fireModuleDataEvent(ModuleDataEvent moduleDataEvent) { try { pcs.firePropertyChange(IngestEvent.DATA.toString(), moduleDataEvent, null); } catch (Exception e) { @@ -218,7 +230,7 @@ public class IngestManager { * * @param moduleContentEvent */ - static synchronized void fireModuleContentEvent(ModuleContentEvent moduleContentEvent) { + synchronized static void fireModuleContentEvent(ModuleContentEvent moduleContentEvent) { try { pcs.firePropertyChange(IngestEvent.CONTENT_CHANGED.toString(), moduleContentEvent, null); } catch (Exception e) { @@ -229,126 +241,63 @@ public class IngestManager { } } - /** - * Multiple data-sources version of scheduleDataSource() method. Enqueues - * multiple sources inputs (Content objects) and associated modules at once - * - * @param modules modules to scheduleDataSource on every data source - * @param inputs input data sources to enqueue and scheduleDataSource the - * ingest modules on - */ - void scheduleDataSourceTasks(final List dataSources, final List moduleTemplates, boolean processUnallocatedSpace) { + synchronized void startIngestJob(final Content dataSource, final List moduleTemplates, boolean processUnallocatedSpace) { + List dataSources = new ArrayList<>(); + dataSources.add(dataSource); + startIngestJobs(dataSources, moduleTemplates, processUnallocatedSpace); + } + + synchronized void startIngestJobs(final List dataSources, final List moduleTemplates, boolean processUnallocatedSpace) { if (!isIngestRunning() && ingestMessageBox != null) { ingestMessageBox.clearMessages(); } - taskSchedulingWorker = new TaskSchedulingWorker(dataSources, moduleTemplates, processUnallocatedSpace); - taskSchedulingWorker.execute(); + createThreadPools(); + + long taskId = ingestTaskId.incrementAndGet(); + Future dataSourceIngestTask = startIngestJobsExecutor.submit(new StartIngestJobsTask(taskId, dataSources, moduleTemplates, processUnallocatedSpace)); + ingestTasks.put(taskId, dataSourceIngestTask); if (ingestMessageBox != null) { ingestMessageBox.restoreMessages(); } } - /** - * IngestManager entry point, enqueues data to be processed and starts new - * ingest as needed, or just enqueues data to an existing pipeline. - * - * Spawns background thread which enumerates all sorted files and executes - * chosen modules per file in a pre-determined order. Notifies modules when - * work is complete or should be interrupted using complete() and stop() - * calls. Does not block and can be called multiple times to enqueue more - * work to already running background ingest process. - * - * @param modules modules to scheduleDataSource on the data source input - * @param input input data source Content objects to scheduleDataSource the - * ingest modules on - */ - void scheduleDataSourceTask(final Content dataSource, final List moduleTemplates, boolean processUnallocatedSpace) { - List dataSources = new ArrayList<>(); - dataSources.add(dataSource); - scheduleDataSourceTasks(dataSources, moduleTemplates, processUnallocatedSpace); - } - - /** - * Schedule a file for ingest and add it to ongoing file ingest process on - * the same data source. Scheduler updates the current progress. - * - * The file to be added is usually a product of a currently ran ingest. Now - * we want to process this new file with the same ingest context. - * - * @param file file to be scheduled - * @param pipelineContext ingest context used to ingest parent of the file - * to be scheduled - */ - void scheduleFile(long ingestJobId, AbstractFile file) { + synchronized void addFileToIngestJob(long ingestJobId, AbstractFile file) { IngestJob job = this.ingestJobs.get(ingestJobId); - if (job == null) { + if (job != null) { + scheduler.getFileScheduler().scheduleFile(job, file); + } else { logger.log(Level.SEVERE, "Unable to map ingest job id (id = {0}) to an ingest job, failed to schedule file (id = {1})", new Object[]{ingestJobId, file.getId()}); MessageNotifyUtil.Notify.show(NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr"), "Unable to associate " + file.getName() + " with ingest job, file will not be processed by ingest nodules", MessageNotifyUtil.MessageType.ERROR); } - - scheduler.getFileScheduler().scheduleFile(job, file); } - private synchronized void startAll() { - // Make sure the ingest monitor is running. + private synchronized void startIngestTasks() { if (!ingestMonitor.isRunning()) { ingestMonitor.start(); } + + long taskId = ingestTaskId.incrementAndGet(); + Future dataSourceIngestTask = dataSourceIngestTasksExecutor.submit(new RunDataSourceIngestModulesTask(taskId)); + ingestTasks.put(taskId, dataSourceIngestTask); - // Make sure a data source task worker is running. - // TODO: There is a race condition here with SwingWorker.isDone(). - // The highly unlikely chance that no data source task worker will - // run for this job needs to be addressed. Fix by using a thread pool - // and converting the SwingWorkers to Runnables. - if (dataSourceTaskWorker == null || dataSourceTaskWorker.isDone()) { - dataSourceTaskWorker = new DataSourceTaskWorker(getNextThreadId()); - dataSourceTaskWorker.execute(); - } - - // Make sure the requested number of file task workers are running. - // TODO: There is a race condition here with SwingWorker.isDone(). - // The highly unlikely chance that no file task workers or the wrong - // number of file task workers will run for this job needs to be - // addressed. Fix by using a thread pool and converting the SwingWorkers - // to Runnables. int workersRequested = getNumberOfFileIngestThreads(); - int workersRunning = 0; - for (FileTaskWorker worker : fileTaskWorkers) { - if (worker != null) { - if (worker.isDone()) { - if (workersRunning < workersRequested) { - worker = new FileTaskWorker(getNextThreadId()); - worker.execute(); - ++workersRunning; - } else { - worker = null; - } - } else { - ++workersRunning; - } - } else if (workersRunning < workersRequested) { - worker = new FileTaskWorker(getNextThreadId()); - worker.execute(); - ++workersRunning; - } - } - while (workersRunning < workersRequested - && fileTaskWorkers.size() < MAX_NUMBER_OF_FILE_INGEST_THREADS) { - FileTaskWorker worker = new FileTaskWorker(getNextThreadId()); - fileTaskWorkers.add(worker); - worker.execute(); - ++workersRunning; + for (int i = 0; i < workersRequested; ++i) { + taskId = ingestTaskId.incrementAndGet(); + Future fileIngestTask = fileIngestTasksExecutor.submit(new RunFileSourceIngestModulesTask(taskId)); + ingestTasks.put(taskId, fileIngestTask); } } - synchronized void reportThreadDone(long threadId) { + synchronized void reportIngestTaskDone(long taskId) { + ingestTasks.remove(taskId); + List completedJobs = new ArrayList<>(); for (IngestJob job : ingestJobs.values()) { - job.releaseIngestPipelinesForThread(threadId); + job.releaseIngestPipelinesForThread(taskId); if (job.areIngestPipelinesShutDown()) { completedJobs.add(job.getId()); } @@ -359,12 +308,11 @@ public class IngestManager { } } - synchronized void stopAll() { + synchronized void cancelIngestTasks() { +// synchronized void cancelIngestTasks(int waitTime, TimeUnit unit) { // RJCTODO // First get the task scheduling worker to stop adding new tasks. - if (taskSchedulingWorker != null) { - taskSchedulingWorker.cancel(true); - taskSchedulingWorker = null; - } +// boolean res = shutDownThreadPool(startIngestJobsExecutor, 1, TimeUnit.SECONDS); +// startIngestJobsExecutor = null; // Now mark all of the ingest jobs as cancelled. This way the ingest // modules will know they are being shut down due to cancellation when @@ -373,59 +321,58 @@ public class IngestManager { job.cancel(); } - // Jettision the remaining tasks. This will dispose of any tasks that + // Jettision the remaining data tasks. This will dispose of any tasks that // the scheduling worker queued up before it was cancelled. - scheduler.getFileScheduler().empty(); - scheduler.getDataSourceScheduler().empty(); - - // Cancel the data source task worker. It will release its pipelines - // in its done() method and the pipelines will shut down their modules. - if (dataSourceTaskWorker != null) { - dataSourceTaskWorker.cancel(true); - dataSourceTaskWorker = null; - } - - // Cancel the file task workers. They will release their pipelines - // in their done() methods and the pipelines will shut down their - // modules. - for (FileTaskWorker worker : fileTaskWorkers) { - if (worker != null) { - worker.cancel(true); - worker = null; - } +// scheduler.getFileScheduler().empty(); +// scheduler.getDataSourceScheduler().empty(); + + // Cancel all of the ingest module running tasks. + for (Future task : ingestTasks.values()) { + task.cancel(true); } +// res = shutDownThreadPool(dataSourceIngestTasksExecutor, 30, TimeUnit.SECONDS); +// dataSourceIngestTasksExecutor = null; +// res = shutDownThreadPool(fileIngestTasksExecutor, 30, TimeUnit.SECONDS); +// fileIngestTasksExecutor = null; + // Jettision the remaining tasks again to try to dispose of any tasks // queued up task workers before they were cancelled. - scheduler.getFileScheduler().empty(); - scheduler.getDataSourceScheduler().empty(); +// scheduler.getFileScheduler().empty(); +// scheduler.getDataSourceScheduler().empty(); } + // The following method implementation is adapted from: + // http://docs.oracle.com/javase/7/docs/api/java/util/concurrent/ExecutorService.html#awaitTermination(long, java.util.concurrent.TimeUnit) + private boolean shutDownThreadPool(ExecutorService pool, int waitTime, TimeUnit unit) { + boolean succeeded = true; + // Prevent submission of new tasks. + pool.shutdown(); + try { + // Wait a while for existing tasks to terminate. + if (!pool.awaitTermination(1, TimeUnit.SECONDS)) { + // Cancel currently executing tasks. + pool.shutdownNow(); + if (!pool.awaitTermination(waitTime, unit)) { + succeeded = false; + } + } + } catch (InterruptedException ex) { + // (Re-)Cancel if current thread also interrupted. + pool.shutdownNow(); + // Preserve interrupt status. + Thread.currentThread().interrupt(); + } + return succeeded; + } + /** * Test if any ingest jobs are in progress. * * @return True if any ingest jobs are in progress, false otherwise */ public synchronized boolean isIngestRunning() { - // TODO: There is a race condition here with SwingWorker.isDone(). - // It probably needs to be addressed at a later date. If we replace the - // SwingWorkers with a thread pool and Runnables, one solution would be - // to check the ingest jobs list. - if (taskSchedulingWorker != null && !taskSchedulingWorker.isDone()) { - return true; - } - - if (dataSourceTaskWorker != null && !dataSourceTaskWorker.isDone()) { - return true; - } - - for (FileTaskWorker worker : fileTaskWorkers) { - if (worker != null && !worker.isDone()) { - return true; - } - } - - return false; + return (ingestJobs.isEmpty() == false); } /** @@ -456,95 +403,89 @@ public class IngestManager { } } - private class TaskSchedulingWorker extends SwingWorker { + private class StartIngestJobsTask implements Runnable { + private final long id; private final List dataSources; private final List moduleTemplates; private final boolean processUnallocatedSpace; private ProgressHandle progress; private volatile boolean finished = false; - TaskSchedulingWorker(List dataSources, List moduleTemplates, boolean processUnallocatedSpace) { + StartIngestJobsTask(long taskId, List dataSources, List moduleTemplates, boolean processUnallocatedSpace) { + this.id = taskId; this.dataSources = dataSources; this.moduleTemplates = moduleTemplates; this.processUnallocatedSpace = processUnallocatedSpace; } @Override - protected Object doInBackground() throws Exception { - final String displayName = "Queueing ingest tasks"; - progress = ProgressHandleFactory.createHandle(displayName, new Cancellable() { - @Override - public boolean cancel() { - if (progress != null) { - progress.setDisplayName(displayName + " (Cancelling...)"); - } - IngestManager.getInstance().stopAll(); - return true; - } - }); - - progress.start(2 * dataSources.size()); - int processed = 0; - for (Content dataSource : dataSources) { - if (isCancelled()) { - return null; - } - final String inputName = dataSource.getName(); - IngestJob ingestJob = new IngestJob(IngestManager.this.getNextDataSourceTaskId(), dataSource, moduleTemplates, processUnallocatedSpace); - - List errors = ingestJob.startUpIngestPipelines(); - if (!errors.isEmpty()) { - StringBuilder failedModules = new StringBuilder(); - for (int i = 0; i < errors.size(); ++i) { - IngestModuleError error = errors.get(i); - String moduleName = error.getModuleDisplayName(); - logger.log(Level.SEVERE, "The " + moduleName + " module failed to start up", error.getModuleError()); - failedModules.append(moduleName); - if ((errors.size() > 1) && (i != (errors.size() - 1))) { - failedModules.append(","); + public void run() { + try { + final String displayName = "Queueing ingest tasks"; + progress = ProgressHandleFactory.createHandle(displayName, new Cancellable() { + @Override + public boolean cancel() { + if (progress != null) { + progress.setDisplayName(displayName + " (Cancelling...)"); } + IngestManager.getInstance().cancelIngestTasks(); + return true; } - MessageNotifyUtil.Message.error( // RJCTODO: Fix this - "Failed to start the following ingest modules: " + failedModules.toString() + " .\n\n" - + "No ingest modules will be run. Please disable the module " - + "or fix the error and restart ingest by right clicking on " - + "the data source and selecting Run Ingest Modules.\n\n" - + "Error: " + errors.get(0).getModuleError().getMessage()); - return null; + }); + + progress.start(2 * dataSources.size()); + int processed = 0; + for (Content dataSource : dataSources) { + if (Thread.currentThread().isInterrupted()) { + break; + } + + final String inputName = dataSource.getName(); + IngestJob ingestJob = new IngestJob(IngestManager.this.ingestJobId.incrementAndGet(), dataSource, moduleTemplates, processUnallocatedSpace); + List errors = ingestJob.startUpIngestPipelines(); + if (!errors.isEmpty()) { + StringBuilder failedModules = new StringBuilder(); + for (int i = 0; i < errors.size(); ++i) { + IngestModuleError error = errors.get(i); + String moduleName = error.getModuleDisplayName(); + logger.log(Level.SEVERE, "The " + moduleName + " module failed to start up", error.getModuleError()); + failedModules.append(moduleName); + if ((errors.size() > 1) && (i != (errors.size() - 1))) { + failedModules.append(","); + } + } + MessageNotifyUtil.Message.error( // RJCTODO: Fix this to show all errors + "Failed to start the following ingest modules: " + failedModules.toString() + " .\n\n" + + "No ingest modules will be run. Please disable the module " + + "or fix the error and restart ingest by right clicking on " + + "the data source and selecting Run Ingest Modules.\n\n" + + "Error: " + errors.get(0).getModuleError().getMessage()); + break; + } + + // Save the ingest job for later cleanup of pipelines. + ingestJobs.put(ingestJob.getId(), ingestJob); + + // Queue the data source ingest tasks for the ingest job. + progress.progress("DataSource Ingest" + " " + inputName, processed); + scheduler.getDataSourceScheduler().schedule(ingestJob); + progress.progress("DataSource Ingest" + " " + inputName, ++processed); + + // Queue the file ingest tasks for the ingest job. + progress.progress("File Ingest" + " " + inputName, processed); + scheduler.getFileScheduler().scheduleIngestOfFiles(ingestJob); + progress.progress("File Ingest" + " " + inputName, ++processed); } - // Save the ingest job for later cleanup of pipelines. - ingestJobs.put(ingestJob.getId(), ingestJob); - - // Queue the data source ingest tasks for the ingest job. - progress.progress("DataSource Ingest" + " " + inputName, processed); - scheduler.getDataSourceScheduler().schedule(ingestJob); - progress.progress("DataSource Ingest" + " " + inputName, ++processed); - - // Queue the file ingest tasks for the ingest job. - progress.progress("File Ingest" + " " + inputName, processed); - scheduler.getFileScheduler().scheduleIngestOfFiles(ingestJob); - progress.progress("File Ingest" + " " + inputName, ++processed); - } - - return null; - } - - @Override - protected void done() { - try { - super.get(); - } catch (CancellationException | InterruptedException ex) { + if (!Thread.currentThread().isInterrupted()) { + startIngestTasks(); + } } catch (Exception ex) { - logger.log(Level.SEVERE, "Error while scheduling ingest jobs", ex); - MessageNotifyUtil.Message.error("An error occurred while starting ingest. Results may only be partial"); + // RJCTODO: } finally { - if (!isCancelled()) { - startAll(); - } + // RJCTODO: Release progress.finish(); - finished = true; } } @@ -553,103 +494,71 @@ public class IngestManager { } } - /** - * Performs data source ingest tasks for one or more ingest jobs on a worker - * thread. - */ - class DataSourceTaskWorker extends SwingWorker { + private class RunDataSourceIngestModulesTask implements Runnable { private final long id; - private volatile boolean finished = false; - DataSourceTaskWorker(long threadId) { + RunDataSourceIngestModulesTask(long threadId) { this.id = threadId; } @Override - protected Void doInBackground() throws Exception { - IngestScheduler.DataSourceScheduler scheduler = IngestScheduler.getInstance().getDataSourceScheduler(); - while (scheduler.hasNext()) { - if (isCancelled()) { - return null; - } - IngestJob job = scheduler.next(); - DataSourceIngestPipeline pipeline = job.getDataSourceIngestPipelineForThread(this.id); - pipeline.process(this, job.getDataSourceTaskProgressBar()); - if (isCancelled()) { - return null; - } - } - return null; - } - - @Override - protected void done() { + public void run() { try { - super.get(); - } catch (CancellationException | InterruptedException e) { + IngestScheduler.DataSourceScheduler scheduler = IngestScheduler.getInstance().getDataSourceScheduler(); + while (scheduler.hasNext()) { + if (Thread.currentThread().isInterrupted()) { + break; + } + IngestJob job = scheduler.next(); + DataSourceIngestPipeline pipeline = job.getDataSourceIngestPipelineForThread(this.id); + pipeline.process(job.getDataSourceTaskProgressBar()); + } } catch (Exception ex) { - String message = String.format("Data source ingest thread (id=%d) experienced a fatal error", this.id); + String message = String.format("Data source ingest thread (id=%d) caught exception", this.id); // RJCTODO logger.log(Level.SEVERE, message, ex); } finally { - IngestManager.getInstance().reportThreadDone(this.id); - finished = true; + IngestManager.getInstance().reportIngestTaskDone(this.id); } } - - boolean isFinished() { - return finished; - } } - /** - * Performs file ingest tasks for one or more ingest jobs on a worker - * thread. - */ - class FileTaskWorker extends SwingWorker { + private class RunFileSourceIngestModulesTask implements Runnable { private final long id; - private volatile boolean finished = false; - FileTaskWorker(long threadId) { - this.id = threadId; + RunFileSourceIngestModulesTask(long taskId) { + this.id = taskId; } @Override - protected Object doInBackground() throws Exception { - IngestScheduler.FileScheduler fileScheduler = IngestScheduler.getInstance().getFileScheduler(); - while (fileScheduler.hasNext()) { - if (isCancelled()) { - return null; - } - IngestScheduler.FileScheduler.FileTask task = fileScheduler.next(); - IngestJob job = task.getJob(); - FileIngestPipeline pipeline = job.getFileIngestPipelineForThread(this.id); - job.handleFileTaskStarted(task); - pipeline.process(task.getFile()); - if (isCancelled()) { - return null; - } - } - return null; - } - - @Override - protected void done() { + public void run() { try { - super.get(); - } catch (CancellationException | InterruptedException e) { + IngestScheduler.FileScheduler fileScheduler = IngestScheduler.getInstance().getFileScheduler(); + while (fileScheduler.hasNext()) { + if (Thread.currentThread().isInterrupted()) { + break; + } + IngestScheduler.FileScheduler.FileTask task = fileScheduler.next(); + IngestJob job = task.getJob(); + FileIngestPipeline pipeline = job.getFileIngestPipelineForThread(this.id); + job.handleFileTaskStarted(task); + pipeline.process(task.getFile()); + } } catch (Exception ex) { - String message = String.format("File ingest thread {0} experienced a fatal error", this.id); + String message = String.format("Data source ingest thread (id=%d) caught exception", this.id); // RJCTODO logger.log(Level.SEVERE, message, ex); } finally { - IngestManager.getInstance().reportThreadDone(this.id); - finished = true; + IngestManager.getInstance().reportIngestTaskDone(this.id); } } - - boolean isFinished() { - return finished; - } } + + class IngestCancellationTask implements Runnable { + + @Override + public void run() { + // RJCTODO: Run + } + } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestMessageTopComponent.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestMessageTopComponent.java index d8971c476e..f05de82635 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestMessageTopComponent.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestMessageTopComponent.java @@ -224,7 +224,7 @@ import org.sleuthkit.datamodel.Content; manager = IngestManager.getInstance(); } try { - manager.stopAll(); + manager.cancelIngestTasks(); } finally { //clear inbox clearMessages(); diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestMonitor.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestMonitor.java index dde9704379..3ec81082dd 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestMonitor.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestMonitor.java @@ -163,9 +163,9 @@ public final class IngestMonitor { if (checkDiskSpace() == false) { //stop ingest if running final String diskPath = root.getAbsolutePath(); - MONITOR_LOGGER.log(Level.SEVERE, "Stopping ingest due to low disk space on disk " + diskPath); - logger.log(Level.SEVERE, "Stopping ingest due to low disk space on disk " + diskPath); - manager.stopAll(); + MONITOR_LOGGER.log(Level.SEVERE, "Stopping ingest due to low disk space on disk {0}", diskPath); + logger.log(Level.SEVERE, "Stopping ingest due to low disk space on disk {0}", diskPath); + manager.cancelIngestTasks(); IngestServices.getInstance().postMessage(IngestMessage.createManagerErrorMessage( NbBundle.getMessage(this.getClass(), "IngestMonitor.mgrErrMsg.lowDiskSpace.title", diskPath), NbBundle.getMessage(this.getClass(), "IngestMonitor.mgrErrMsg.lowDiskSpace.msg", diskPath))); From 0b471112c919f7813aaf31b9ebc2db5740021125 Mon Sep 17 00:00:00 2001 From: alexjacks92 Date: Mon, 7 Apr 2014 13:35:52 -0400 Subject: [PATCH 07/15] Removing old diff files before moving new ones in. --- test/script/regression.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/script/regression.py b/test/script/regression.py index a80e301383..de7e48896b 100755 --- a/test/script/regression.py +++ b/test/script/regression.py @@ -1822,6 +1822,12 @@ def setupAttachments(attachments, test_config): for file in attachments: filename = ntpath.basename(file) destination = os.path.join(test_config.diff_dir, filename) + + # removing old files + call = ['rm', str(test_config.diff_dir) + '*.txt'] + print("deleting file with command: " + call) + subprocess.call(call) + call = ['cp', file, destination] subprocess.call(call) From f7f0c46697d5f5036eb9f66218e1166839e1b573 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Mon, 7 Apr 2014 21:20:22 -0400 Subject: [PATCH 08/15] Thread pool shutdown variation --- .../sleuthkit/autopsy/ingest/IngestJob.java | 4 +- .../autopsy/ingest/IngestManager.java | 388 +++++++++--------- .../ingest/IngestMessageTopComponent.java | 2 +- .../autopsy/ingest/IngestMonitor.java | 2 +- 4 files changed, 204 insertions(+), 192 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java index 80f75818d6..57e0f4e16a 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java @@ -85,7 +85,7 @@ final class IngestJob { "IngestJob.progress.cancelling", displayName)); } - IngestManager.getInstance().cancelIngestTasks(); + IngestManager.getInstance().cancelIngestJobs(); return true; } }); @@ -104,7 +104,7 @@ final class IngestJob { NbBundle.getMessage(this.getClass(), "IngestJob.progress.cancelling", displayName)); } - IngestManager.getInstance().cancelIngestTasks(); + IngestManager.getInstance().cancelIngestJobs(); return true; } }); diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java index 44286f87dc..5f1e746675 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java @@ -25,7 +25,6 @@ import java.util.HashMap; import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import java.util.logging.Level; @@ -39,6 +38,7 @@ import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.Content; import java.util.prefs.Preferences; +import javax.swing.SwingWorker; /** * Manages the execution of ingest jobs. @@ -53,6 +53,7 @@ public class IngestManager { private static final PropertyChangeSupport pcs = new PropertyChangeSupport(IngestManager.class); private static final Preferences userPreferences = NbPreferences.forModule(IngestManager.class); private static IngestManager instance; + private final IngestJobsManager jobsManager = new IngestJobsManager(); private final IngestScheduler scheduler = IngestScheduler.getInstance(); private final IngestMonitor ingestMonitor = new IngestMonitor(); private ExecutorService startIngestJobsExecutor = null; @@ -60,9 +61,6 @@ public class IngestManager { private ExecutorService fileIngestTasksExecutor = null; private AtomicLong ingestJobId = new AtomicLong(0L); private AtomicLong ingestTaskId = new AtomicLong(0L); - private final HashMap ingestJobs = new HashMap<>(); // Maps job ids to jobs - private final HashMap> ingestTasks = new HashMap<>(); // Maps task ids to Runnable tasks -// private TaskSchedulingWorker taskSchedulingWorker = null; private volatile IngestUI ingestMessageBox; /** @@ -81,33 +79,6 @@ public class IngestManager { createThreadPools(); } - private synchronized void createThreadPools() { - if (startIngestJobsExecutor == null) { - startIngestJobsExecutor = Executors.newSingleThreadExecutor(); - } - if (dataSourceIngestTasksExecutor == null) { - dataSourceIngestTasksExecutor = Executors.newSingleThreadExecutor(); - } - if (fileIngestTasksExecutor == null) { - fileIngestTasksExecutor = Executors.newFixedThreadPool(MAX_NUMBER_OF_FILE_INGEST_THREADS); - } - } - - public synchronized static int getNumberOfFileIngestThreads() { - return userPreferences.getInt(NUMBER_OF_FILE_INGEST_THREADS_KEY, DEFAULT_NUMBER_OF_FILE_INGEST_THREADS); - } - - public static void setNumberOfFileIngestThreads(int numberOfThreads) { - if (numberOfThreads < MIN_NUMBER_OF_FILE_INGEST_THREADS - || numberOfThreads > MAX_NUMBER_OF_FILE_INGEST_THREADS) { - numberOfThreads = DEFAULT_NUMBER_OF_FILE_INGEST_THREADS; - } - - synchronized (IngestManager.class) { - userPreferences.putInt(NUMBER_OF_FILE_INGEST_THREADS_KEY, numberOfThreads); - } - } - /** * Finds the ingest messages in box TopComponent. Called by the custom * installer for this package once the window system is initialized. @@ -118,6 +89,53 @@ public class IngestManager { } } + public synchronized static int getNumberOfFileIngestThreads() { + return userPreferences.getInt(NUMBER_OF_FILE_INGEST_THREADS_KEY, DEFAULT_NUMBER_OF_FILE_INGEST_THREADS); + } + + public synchronized static void setNumberOfFileIngestThreads(int numberOfThreads) { + if (numberOfThreads < MIN_NUMBER_OF_FILE_INGEST_THREADS + || numberOfThreads > MAX_NUMBER_OF_FILE_INGEST_THREADS) { + numberOfThreads = DEFAULT_NUMBER_OF_FILE_INGEST_THREADS; + } + + userPreferences.putInt(NUMBER_OF_FILE_INGEST_THREADS_KEY, numberOfThreads); + } + + synchronized void startIngestJobs(final List dataSources, final List moduleTemplates, boolean processUnallocatedSpace) { + if (!isIngestRunning() && ingestMessageBox != null) { + ingestMessageBox.clearMessages(); + } + + createThreadPools(); + startIngestJobsExecutor.submit(new StartIngestJobsTask(ingestTaskId.incrementAndGet(), dataSources, moduleTemplates, processUnallocatedSpace)); + + if (ingestMessageBox != null) { + ingestMessageBox.restoreMessages(); + } + } + + /** + * Test if any ingest jobs are in progress. + * + * @return True if any ingest jobs are in progress, false otherwise + */ + public boolean isIngestRunning() { + return jobsManager.hasJobs(); + } + + synchronized void addFileToIngestJob(long ingestJobId, AbstractFile file) { + if (!jobsManager.addFileToIngestJob(ingestJobId, file)) { + MessageNotifyUtil.Notify.show(NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr"), + "Unable to associate " + file.getName() + " with ingest job, file will not be processed by ingest nodules", + MessageNotifyUtil.MessageType.ERROR); + } + } + + synchronized void cancelIngestJobs() { + new IngestCancellationWorker(1, TimeUnit.SECONDS).execute(); + } + /** * Ingest events. */ @@ -172,15 +190,15 @@ public class IngestManager { * * @param listener PropertyChangeListener to register */ - synchronized public static void addPropertyChangeListener(final PropertyChangeListener listener) { + public static void addPropertyChangeListener(final PropertyChangeListener listener) { pcs.addPropertyChangeListener(listener); } - synchronized public static void removePropertyChangeListener(final PropertyChangeListener listener) { + public static void removePropertyChangeListener(final PropertyChangeListener listener) { pcs.removePropertyChangeListener(listener); } - synchronized static void fireModuleEvent(String eventType, String moduleName) { + static void fireModuleEvent(String eventType, String moduleName) { try { pcs.firePropertyChange(eventType, moduleName, null); } catch (Exception e) { @@ -196,7 +214,7 @@ public class IngestManager { * * @param objId ID of file that is done */ - synchronized static void fireFileDone(long objId) { + static void fireFileDone(long objId) { try { pcs.firePropertyChange(IngestEvent.FILE_DONE.toString(), objId, null); } catch (Exception e) { @@ -213,7 +231,7 @@ public class IngestManager { * * @param moduleDataEvent */ - synchronized static void fireModuleDataEvent(ModuleDataEvent moduleDataEvent) { + static void fireModuleDataEvent(ModuleDataEvent moduleDataEvent) { try { pcs.firePropertyChange(IngestEvent.DATA.toString(), moduleDataEvent, null); } catch (Exception e) { @@ -230,7 +248,7 @@ public class IngestManager { * * @param moduleContentEvent */ - synchronized static void fireModuleContentEvent(ModuleContentEvent moduleContentEvent) { + static void fireModuleContentEvent(ModuleContentEvent moduleContentEvent) { try { pcs.firePropertyChange(IngestEvent.CONTENT_CHANGED.toString(), moduleContentEvent, null); } catch (Exception e) { @@ -241,140 +259,6 @@ public class IngestManager { } } - synchronized void startIngestJob(final Content dataSource, final List moduleTemplates, boolean processUnallocatedSpace) { - List dataSources = new ArrayList<>(); - dataSources.add(dataSource); - startIngestJobs(dataSources, moduleTemplates, processUnallocatedSpace); - } - - synchronized void startIngestJobs(final List dataSources, final List moduleTemplates, boolean processUnallocatedSpace) { - if (!isIngestRunning() && ingestMessageBox != null) { - ingestMessageBox.clearMessages(); - } - - createThreadPools(); - - long taskId = ingestTaskId.incrementAndGet(); - Future dataSourceIngestTask = startIngestJobsExecutor.submit(new StartIngestJobsTask(taskId, dataSources, moduleTemplates, processUnallocatedSpace)); - ingestTasks.put(taskId, dataSourceIngestTask); - - if (ingestMessageBox != null) { - ingestMessageBox.restoreMessages(); - } - } - - synchronized void addFileToIngestJob(long ingestJobId, AbstractFile file) { - IngestJob job = this.ingestJobs.get(ingestJobId); - if (job != null) { - scheduler.getFileScheduler().scheduleFile(job, file); - } else { - logger.log(Level.SEVERE, "Unable to map ingest job id (id = {0}) to an ingest job, failed to schedule file (id = {1})", new Object[]{ingestJobId, file.getId()}); - MessageNotifyUtil.Notify.show(NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr"), - "Unable to associate " + file.getName() + " with ingest job, file will not be processed by ingest nodules", - MessageNotifyUtil.MessageType.ERROR); - } - } - - private synchronized void startIngestTasks() { - if (!ingestMonitor.isRunning()) { - ingestMonitor.start(); - } - - long taskId = ingestTaskId.incrementAndGet(); - Future dataSourceIngestTask = dataSourceIngestTasksExecutor.submit(new RunDataSourceIngestModulesTask(taskId)); - ingestTasks.put(taskId, dataSourceIngestTask); - - int workersRequested = getNumberOfFileIngestThreads(); - for (int i = 0; i < workersRequested; ++i) { - taskId = ingestTaskId.incrementAndGet(); - Future fileIngestTask = fileIngestTasksExecutor.submit(new RunFileSourceIngestModulesTask(taskId)); - ingestTasks.put(taskId, fileIngestTask); - } - } - - synchronized void reportIngestTaskDone(long taskId) { - ingestTasks.remove(taskId); - - List completedJobs = new ArrayList<>(); - for (IngestJob job : ingestJobs.values()) { - job.releaseIngestPipelinesForThread(taskId); - if (job.areIngestPipelinesShutDown()) { - completedJobs.add(job.getId()); - } - } - - for (Long jobId : completedJobs) { - ingestJobs.remove(jobId); - } - } - - synchronized void cancelIngestTasks() { -// synchronized void cancelIngestTasks(int waitTime, TimeUnit unit) { // RJCTODO - // First get the task scheduling worker to stop adding new tasks. -// boolean res = shutDownThreadPool(startIngestJobsExecutor, 1, TimeUnit.SECONDS); -// startIngestJobsExecutor = null; - - // Now mark all of the ingest jobs as cancelled. This way the ingest - // modules will know they are being shut down due to cancellation when - // the cancelled ingest workers release their pipelines. - for (IngestJob job : ingestJobs.values()) { - job.cancel(); - } - - // Jettision the remaining data tasks. This will dispose of any tasks that - // the scheduling worker queued up before it was cancelled. -// scheduler.getFileScheduler().empty(); -// scheduler.getDataSourceScheduler().empty(); - - // Cancel all of the ingest module running tasks. - for (Future task : ingestTasks.values()) { - task.cancel(true); - } - -// res = shutDownThreadPool(dataSourceIngestTasksExecutor, 30, TimeUnit.SECONDS); -// dataSourceIngestTasksExecutor = null; -// res = shutDownThreadPool(fileIngestTasksExecutor, 30, TimeUnit.SECONDS); -// fileIngestTasksExecutor = null; - - // Jettision the remaining tasks again to try to dispose of any tasks - // queued up task workers before they were cancelled. -// scheduler.getFileScheduler().empty(); -// scheduler.getDataSourceScheduler().empty(); - } - - // The following method implementation is adapted from: - // http://docs.oracle.com/javase/7/docs/api/java/util/concurrent/ExecutorService.html#awaitTermination(long, java.util.concurrent.TimeUnit) - private boolean shutDownThreadPool(ExecutorService pool, int waitTime, TimeUnit unit) { - boolean succeeded = true; - // Prevent submission of new tasks. - pool.shutdown(); - try { - // Wait a while for existing tasks to terminate. - if (!pool.awaitTermination(1, TimeUnit.SECONDS)) { - // Cancel currently executing tasks. - pool.shutdownNow(); - if (!pool.awaitTermination(waitTime, unit)) { - succeeded = false; - } - } - } catch (InterruptedException ex) { - // (Re-)Cancel if current thread also interrupted. - pool.shutdownNow(); - // Preserve interrupt status. - Thread.currentThread().interrupt(); - } - return succeeded; - } - - /** - * Test if any ingest jobs are in progress. - * - * @return True if any ingest jobs are in progress, false otherwise - */ - public synchronized boolean isIngestRunning() { - return (ingestJobs.isEmpty() == false); - } - /** * Module publishes message using InegestManager handle Does not block. The * message gets enqueued in the GUI thread and displayed in a widget @@ -403,6 +287,103 @@ public class IngestManager { } } + private synchronized void startIngestTasks() { + if (!ingestMonitor.isRunning()) { + ingestMonitor.start(); + } + + dataSourceIngestTasksExecutor.submit(new RunDataSourceIngestModulesTask(ingestTaskId.incrementAndGet())); + + int numberOfFileTasksRequested = getNumberOfFileIngestThreads(); + for (int i = 0; i < numberOfFileTasksRequested; ++i) { + fileIngestTasksExecutor.submit(new RunFileSourceIngestModulesTask(ingestTaskId.incrementAndGet())); + } + } + + private synchronized void createThreadPools() { + if (startIngestJobsExecutor == null) { + startIngestJobsExecutor = Executors.newSingleThreadExecutor(); + } + if (dataSourceIngestTasksExecutor == null) { + dataSourceIngestTasksExecutor = Executors.newSingleThreadExecutor(); + } + if (fileIngestTasksExecutor == null) { + fileIngestTasksExecutor = Executors.newFixedThreadPool(MAX_NUMBER_OF_FILE_INGEST_THREADS); + } + } + + private synchronized boolean shutDownThreadPools(int timeOut, TimeUnit timeOutUnits) { + boolean success = true; + if (!shutDownThreadPool(startIngestJobsExecutor, timeOut, timeOutUnits)) { + success = false; + } + if (!shutDownThreadPool(dataSourceIngestTasksExecutor, timeOut, timeOutUnits)) { + success = false; + } + if (!shutDownThreadPool(fileIngestTasksExecutor, timeOut, timeOutUnits)) { + success = false; + } + startIngestJobsExecutor = null; + dataSourceIngestTasksExecutor = null; + fileIngestTasksExecutor = null; + return success; + } + + private boolean shutDownThreadPool(ExecutorService pool, int waitTime, TimeUnit unit) { + try { + pool.shutdownNow(); + return pool.awaitTermination(waitTime, unit); + } catch (InterruptedException ex) { + pool.shutdownNow(); + Thread.currentThread().interrupt(); // Preserve interrupted status. + return false; + } + } + + private class IngestJobsManager { + + private final HashMap jobs = new HashMap<>(); // Maps job ids to jobs + + synchronized void addJob(IngestJob job) { + jobs.put(job.getId(), job); + } + + synchronized boolean hasJobs() { + return (jobs.isEmpty() == false); + } + + synchronized boolean addFileToIngestJob(long ingestJobId, AbstractFile file) { + IngestJob job = jobs.get(ingestJobId); + if (job != null) { + scheduler.getFileScheduler().scheduleFile(job, file); + return true; + } else { + logger.log(Level.SEVERE, "Unable to map ingest job id (id = {0}) to an ingest job, failed to add file (id = {1})", new Object[]{ingestJobId, file.getId()}); + return false; + } + } + + synchronized void cancelJobs() { + for (IngestJob job : jobs.values()) { + job.cancel(); + } + } + + synchronized void reportIngestTaskDone(long taskId) { + List completedJobs = new ArrayList<>(); + for (IngestJob job : jobs.values()) { + job.releaseIngestPipelinesForThread(taskId); + if (job.areIngestPipelinesShutDown() == true) { + completedJobs.add(job.getId()); + } + } + + for (Long jobId : completedJobs) { + jobs.remove(jobId); + } + } + } + private class StartIngestJobsTask implements Runnable { private final long id; @@ -429,7 +410,7 @@ public class IngestManager { if (progress != null) { progress.setDisplayName(displayName + " (Cancelling...)"); } - IngestManager.getInstance().cancelIngestTasks(); + IngestManager.getInstance().cancelIngestJobs(); return true; } }); @@ -465,7 +446,7 @@ public class IngestManager { } // Save the ingest job for later cleanup of pipelines. - ingestJobs.put(ingestJob.getId(), ingestJob); + jobsManager.addJob(ingestJob); // Queue the data source ingest tasks for the ingest job. progress.progress("DataSource Ingest" + " " + inputName, processed); @@ -482,9 +463,9 @@ public class IngestManager { startIngestTasks(); } } catch (Exception ex) { - // RJCTODO: + String message = String.format("StartIngestJobsTask (id=%d) caught exception", id); + logger.log(Level.SEVERE, message, ex); } finally { - // RJCTODO: Release progress.finish(); } } @@ -498,8 +479,8 @@ public class IngestManager { private final long id; - RunDataSourceIngestModulesTask(long threadId) { - this.id = threadId; + RunDataSourceIngestModulesTask(long taskId) { + id = taskId; } @Override @@ -511,14 +492,14 @@ public class IngestManager { break; } IngestJob job = scheduler.next(); - DataSourceIngestPipeline pipeline = job.getDataSourceIngestPipelineForThread(this.id); + DataSourceIngestPipeline pipeline = job.getDataSourceIngestPipelineForThread(id); pipeline.process(job.getDataSourceTaskProgressBar()); } } catch (Exception ex) { - String message = String.format("Data source ingest thread (id=%d) caught exception", this.id); // RJCTODO + String message = String.format("RunDataSourceIngestModulesTask (id=%d) caught exception", id); logger.log(Level.SEVERE, message, ex); } finally { - IngestManager.getInstance().reportIngestTaskDone(this.id); + jobsManager.reportIngestTaskDone(id); } } } @@ -528,7 +509,7 @@ public class IngestManager { private final long id; RunFileSourceIngestModulesTask(long taskId) { - this.id = taskId; + id = taskId; } @Override @@ -541,24 +522,55 @@ public class IngestManager { } IngestScheduler.FileScheduler.FileTask task = fileScheduler.next(); IngestJob job = task.getJob(); - FileIngestPipeline pipeline = job.getFileIngestPipelineForThread(this.id); + FileIngestPipeline pipeline = job.getFileIngestPipelineForThread(id); job.handleFileTaskStarted(task); pipeline.process(task.getFile()); } } catch (Exception ex) { - String message = String.format("Data source ingest thread (id=%d) caught exception", this.id); // RJCTODO + String message = String.format("RunFileSourceIngestModulesTask (id=%d) caught exception", id); logger.log(Level.SEVERE, message, ex); } finally { - IngestManager.getInstance().reportIngestTaskDone(this.id); + jobsManager.reportIngestTaskDone(id); } } } - class IngestCancellationTask implements Runnable { + class IngestCancellationWorker extends SwingWorker { + + private final int timeOut; + private final TimeUnit timeOutUnits; + + IngestCancellationWorker(int timeOut, TimeUnit timeOutUnits) { + this.timeOut = timeOut; + this.timeOutUnits = timeOutUnits; + } @Override - public void run() { - // RJCTODO: Run + protected Boolean doInBackground() throws Exception { + // First mark all of the ingest jobs as cancelled. This way the + // ingest modules will know they are being shut down due to + // cancellation when the cancelled run ingest module tasks release + // their pipelines. This also makes sure the lock on the jobs + // manager is released before the run ingest module tasks start + // releasing pipelines. + jobsManager.cancelJobs(); + + // Jettision the remaining data source and file ingest tasks. This + // will could break the the run ingest module tasks out of their + // loops even before the pools mark their threads as interrupted. + scheduler.getFileScheduler().empty(); + scheduler.getDataSourceScheduler().empty(); + + boolean success = shutDownThreadPools(timeOut, timeOutUnits); + + // Jettision data source and file ingest tasks again to try to + // dispose of any tasks that slipped into the queues. + scheduler.getFileScheduler().empty(); + scheduler.getDataSourceScheduler().empty(); + + return success; } - } + + // TODO: Add done() override to notify a listener of success or failure + } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestMessageTopComponent.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestMessageTopComponent.java index f05de82635..b5feb6220c 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestMessageTopComponent.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestMessageTopComponent.java @@ -224,7 +224,7 @@ import org.sleuthkit.datamodel.Content; manager = IngestManager.getInstance(); } try { - manager.cancelIngestTasks(); + manager.cancelIngestJobs(); } finally { //clear inbox clearMessages(); diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestMonitor.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestMonitor.java index 3ec81082dd..5c311363fd 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestMonitor.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestMonitor.java @@ -165,7 +165,7 @@ public final class IngestMonitor { final String diskPath = root.getAbsolutePath(); MONITOR_LOGGER.log(Level.SEVERE, "Stopping ingest due to low disk space on disk {0}", diskPath); logger.log(Level.SEVERE, "Stopping ingest due to low disk space on disk {0}", diskPath); - manager.cancelIngestTasks(); + manager.cancelIngestJobs(); IngestServices.getInstance().postMessage(IngestMessage.createManagerErrorMessage( NbBundle.getMessage(this.getClass(), "IngestMonitor.mgrErrMsg.lowDiskSpace.title", diskPath), NbBundle.getMessage(this.getClass(), "IngestMonitor.mgrErrMsg.lowDiskSpace.msg", diskPath))); From cb6b9f7f5bcd0646f2f16e52cbc1bf41e0115559 Mon Sep 17 00:00:00 2001 From: alexjacks92 Date: Tue, 8 Apr 2014 09:59:28 -0400 Subject: [PATCH 09/15] Delete existing diff files before adding new ones. --- test/script/regression.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/test/script/regression.py b/test/script/regression.py index de7e48896b..b4afcbd5a9 100755 --- a/test/script/regression.py +++ b/test/script/regression.py @@ -1818,16 +1818,19 @@ def setupAttachments(attachments, test_config): attachments: a listof_String, the files to be moved test_config: TestConfiguration, used to determine where to move the files to """ - + call = ['pwd'] + subprocess.call(call) + + # remove old diff files + filelist = [f for f in os.listdir(test_config.diff_dir) if (f.endswith(".txt") or f.endswith(".html"))] + for f in filelist: + if os.path.isfile(test_config.diff_dir + "/" + f): + os.remove(test_config.diff_dir + "/" + f) + + # move in the new diff files for file in attachments: filename = ntpath.basename(file) destination = os.path.join(test_config.diff_dir, filename) - - # removing old files - call = ['rm', str(test_config.diff_dir) + '*.txt'] - print("deleting file with command: " + call) - subprocess.call(call) - call = ['cp', file, destination] subprocess.call(call) From f26073ee136534e842fe7219725c4deb3dec93b4 Mon Sep 17 00:00:00 2001 From: alexjacks92 Date: Tue, 8 Apr 2014 12:10:11 -0400 Subject: [PATCH 10/15] Cleaning up code to work when you aren't running a jenkins job. --- test/script/regression.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/script/regression.py b/test/script/regression.py index b4afcbd5a9..7c33e62d33 100755 --- a/test/script/regression.py +++ b/test/script/regression.py @@ -348,7 +348,8 @@ class TestRunner(object): test_data.ant.append("-Dgold_path=" + test_config.gold) test_data.ant.append("-Dout_path=" + make_local_path(test_data.output_path)) - test_data.ant.append("-Ddiff_dir="+ test_config.diff_dir) + if test_config.jenkins: + test_data.ant.append("-Ddiff_dir="+ test_config.diff_dir) test_data.ant.append("-Dignore_unalloc=" + "%s" % test_config.args.unallocated) test_data.ant.append("-Dtest.timeout=" + str(test_config.timeout)) From 36c9044576f3b567c931763503d7fea3dcd1914a Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Tue, 8 Apr 2014 16:02:33 -0400 Subject: [PATCH 11/15] Fixed ingest job cancellation --- .../DataSourceIngestModuleStatusHelper.java | 21 +- .../ingest/DataSourceIngestPipeline.java | 19 +- .../autopsy/ingest/FileIngestPipeline.java | 11 +- .../sleuthkit/autopsy/ingest/IngestJob.java | 32 ++- .../autopsy/ingest/IngestManager.java | 220 +++++++----------- 5 files changed, 116 insertions(+), 187 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleStatusHelper.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleStatusHelper.java index 4779f59c63..33474e8587 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleStatusHelper.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleStatusHelper.java @@ -18,7 +18,6 @@ */ package org.sleuthkit.autopsy.ingest; -import javax.swing.SwingWorker; import org.netbeans.api.progress.ProgressHandle; /** @@ -27,11 +26,11 @@ import org.netbeans.api.progress.ProgressHandle; */ public class DataSourceIngestModuleStatusHelper { - private final ProgressHandle progress; + private final IngestJob ingestJob; private final String moduleDisplayName; - DataSourceIngestModuleStatusHelper(ProgressHandle progress, String moduleDisplayName) { - this.progress = progress; + DataSourceIngestModuleStatusHelper(IngestJob ingestJob, String moduleDisplayName) { + this.ingestJob = ingestJob; this.moduleDisplayName = moduleDisplayName; } @@ -43,7 +42,7 @@ public class DataSourceIngestModuleStatusHelper { * @return True if the task has been canceled, false otherwise. */ public boolean isIngestJobCancelled() { - return (Thread.currentThread().isInterrupted()); // RJCTODO: This is not right? Appears to be right... + return (ingestJob.isCancelled()); } /** @@ -55,9 +54,7 @@ public class DataSourceIngestModuleStatusHelper { * data source. */ public void switchToDeterminate(int workUnits) { - if (progress != null) { - progress.switchToDeterminate(workUnits); - } + ingestJob.getDataSourceTaskProgressBar().switchToDeterminate(workUnits); } /** @@ -65,9 +62,7 @@ public class DataSourceIngestModuleStatusHelper { * the total work units to process the data source is unknown. */ public void switchToIndeterminate() { - if (progress != null) { - progress.switchToIndeterminate(); - } + ingestJob.getDataSourceTaskProgressBar().switchToIndeterminate(); } /** @@ -77,8 +72,6 @@ public class DataSourceIngestModuleStatusHelper { * @param workUnits Number of work units performed so far by the module. */ public void progress(int workUnits) { - if (progress != null) { - progress.progress(this.moduleDisplayName, workUnits); - } + ingestJob.getDataSourceTaskProgressBar().progress(this.moduleDisplayName, workUnits); } } \ No newline at end of file diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java index 8a6f07786d..2a135d4c96 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java @@ -22,10 +22,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.logging.Level; -import javax.swing.SwingWorker; import org.netbeans.api.progress.ProgressHandle; -import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.datamodel.Content; /** @@ -34,7 +31,6 @@ import org.sleuthkit.datamodel.Content; */ final class DataSourceIngestPipeline { - private static final Logger logger = Logger.getLogger(DataSourceIngestPipeline.class.getName()); private final IngestJob job; private final List moduleTemplates; private List modules = new ArrayList<>(); @@ -59,7 +55,6 @@ final class DataSourceIngestPipeline { try { module.startUp(context); modulesByClass.put(module.getClassName(), module); - IngestManager.fireModuleEvent(IngestManager.IngestEvent.STARTED.toString(), module.getDisplayName()); } catch (Exception ex) { errors.add(new IngestModuleError(module.getDisplayName(), ex)); } @@ -81,19 +76,14 @@ final class DataSourceIngestPipeline { return errors; } - List process(ProgressHandle progress) { + List process() { List errors = new ArrayList<>(); - Content dataSource = this.job.getDataSource(); - logger.log(Level.INFO, "Processing data source {0}", dataSource.getName()); for (DataSourceIngestModuleDecorator module : this.modules) { try { - module.process(dataSource, new DataSourceIngestModuleStatusHelper(progress, module.getDisplayName())); + module.process(job.getDataSource(), new DataSourceIngestModuleStatusHelper(job, module.getDisplayName())); } catch (Exception ex) { errors.add(new IngestModuleError(module.getDisplayName(), ex)); } - if (job.isCancelled()) { - break; - } } return errors; } @@ -105,8 +95,9 @@ final class DataSourceIngestPipeline { module.shutDown(ingestJobCancelled); } catch (Exception ex) { errors.add(new IngestModuleError(module.getDisplayName(), ex)); - } finally { - IngestManager.fireModuleEvent(IngestManager.IngestEvent.COMPLETED.toString(), module.getDisplayName()); + } + if (job.isCancelled()) { + break; } } return errors; diff --git a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java index 61319f441e..13137318e4 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java @@ -22,10 +22,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.logging.Level; -import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.datamodel.AbstractFile; -import org.sleuthkit.datamodel.Content; /** * A file ingest pipeline composed of a sequence of file ingest modules @@ -33,7 +30,6 @@ import org.sleuthkit.datamodel.Content; */ final class FileIngestPipeline { - private static final Logger logger = Logger.getLogger(FileIngestPipeline.class.getName()); private final IngestJob job; private final List moduleTemplates; private List modules = new ArrayList<>(); @@ -58,7 +54,6 @@ final class FileIngestPipeline { try { module.startUp(context); modulesByClass.put(module.getClassName(), module); - IngestManager.fireModuleEvent(IngestManager.IngestEvent.STARTED.toString(), template.getModuleName()); } catch (Exception ex) { errors.add(new IngestModuleError(module.getDisplayName(), ex)); } @@ -93,7 +88,9 @@ final class FileIngestPipeline { } } file.close(); - IngestManager.fireFileDone(file.getId()); + if (job.isCancelled()) { + IngestManager.fireFileDone(file.getId()); + } return errors; } @@ -104,8 +101,6 @@ final class FileIngestPipeline { module.shutDown(ingestJobCancelled); } catch (Exception ex) { errors.add(new IngestModuleError(module.getDisplayName(), ex)); - } finally { - IngestManager.fireModuleEvent(IngestManager.IngestEvent.COMPLETED.toString(), module.getDisplayName()); } } return errors; diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java index 57e0f4e16a..af5e9b8f3a 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java @@ -46,7 +46,7 @@ final class IngestJob { private ProgressHandle fileTasksProgress; int totalEnqueuedFiles = 0; private int processedFiles = 0; - private boolean cancelled; + private volatile boolean cancelled; IngestJob(long id, Content dataSource, List ingestModuleTemplates, boolean processUnallocatedSpace) { this.id = id; @@ -159,13 +159,13 @@ final class IngestJob { synchronized List releaseIngestPipelinesForThread(long threadId) { List errors = new ArrayList<>(); - + DataSourceIngestPipeline dataSourceIngestPipeline = dataSourceIngestPipelines.get(threadId); if (dataSourceIngestPipeline != null) { errors.addAll(dataSourceIngestPipeline.shutDown(cancelled)); + dataSourceIngestPipelines.remove(threadId); } - dataSourceIngestPipelines.remove(threadId); - if (dataSourceIngestPipelines.isEmpty() && dataSourceTaskProgress != null) { + if (initialDataSourceIngestPipeline == null && dataSourceIngestPipelines.isEmpty() && dataSourceTaskProgress != null) { dataSourceTaskProgress.finish(); dataSourceTaskProgress = null; } @@ -173,9 +173,9 @@ final class IngestJob { FileIngestPipeline fileIngestPipeline = fileIngestPipelines.get(threadId); if (fileIngestPipeline != null) { errors.addAll(fileIngestPipeline.shutDown(cancelled)); + fileIngestPipelines.remove(threadId); } - fileIngestPipelines.remove(threadId); - if (fileIngestPipelines.isEmpty() && fileTasksProgress != null) { + if (initialFileIngestPipeline == null && fileIngestPipelines.isEmpty() && fileTasksProgress != null) { fileTasksProgress.finish(); fileTasksProgress = null; } @@ -184,14 +184,17 @@ final class IngestJob { } synchronized boolean areIngestPipelinesShutDown() { - return (dataSourceIngestPipelines.isEmpty() && fileIngestPipelines.isEmpty()); + return (initialDataSourceIngestPipeline == null + && dataSourceIngestPipelines.isEmpty() + && initialFileIngestPipeline == null + && fileIngestPipelines.isEmpty()); } synchronized ProgressHandle getDataSourceTaskProgressBar() { return this.dataSourceTaskProgress; } - synchronized void handleFileTaskStarted(IngestScheduler.FileScheduler.FileTask task) { + synchronized void updateFileTasksProgressBar(String currentFileName) { int newTotalEnqueuedFiles = fileScheduler.getFilesEnqueuedEst(); if (newTotalEnqueuedFiles > totalEnqueuedFiles) { totalEnqueuedFiles = newTotalEnqueuedFiles + 1; @@ -202,14 +205,23 @@ final class IngestJob { ++processedFiles; } - fileTasksProgress.progress(task.getFile().getName(), processedFiles); + fileTasksProgress.progress(currentFileName, processedFiles); } synchronized void cancel() { + if (initialDataSourceIngestPipeline != null) { + initialDataSourceIngestPipeline.shutDown(true); + initialDataSourceIngestPipeline = null; + } + if (initialFileIngestPipeline != null) { + initialFileIngestPipeline.shutDown(true); + initialFileIngestPipeline = null; + } + cancelled = true; } - synchronized boolean isCancelled() { + boolean isCancelled() { return cancelled; } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java index 5f1e746675..59f0bf7789 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java @@ -23,9 +23,10 @@ import java.beans.PropertyChangeSupport; import java.util.ArrayList; import java.util.HashMap; import java.util.List; +import java.util.concurrent.CancellationException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; +import java.util.concurrent.Future; import java.util.concurrent.atomic.AtomicLong; import java.util.logging.Level; import org.openide.util.NbBundle; @@ -53,12 +54,13 @@ public class IngestManager { private static final PropertyChangeSupport pcs = new PropertyChangeSupport(IngestManager.class); private static final Preferences userPreferences = NbPreferences.forModule(IngestManager.class); private static IngestManager instance; - private final IngestJobsManager jobsManager = new IngestJobsManager(); private final IngestScheduler scheduler = IngestScheduler.getInstance(); private final IngestMonitor ingestMonitor = new IngestMonitor(); - private ExecutorService startIngestJobsExecutor = null; - private ExecutorService dataSourceIngestTasksExecutor = null; - private ExecutorService fileIngestTasksExecutor = null; + private final ExecutorService startIngestJobsExecutor = Executors.newSingleThreadExecutor(); + private final ExecutorService dataSourceIngestTasksExecutor = Executors.newSingleThreadExecutor(); + private final ExecutorService fileIngestTasksExecutor = Executors.newFixedThreadPool(MAX_NUMBER_OF_FILE_INGEST_THREADS); + private final HashMap ingestJobs = new HashMap<>(); // Maps job ids to jobs + private final HashMap> ingestTasks = new HashMap<>(); // Maps task ids to task cancellation handles private AtomicLong ingestJobId = new AtomicLong(0L); private AtomicLong ingestTaskId = new AtomicLong(0L); private volatile IngestUI ingestMessageBox; @@ -76,12 +78,11 @@ public class IngestManager { } private IngestManager() { - createThreadPools(); } /** - * Finds the ingest messages in box TopComponent. Called by the custom - * installer for this package once the window system is initialized. + * Finds the top component for the ingest messages in box. Called by the + * custom installer for this package once the window system is initialized. */ void initIngestMessageInbox() { if (this.ingestMessageBox == null) { @@ -107,8 +108,9 @@ public class IngestManager { ingestMessageBox.clearMessages(); } - createThreadPools(); - startIngestJobsExecutor.submit(new StartIngestJobsTask(ingestTaskId.incrementAndGet(), dataSources, moduleTemplates, processUnallocatedSpace)); + long taskId = ingestTaskId.incrementAndGet(); + Future task = startIngestJobsExecutor.submit(new StartIngestJobsTask(taskId, dataSources, moduleTemplates, processUnallocatedSpace)); + ingestTasks.put(taskId, task); if (ingestMessageBox != null) { ingestMessageBox.restoreMessages(); @@ -121,19 +123,18 @@ public class IngestManager { * @return True if any ingest jobs are in progress, false otherwise */ public boolean isIngestRunning() { - return jobsManager.hasJobs(); + return (ingestJobs.isEmpty() == false); } synchronized void addFileToIngestJob(long ingestJobId, AbstractFile file) { - if (!jobsManager.addFileToIngestJob(ingestJobId, file)) { - MessageNotifyUtil.Notify.show(NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr"), - "Unable to associate " + file.getName() + " with ingest job, file will not be processed by ingest nodules", - MessageNotifyUtil.MessageType.ERROR); + IngestJob job = ingestJobs.get(ingestJobId); + if (job != null) { + scheduler.getFileScheduler().scheduleFile(job, file); } } synchronized void cancelIngestJobs() { - new IngestCancellationWorker(1, TimeUnit.SECONDS).execute(); + new IngestCancellationWorker().execute(); } /** @@ -141,7 +142,6 @@ public class IngestManager { */ public enum IngestEvent { - // RJCTODO: Update comments /** * Event sent when an ingest module has been started. Second argument of * the property change is a string form of the module name and the third @@ -292,95 +292,55 @@ public class IngestManager { ingestMonitor.start(); } - dataSourceIngestTasksExecutor.submit(new RunDataSourceIngestModulesTask(ingestTaskId.incrementAndGet())); - + long taskId = ingestTaskId.incrementAndGet(); + Future task = dataSourceIngestTasksExecutor.submit(new RunDataSourceIngestModulesTask(taskId)); + ingestTasks.put(taskId, task); + int numberOfFileTasksRequested = getNumberOfFileIngestThreads(); for (int i = 0; i < numberOfFileTasksRequested; ++i) { - fileIngestTasksExecutor.submit(new RunFileSourceIngestModulesTask(ingestTaskId.incrementAndGet())); + taskId = ingestTaskId.incrementAndGet(); + task = fileIngestTasksExecutor.submit(new RunFileSourceIngestModulesTask(taskId)); + ingestTasks.put(taskId, task); } } - private synchronized void createThreadPools() { - if (startIngestJobsExecutor == null) { - startIngestJobsExecutor = Executors.newSingleThreadExecutor(); + private synchronized void stopIngestTasks() { + // First mark all of the ingest jobs as cancelled. This way the + // ingest modules will know they are being shut down due to + // cancellation when the cancelled run ingest module tasks release + // their pipelines. + for (IngestJob job : ingestJobs.values()) { + job.cancel(); } - if (dataSourceIngestTasksExecutor == null) { - dataSourceIngestTasksExecutor = Executors.newSingleThreadExecutor(); - } - if (fileIngestTasksExecutor == null) { - fileIngestTasksExecutor = Executors.newFixedThreadPool(MAX_NUMBER_OF_FILE_INGEST_THREADS); + + // Cancel the run ingest module tasks, setting the state of the threads + // running them to interrupted. + for (Future task : ingestTasks.values()) { + task.cancel(true); } + + // Jettision the remaining data source and file ingest tasks. + scheduler.getFileScheduler().empty(); + scheduler.getDataSourceScheduler().empty(); } - private synchronized boolean shutDownThreadPools(int timeOut, TimeUnit timeOutUnits) { - boolean success = true; - if (!shutDownThreadPool(startIngestJobsExecutor, timeOut, timeOutUnits)) { - success = false; - } - if (!shutDownThreadPool(dataSourceIngestTasksExecutor, timeOut, timeOutUnits)) { - success = false; - } - if (!shutDownThreadPool(fileIngestTasksExecutor, timeOut, timeOutUnits)) { - success = false; - } - startIngestJobsExecutor = null; - dataSourceIngestTasksExecutor = null; - fileIngestTasksExecutor = null; - return success; + synchronized void reportStartIngestJobsTaskDone(long taskId) { + ingestTasks.remove(taskId); } - private boolean shutDownThreadPool(ExecutorService pool, int waitTime, TimeUnit unit) { - try { - pool.shutdownNow(); - return pool.awaitTermination(waitTime, unit); - } catch (InterruptedException ex) { - pool.shutdownNow(); - Thread.currentThread().interrupt(); // Preserve interrupted status. - return false; - } - } + synchronized void reportRunIngestModulesTaskDone(long taskId) { + ingestTasks.remove(taskId); - private class IngestJobsManager { - - private final HashMap jobs = new HashMap<>(); // Maps job ids to jobs - - synchronized void addJob(IngestJob job) { - jobs.put(job.getId(), job); - } - - synchronized boolean hasJobs() { - return (jobs.isEmpty() == false); - } - - synchronized boolean addFileToIngestJob(long ingestJobId, AbstractFile file) { - IngestJob job = jobs.get(ingestJobId); - if (job != null) { - scheduler.getFileScheduler().scheduleFile(job, file); - return true; - } else { - logger.log(Level.SEVERE, "Unable to map ingest job id (id = {0}) to an ingest job, failed to add file (id = {1})", new Object[]{ingestJobId, file.getId()}); - return false; + List completedJobs = new ArrayList<>(); + for (IngestJob job : ingestJobs.values()) { + job.releaseIngestPipelinesForThread(taskId); + if (job.areIngestPipelinesShutDown() == true) { + completedJobs.add(job.getId()); } } - synchronized void cancelJobs() { - for (IngestJob job : jobs.values()) { - job.cancel(); - } - } - - synchronized void reportIngestTaskDone(long taskId) { - List completedJobs = new ArrayList<>(); - for (IngestJob job : jobs.values()) { - job.releaseIngestPipelinesForThread(taskId); - if (job.areIngestPipelinesShutDown() == true) { - completedJobs.add(job.getId()); - } - } - - for (Long jobId : completedJobs) { - jobs.remove(jobId); - } + for (Long jobId : completedJobs) { + ingestJobs.remove(jobId); } } @@ -391,7 +351,6 @@ public class IngestManager { private final List moduleTemplates; private final boolean processUnallocatedSpace; private ProgressHandle progress; - private volatile boolean finished = false; StartIngestJobsTask(long taskId, List dataSources, List moduleTemplates, boolean processUnallocatedSpace) { this.id = taskId; @@ -422,7 +381,6 @@ public class IngestManager { break; } - final String inputName = dataSource.getName(); IngestJob ingestJob = new IngestJob(IngestManager.this.ingestJobId.incrementAndGet(), dataSource, moduleTemplates, processUnallocatedSpace); List errors = ingestJob.startUpIngestPipelines(); if (!errors.isEmpty()) { @@ -436,19 +394,23 @@ public class IngestManager { failedModules.append(","); } } - MessageNotifyUtil.Message.error( // RJCTODO: Fix this to show all errors + MessageNotifyUtil.Message.error( // RJCTODO: Fix this to show all errors, probably should specify data source name "Failed to start the following ingest modules: " + failedModules.toString() + " .\n\n" + "No ingest modules will be run. Please disable the module " + "or fix the error and restart ingest by right clicking on " + "the data source and selecting Run Ingest Modules.\n\n" + "Error: " + errors.get(0).getModuleError().getMessage()); + ingestJob.cancel(); break; } // Save the ingest job for later cleanup of pipelines. - jobsManager.addJob(ingestJob); + synchronized (IngestManager.this) { + ingestJobs.put(ingestJob.getId(), ingestJob); + } // Queue the data source ingest tasks for the ingest job. + final String inputName = dataSource.getName(); progress.progress("DataSource Ingest" + " " + inputName, processed); scheduler.getDataSourceScheduler().schedule(ingestJob); progress.progress("DataSource Ingest" + " " + inputName, ++processed); @@ -457,22 +419,20 @@ public class IngestManager { progress.progress("File Ingest" + " " + inputName, processed); scheduler.getFileScheduler().scheduleIngestOfFiles(ingestJob); progress.progress("File Ingest" + " " + inputName, ++processed); - } - if (!Thread.currentThread().isInterrupted()) { - startIngestTasks(); + if (!Thread.currentThread().isInterrupted()) { + startIngestTasks(); + } } } catch (Exception ex) { String message = String.format("StartIngestJobsTask (id=%d) caught exception", id); logger.log(Level.SEVERE, message, ex); + MessageNotifyUtil.Message.error("An error occurred while starting ingest. Results may only be partial"); } finally { progress.finish(); + reportStartIngestJobsTaskDone(id); } } - - boolean isFinished() { - return finished; - } } private class RunDataSourceIngestModulesTask implements Runnable { @@ -492,14 +452,13 @@ public class IngestManager { break; } IngestJob job = scheduler.next(); - DataSourceIngestPipeline pipeline = job.getDataSourceIngestPipelineForThread(id); - pipeline.process(job.getDataSourceTaskProgressBar()); + job.getDataSourceIngestPipelineForThread(id).process(); } } catch (Exception ex) { String message = String.format("RunDataSourceIngestModulesTask (id=%d) caught exception", id); logger.log(Level.SEVERE, message, ex); } finally { - jobsManager.reportIngestTaskDone(id); + reportRunIngestModulesTaskDone(id); } } } @@ -522,55 +481,34 @@ public class IngestManager { } IngestScheduler.FileScheduler.FileTask task = fileScheduler.next(); IngestJob job = task.getJob(); - FileIngestPipeline pipeline = job.getFileIngestPipelineForThread(id); - job.handleFileTaskStarted(task); - pipeline.process(task.getFile()); + job.updateFileTasksProgressBar(task.getFile().getName()); + job.getFileIngestPipelineForThread(id).process(task.getFile()); } } catch (Exception ex) { String message = String.format("RunFileSourceIngestModulesTask (id=%d) caught exception", id); logger.log(Level.SEVERE, message, ex); } finally { - jobsManager.reportIngestTaskDone(id); + reportRunIngestModulesTaskDone(id); } } } - class IngestCancellationWorker extends SwingWorker { + class IngestCancellationWorker extends SwingWorker { - private final int timeOut; - private final TimeUnit timeOutUnits; - - IngestCancellationWorker(int timeOut, TimeUnit timeOutUnits) { - this.timeOut = timeOut; - this.timeOutUnits = timeOutUnits; + @Override + protected Void doInBackground() throws Exception { + stopIngestTasks(); + return null; } @Override - protected Boolean doInBackground() throws Exception { - // First mark all of the ingest jobs as cancelled. This way the - // ingest modules will know they are being shut down due to - // cancellation when the cancelled run ingest module tasks release - // their pipelines. This also makes sure the lock on the jobs - // manager is released before the run ingest module tasks start - // releasing pipelines. - jobsManager.cancelJobs(); - - // Jettision the remaining data source and file ingest tasks. This - // will could break the the run ingest module tasks out of their - // loops even before the pools mark their threads as interrupted. - scheduler.getFileScheduler().empty(); - scheduler.getDataSourceScheduler().empty(); - - boolean success = shutDownThreadPools(timeOut, timeOutUnits); - - // Jettision data source and file ingest tasks again to try to - // dispose of any tasks that slipped into the queues. - scheduler.getFileScheduler().empty(); - scheduler.getDataSourceScheduler().empty(); - - return success; + protected void done() { + try { + super.get(); + } catch (CancellationException | InterruptedException ex) { + } catch (Exception ex) { + logger.log(Level.SEVERE, "Error while cancelling ingest jobs", ex); + } } - - // TODO: Add done() override to notify a listener of success or failure } } From 9c928c17d5ff3fea1ec4e7c700aaf00257696f9e Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Tue, 8 Apr 2014 16:10:54 -0400 Subject: [PATCH 12/15] Fixed bug in DataSourceIngestPipeline --- .../sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java index 2a135d4c96..bac11238b5 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java @@ -22,7 +22,6 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -import org.netbeans.api.progress.ProgressHandle; import org.sleuthkit.datamodel.Content; /** @@ -84,6 +83,9 @@ final class DataSourceIngestPipeline { } catch (Exception ex) { errors.add(new IngestModuleError(module.getDisplayName(), ex)); } + if (job.isCancelled()) { + break; + } } return errors; } @@ -96,9 +98,6 @@ final class DataSourceIngestPipeline { } catch (Exception ex) { errors.add(new IngestModuleError(module.getDisplayName(), ex)); } - if (job.isCancelled()) { - break; - } } return errors; } From 181c3828b72567ab56bc6945f973daae6cd90275 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Tue, 8 Apr 2014 16:13:44 -0400 Subject: [PATCH 13/15] Fixed bug in FileIngestPipeline --- Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java index 13137318e4..e221b8b33b 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java @@ -88,7 +88,7 @@ final class FileIngestPipeline { } } file.close(); - if (job.isCancelled()) { + if (!job.isCancelled()) { IngestManager.fireFileDone(file.getId()); } return errors; From a07153261e568fc735594116a6fdb0463bdbb17e Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Wed, 9 Apr 2014 11:09:19 -0400 Subject: [PATCH 14/15] Changed ingest module lifecycle events to ingest job lifecycle events --- .../DirectoryTreeTopComponent.java | 86 +++++++++---------- .../autopsy/ingest/FileIngestPipeline.java | 2 +- .../autopsy/ingest/IngestManager.java | 57 ++++++------ .../hashdatabase/HashLookupSettingsPanel.java | 16 +++- .../KeywordSearchEditListPanel.java | 59 +++++-------- .../KeywordSearchGlobalListSettingsPanel.java | 2 +- .../KeywordSearchListsViewerPanel.java | 49 +++++------ 7 files changed, 121 insertions(+), 150 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java b/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java index d5e82204b2..53242a04b3 100644 --- a/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java +++ b/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java @@ -73,6 +73,7 @@ import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskException; import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; + /** * Top component which displays something. */ @@ -82,7 +83,7 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat private transient ExplorerManager em = new ExplorerManager(); private static DirectoryTreeTopComponent instance; private DataResultTopComponent dataResult = new DataResultTopComponent(true, NbBundle.getMessage(this.getClass(), - "DirectoryTreeTopComponent.title.text")); + "DirectoryTreeTopComponent.title.text")); private LinkedList backList; private LinkedList forwardList; /** @@ -222,12 +223,12 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat private void backButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_backButtonActionPerformed // change the cursor to "waiting cursor" for this operation this.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR)); - + // the end is the current place, String[] currentNodePath = backList.pollLast(); forwardList.addLast(currentNodePath); forwardButton.setEnabled(true); - + /* We peek instead of poll because we use its existence * in the list later on so that we do not reset the forward list * after the selection occurs. */ @@ -239,7 +240,7 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat } else { backButton.setEnabled(false); } - + // update the selection on directory tree setSelectedNode(newCurrentNodePath, null); @@ -256,10 +257,10 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat } else { forwardButton.setEnabled(false); } - + backList.addLast(newCurrentNodePath); backButton.setEnabled(true); - + // update the selection on directory tree setSelectedNode(newCurrentNodePath, null); @@ -543,8 +544,7 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat // The current case has been closed. Reset the ExplorerManager. Node emptyNode = new AbstractNode(Children.LEAF); em.setRootContext(emptyNode); - } - else if (newValue != null) { + } else if (newValue != null) { // A new case has been opened. Reset the forward and back // buttons. Note that a call to CoreComponentControl.openCoreWindows() // by the new Case object will lead to a componentOpened() call @@ -598,7 +598,8 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat refreshTree(event.getArtifactType()); } }); - } else if (changed.equals(IngestEvent.COMPLETED.toString())) { + } else if (changed.equals(IngestEvent.INGEST_JOB_COMPLETED.toString()) + || changed.equals(IngestEvent.INGEST_JOB_CANCELLED.toString())) { SwingUtilities.invokeLater(new Runnable() { @Override public void run() { @@ -654,7 +655,7 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat return; } Node originNode = origin.getNode(); - + //set node, wrap in filter node first to filter out children Node drfn = new DataResultFilterNode(originNode, DirectoryTreeTopComponent.this.em); Node kffn = new KnownFileFilterNode(drfn, KnownFileFilterNode.getSelectionContext(originNode)); @@ -667,9 +668,8 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat displayName = content.getUniquePath(); } catch (TskCoreException ex) { logger.log(Level.SEVERE, "Exception while calling Content.getUniquePath() for node: " + originNode); - } - } - else if (originNode.getLookup().lookup(String.class) != null) { + } + } else if (originNode.getLookup().lookup(String.class) != null) { displayName = originNode.getLookup().lookup(String.class); } dataResult.setPath(displayName); @@ -695,12 +695,12 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat // update the back and forward list updateHistory(em.getSelectedNodes()); } - + private void updateHistory(Node[] selectedNodes) { if (selectedNodes.length == 0) { return; } - + Node selectedNode = selectedNodes[0]; String selectedNodeName = selectedNode.getName(); @@ -729,7 +729,7 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat forwardButton.setEnabled(false); // disable the forward Button } } - + /** * Resets the back and forward list, and also disable the back and forward * buttons. @@ -752,8 +752,6 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat pcs.removePropertyChangeListener(listener); } - - /** * Gets the tree on this DirectoryTreeTopComponent. * @@ -768,13 +766,13 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat */ public void refreshContentTreeSafe() { SwingUtilities.invokeLater(new Runnable() { - @Override - public void run() { - refreshContentTree(); - } - }); + @Override + public void run() { + refreshContentTree(); + } + }); } - + /** * Refreshes changed content nodes */ @@ -865,7 +863,7 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat /** * Set the selected node using a path to a previously selected node. * - * @param previouslySelectedNodePath Path to a previously selected node. + * @param previouslySelectedNodePath Path to a previously selected node. * @param rootNodeName Name of the root node to match, may be null. */ private void setSelectedNode(final String[] previouslySelectedNodePath, final String rootNodeName) { @@ -876,28 +874,26 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat @Override public void run() { if (previouslySelectedNodePath.length > 0 && (rootNodeName == null || previouslySelectedNodePath[0].equals(rootNodeName))) { - Node selectedNode = null; + Node selectedNode = null; ArrayList selectedNodePath = new ArrayList<>(Arrays.asList(previouslySelectedNodePath)); while (null == selectedNode && !selectedNodePath.isEmpty()) { try { - selectedNode = NodeOp.findPath(em.getRootContext(), selectedNodePath.toArray(new String[0])); - } - catch (NodeNotFoundException ex) { + selectedNode = NodeOp.findPath(em.getRootContext(), selectedNodePath.toArray(new String[0])); + } catch (NodeNotFoundException ex) { // The selected node may have been deleted (e.g., a deleted tag), so truncate the path and try again. if (selectedNodePath.size() > 1) { selectedNodePath.remove(selectedNodePath.size() - 1); - } - else { + } else { StringBuilder nodePath = new StringBuilder(); for (int i = 0; i < previouslySelectedNodePath.length; ++i) { nodePath.append(previouslySelectedNodePath[i]).append("/"); } logger.log(Level.WARNING, "Failed to find any nodes to select on path " + nodePath.toString(), ex); - break; + break; } - } + } } - + if (null != selectedNode) { if (rootNodeName != null) { //called from tree auto refresh context @@ -905,9 +901,8 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat backList.pollLast(); } try { - em.setExploredContextAndSelection(selectedNode, new Node[]{selectedNode}); - } - catch (PropertyVetoException ex) { + em.setExploredContextAndSelection(selectedNode, new Node[]{selectedNode}); + } catch (PropertyVetoException ex) { logger.log(Level.WARNING, "Property veto from ExplorerManager setting selection to " + selectedNode.getName(), ex); } } @@ -970,11 +965,11 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat } catch (TskException ex) { logger.log(Level.WARNING, "Error retrieving attributes", ex); } - } else if ( type.equals(BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT) || - type.equals(BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_ARTIFACT_HIT) ) { + } else if (type.equals(BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT) + || type.equals(BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_ARTIFACT_HIT)) { Node interestingItemsRootNode = resultsChilds.findChild(type.getLabel()); Children interestingItemsRootChildren = interestingItemsRootNode.getChildren(); - try { + try { String setName = null; List attributes = art.getAttributes(); for (BlackboardAttribute att : attributes) { @@ -1030,16 +1025,15 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat } void fireViewerComplete() { - + try { firePropertyChange(BlackboardResultViewer.FINISHED_DISPLAY_EVT, 0, 1); - } - catch (Exception e) { + } catch (Exception e) { logger.log(Level.SEVERE, "DirectoryTreeTopComponent listener threw exception", e); MessageNotifyUtil.Notify.show(NbBundle.getMessage(this.getClass(), "DirectoryTreeTopComponent.moduleErr"), - NbBundle.getMessage(this.getClass(), - "DirectoryTreeTopComponent.moduleErr.msg"), - MessageNotifyUtil.MessageType.ERROR); + NbBundle.getMessage(this.getClass(), + "DirectoryTreeTopComponent.moduleErr.msg"), + MessageNotifyUtil.MessageType.ERROR); } } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java index e221b8b33b..2cd7d838e4 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java @@ -89,7 +89,7 @@ final class FileIngestPipeline { } file.close(); if (!job.isCancelled()) { - IngestManager.fireFileDone(file.getId()); + IngestManager.fireFileIngestDone(file.getId()); } return errors; } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java index 59f0bf7789..f48eb9e83d 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java @@ -122,7 +122,7 @@ public class IngestManager { * * @return True if any ingest jobs are in progress, false otherwise */ - public boolean isIngestRunning() { + public boolean isIngestRunning() { // RJCTODO: Address synch or not synch and listeners return (ingestJobs.isEmpty() == false); } @@ -133,7 +133,7 @@ public class IngestManager { } } - synchronized void cancelIngestJobs() { + void cancelIngestJobs() { new IngestCancellationWorker().execute(); } @@ -143,29 +143,24 @@ public class IngestManager { public enum IngestEvent { /** - * Event sent when an ingest module has been started. Second argument of - * the property change is a string form of the module name and the third - * argument is null. + * Property change event fired when an ingest job is started. The ingest + * job id is in old value field of the PropertyChangeEvent object. */ - STARTED, + INGEST_JOB_STARTED, /** - * Event sent when an ingest module has completed processing by its own - * means. Second argument of the property change is a string form of the - * module name and the third argument is null. - * - * This event is generally used by listeners to perform a final data - * view refresh (listeners need to query all data from the blackboard). + * Property change event fired when an ingest job is completed. The + * ingest job id is in old value field of the PropertyChangeEvent + * object. */ - COMPLETED, + INGEST_JOB_COMPLETED, /** - * Event sent when an ingest module has stopped processing, and likely - * not all data has been processed. Second argument of the property - * change is a string form of the module name and third argument is - * null. + * Property change event fired when an ingest job is canceled. The + * ingest job id is in old value field of the PropertyChangeEvent + * object. */ - STOPPED, + INGEST_JOB_CANCELLED, /** - * Event sent when ingest module posts new data to blackboard or + * Event sent when an ingest module posts new data to blackboard or * somewhere else. Second argument of the property change fired contains * ModuleDataEvent object and third argument is null. The object can * contain encapsulated new data created by the module. Listener can @@ -198,9 +193,9 @@ public class IngestManager { pcs.removePropertyChangeListener(listener); } - static void fireModuleEvent(String eventType, String moduleName) { + static void fireIngestJobEvent(String eventType, long jobId) { try { - pcs.firePropertyChange(eventType, moduleName, null); + pcs.firePropertyChange(eventType, jobId, null); } catch (Exception e) { logger.log(Level.SEVERE, "Ingest manager listener threw exception", e); MessageNotifyUtil.Notify.show(NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr"), @@ -212,11 +207,11 @@ public class IngestManager { /** * Fire event when file is done with a pipeline run * - * @param objId ID of file that is done + * @param fileId ID of file that is done */ - static void fireFileDone(long objId) { + static void fireFileIngestDone(long fileId) { try { - pcs.firePropertyChange(IngestEvent.FILE_DONE.toString(), objId, null); + pcs.firePropertyChange(IngestEvent.FILE_DONE.toString(), fileId, null); } catch (Exception e) { logger.log(Level.SEVERE, "Ingest manager listener threw exception", e); MessageNotifyUtil.Notify.show(NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr"), @@ -340,7 +335,8 @@ public class IngestManager { } for (Long jobId : completedJobs) { - ingestJobs.remove(jobId); + IngestJob job = ingestJobs.remove(jobId); + fireIngestJobEvent(job.isCancelled() ? IngestEvent.INGEST_JOB_CANCELLED.toString() : IngestEvent.INGEST_JOB_COMPLETED.toString(), jobId); } } @@ -375,7 +371,7 @@ public class IngestManager { }); progress.start(2 * dataSources.size()); - int processed = 0; + int workUnitsCompleted = 0; for (Content dataSource : dataSources) { if (Thread.currentThread().isInterrupted()) { break; @@ -411,17 +407,18 @@ public class IngestManager { // Queue the data source ingest tasks for the ingest job. final String inputName = dataSource.getName(); - progress.progress("DataSource Ingest" + " " + inputName, processed); + progress.progress("Data source ingest tasks for " + inputName, workUnitsCompleted); // RJCTODO: Improve scheduler.getDataSourceScheduler().schedule(ingestJob); - progress.progress("DataSource Ingest" + " " + inputName, ++processed); + progress.progress("Data source ingest tasks for " + inputName, ++workUnitsCompleted); // Queue the file ingest tasks for the ingest job. - progress.progress("File Ingest" + " " + inputName, processed); + progress.progress("Data source ingest tasks for " + inputName, workUnitsCompleted); scheduler.getFileScheduler().scheduleIngestOfFiles(ingestJob); - progress.progress("File Ingest" + " " + inputName, ++processed); + progress.progress("Data source ingest tasks for " + inputName, ++workUnitsCompleted); if (!Thread.currentThread().isInterrupted()) { startIngestTasks(); + fireIngestJobEvent(IngestEvent.INGEST_JOB_STARTED.toString(), ingestJob.getId()); } } } catch (Exception ex) { diff --git a/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashLookupSettingsPanel.java b/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashLookupSettingsPanel.java index 02b307fc8f..4e1e9ceb70 100644 --- a/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashLookupSettingsPanel.java +++ b/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashLookupSettingsPanel.java @@ -20,6 +20,7 @@ package org.sleuthkit.autopsy.hashdatabase; import java.awt.Color; import java.awt.Component; +import java.awt.EventQueue; import java.awt.Frame; import java.awt.event.KeyEvent; import java.beans.PropertyChangeEvent; @@ -71,8 +72,13 @@ public final class HashLookupSettingsPanel extends IngestModuleGlobalSetttingsPa IngestManager.addPropertyChangeListener(new PropertyChangeListener() { @Override public void propertyChange(PropertyChangeEvent evt) { - if (isFileIngestStatusChangeEvent(evt)) { - updateComponents(); + if (isIngestJobEvent(evt)) { + EventQueue.invokeLater(new Runnable() { + @Override + public void run() { + updateComponents(); + } + }); } } }); @@ -224,8 +230,10 @@ public final class HashLookupSettingsPanel extends IngestModuleGlobalSetttingsPa return shortenedPath; } - private boolean isFileIngestStatusChangeEvent(PropertyChangeEvent evt) { - return evt.getPropertyName().equals(IngestManager.IngestEvent.STARTED.toString()) || evt.getPropertyName().equals(IngestManager.IngestEvent.COMPLETED.toString()) || evt.getPropertyName().equals(IngestManager.IngestEvent.STOPPED.toString()); + private boolean isIngestJobEvent(PropertyChangeEvent evt) { + return evt.getPropertyName().equals(IngestManager.IngestEvent.INGEST_JOB_STARTED.toString()) + || evt.getPropertyName().equals(IngestManager.IngestEvent.INGEST_JOB_COMPLETED.toString()) + || evt.getPropertyName().equals(IngestManager.IngestEvent.INGEST_JOB_CANCELLED.toString()); } @Override diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchEditListPanel.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchEditListPanel.java index fcc1dc7685..db4a4eaf44 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchEditListPanel.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchEditListPanel.java @@ -18,6 +18,7 @@ */ package org.sleuthkit.autopsy.keywordsearch; +import java.awt.EventQueue; import java.awt.event.ActionEvent; import java.awt.event.ActionListener; import java.beans.PropertyChangeEvent; @@ -40,6 +41,8 @@ import javax.swing.event.ListSelectionListener; import javax.swing.filechooser.FileNameExtensionFilter; import javax.swing.table.AbstractTableModel; import javax.swing.table.TableColumn; +import org.openide.nodes.Children; +import org.openide.nodes.Node; import org.sleuthkit.autopsy.corecomponents.OptionsPanel; import org.sleuthkit.autopsy.ingest.IngestManager; import org.sleuthkit.autopsy.ingest.IngestManager.IngestEvent; @@ -52,7 +55,6 @@ class KeywordSearchEditListPanel extends javax.swing.JPanel implements ListSelec private static Logger logger = Logger.getLogger(KeywordSearchEditListPanel.class.getName()); private KeywordTableModel tableModel; private KeywordList currentKeywordList; - private boolean ingestRunning; /** * Creates new form KeywordSearchEditListPanel @@ -101,7 +103,7 @@ class KeywordSearchEditListPanel extends javax.swing.JPanel implements ListSelec } }); - initButtons(); + setButtonStates(); addWordField.setComponentPopupMenu(rightClickMenu); ActionListener actList = new ActionListener() { @@ -124,49 +126,28 @@ class KeywordSearchEditListPanel extends javax.swing.JPanel implements ListSelec pasteMenuItem.addActionListener(actList); selectAllMenuItem.addActionListener(actList); - if (IngestManager.getInstance().isIngestRunning()) { - initIngest(0); - } else { - initIngest(1); - } + setButtonStates(); IngestManager.addPropertyChangeListener(new PropertyChangeListener() { @Override public void propertyChange(PropertyChangeEvent evt) { String changed = evt.getPropertyName(); - Object oldValue = evt.getOldValue(); - if (changed.equals(IngestEvent.COMPLETED.toString()) - && ((String) oldValue).equals(KeywordSearchModuleFactory.getModuleName())) { - initIngest(1); - } else if (changed.equals(IngestEvent.STARTED.toString()) - && ((String) oldValue).equals(KeywordSearchModuleFactory.getModuleName())) { - initIngest(0); - } else if (changed.equals(IngestEvent.STOPPED.toString()) - && ((String) oldValue).equals(KeywordSearchModuleFactory.getModuleName())) { - initIngest(1); + if (changed.equals(IngestEvent.INGEST_JOB_STARTED.toString()) + || changed.equals(IngestEvent.INGEST_JOB_COMPLETED.toString()) + || changed.equals(IngestEvent.INGEST_JOB_CANCELLED.toString())) { + EventQueue.invokeLater(new Runnable() { + @Override + public void run() { + setButtonStates(); + } + }); } } }); } - /** - * Initialize this panel depending on whether ingest is running - * - * @param running case 0: ingest running case 1: ingest not running - */ - private void initIngest(int running) { - switch (running) { - case 0: - ingestRunning = true; - break; - case 1: - ingestRunning = false; - break; - } - initButtons(); - } - - void initButtons() { + void setButtonStates() { + boolean ingestRunning = IngestManager.getInstance().isIngestRunning(); boolean listSet = currentKeywordList != null; boolean isLocked = !listSet ? true : currentKeywordList.isLocked(); boolean noKeywords = !listSet ? true : currentKeywordList.getKeywords().isEmpty(); @@ -441,7 +422,7 @@ class KeywordSearchEditListPanel extends javax.swing.JPanel implements ListSelec chRegex.setSelected(false); addWordField.setText(""); - initButtons(); + setButtonStates(); }//GEN-LAST:event_addWordButtonActionPerformed private void deleteWordButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_deleteWordButtonActionPerformed @@ -449,7 +430,7 @@ class KeywordSearchEditListPanel extends javax.swing.JPanel implements ListSelec tableModel.deleteSelected(keywordTable.getSelectedRows()); KeywordSearchListsXML.getCurrent().addList(currentKeywordList); - initButtons(); + setButtonStates(); } }//GEN-LAST:event_deleteWordButtonActionPerformed @@ -549,11 +530,11 @@ class KeywordSearchEditListPanel extends javax.swing.JPanel implements ListSelec currentKeywordList = loader.getListsL(false).get(index); tableModel.resync(); - initButtons(); + setButtonStates(); } else { currentKeywordList = null; tableModel.resync(); - initButtons(); + setButtonStates(); } } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalListSettingsPanel.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalListSettingsPanel.java index b627e17e21..ac8e74212d 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalListSettingsPanel.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalListSettingsPanel.java @@ -44,7 +44,7 @@ final class KeywordSearchGlobalListSettingsPanel extends javax.swing.JPanel impl if (KeywordSearchUtil.displayConfirmDialog(NbBundle.getMessage(this.getClass(), "KeywordSearchConfigurationPanel1.customizeComponents.title"), NbBundle.getMessage(this.getClass(), "KeywordSearchConfigurationPanel1.customizeComponents.body"), KeywordSearchUtil.DIALOG_MESSAGE_TYPE.WARN)) { String toDelete = editListPanel.getCurrentKeywordList().getName(); editListPanel.setCurrentKeywordList(null); - editListPanel.initButtons(); + editListPanel.setButtonStates(); // RJCTODO: Move this into a deleteList method in the manager KeywordSearchListsXML deleter = KeywordSearchListsXML.getCurrent(); deleter.deleteList(toDelete); diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchListsViewerPanel.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchListsViewerPanel.java index de6d93a918..6b0ca4ad8a 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchListsViewerPanel.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchListsViewerPanel.java @@ -21,6 +21,7 @@ package org.sleuthkit.autopsy.keywordsearch; import java.awt.Component; import java.awt.Cursor; +import java.awt.EventQueue; import java.awt.event.ActionEvent; import java.awt.event.ActionListener; import java.beans.PropertyChangeEvent; @@ -114,26 +115,23 @@ class KeywordSearchListsViewerPanel extends AbstractKeywordSearchPerformer { } }); - if (IngestManager.getInstance().isIngestRunning()) { - initIngest(true); - } else { - initIngest(false); - } + ingestRunning = IngestManager.getInstance().isIngestRunning(); + updateComponents(); IngestManager.addPropertyChangeListener(new PropertyChangeListener() { @Override public void propertyChange(PropertyChangeEvent evt) { String changed = evt.getPropertyName(); - Object oldValue = evt.getOldValue(); - if (changed.equals(IngestEvent.COMPLETED.toString()) - && ((String) oldValue).equals(KeywordSearchModuleFactory.getModuleName())) { - initIngest(false); - } else if (changed.equals(IngestEvent.STARTED.toString()) - && ((String) oldValue).equals(KeywordSearchModuleFactory.getModuleName())) { - initIngest(true); - } else if (changed.equals(IngestEvent.STOPPED.toString()) - && ((String) oldValue).equals(KeywordSearchModuleFactory.getModuleName())) { - initIngest(false); + if (changed.equals(IngestEvent.INGEST_JOB_STARTED.toString()) + || changed.equals(IngestEvent.INGEST_JOB_COMPLETED.toString()) + || changed.equals(IngestEvent.INGEST_JOB_CANCELLED.toString())) { + EventQueue.invokeLater(new Runnable() { + @Override + public void run() { + ingestRunning = IngestManager.getInstance().isIngestRunning(); + updateComponents(); + } + }); } } }); @@ -152,28 +150,21 @@ class KeywordSearchListsViewerPanel extends AbstractKeywordSearchPerformer { searchAddButton.addActionListener(searchAddListener); } - /** - * Initialize this panel depending on whether ingest is running - * - * @param running case 0: ingest running case 1: ingest not running - */ - private void initIngest(boolean running) { - if (running) { - ingestRunning = true; + private void updateComponents() { + ingestRunning = IngestManager.getInstance().isIngestRunning(); + if (ingestRunning) { searchAddButton.setText(NbBundle.getMessage(this.getClass(), "KeywordSearchListsViewerPanel.initIngest.addIngestTitle")); searchAddButton.setToolTipText(NbBundle.getMessage(this.getClass(), "KeywordSearchListsViewerPanel.initIngest.addIngestMsg" )); - listsTableModel.resync(); } else { - ingestRunning = false; searchAddButton.setText(NbBundle.getMessage(this.getClass(), "KeywordSearchListsViewerPanel.initIngest.searchIngestTitle")); searchAddButton.setToolTipText(NbBundle.getMessage(this.getClass(), "KeywordSearchListsViewerPanel.initIngest.addIdxSearchMsg")); - listsTableModel.resync(); } - updateIngestIndexLabel(running); + listsTableModel.resync(); + updateIngestIndexLabel(); } - private void updateIngestIndexLabel(boolean ingestRunning) { + private void updateIngestIndexLabel() { if (ingestRunning) { ingestIndexLabel.setText(NbBundle.getMessage(this.getClass(), "KeywordSearchListsViewerPanel.initIngest.ongoingIngestMsg", filesIndexed)); } @@ -184,7 +175,7 @@ class KeywordSearchListsViewerPanel extends AbstractKeywordSearchPerformer { @Override protected void postFilesIndexedChange() { - updateIngestIndexLabel(ingestRunning); + updateIngestIndexLabel(); } /** From 5b25aa1edde9caba51a5bb42245899c21669fca7 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Wed, 9 Apr 2014 11:15:20 -0400 Subject: [PATCH 15/15] Removed unused imports, TODO comment --- Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java | 2 +- .../autopsy/keywordsearch/KeywordSearchEditListPanel.java | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java index f48eb9e83d..1cc046ac3e 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java @@ -122,7 +122,7 @@ public class IngestManager { * * @return True if any ingest jobs are in progress, false otherwise */ - public boolean isIngestRunning() { // RJCTODO: Address synch or not synch and listeners + public boolean isIngestRunning() { return (ingestJobs.isEmpty() == false); } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchEditListPanel.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchEditListPanel.java index db4a4eaf44..27ba1933e5 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchEditListPanel.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchEditListPanel.java @@ -41,8 +41,6 @@ import javax.swing.event.ListSelectionListener; import javax.swing.filechooser.FileNameExtensionFilter; import javax.swing.table.AbstractTableModel; import javax.swing.table.TableColumn; -import org.openide.nodes.Children; -import org.openide.nodes.Node; import org.sleuthkit.autopsy.corecomponents.OptionsPanel; import org.sleuthkit.autopsy.ingest.IngestManager; import org.sleuthkit.autopsy.ingest.IngestManager.IngestEvent;