From a77d566a82cc7b8c97f7dfd2f15b45baadee43e9 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Thu, 1 May 2014 09:41:01 -0400 Subject: [PATCH] Preserve intermediate state of improved ingest framework --- .../DirectoryTreeTopComponent.java | 3 +- .../autopsy/ingest/Bundle.properties | 8 +- .../autopsy/ingest/Bundle_ja.properties | 4 - .../DataSourceIngestModuleProgress.java | 14 +- .../ingest/DataSourceIngestPipeline.java | 18 +- .../autopsy/ingest/DataSourceIngestTask.java | 4 +- .../ingest/DataSourceIngestTaskScheduler.java | 38 ++-- .../autopsy/ingest/FileIngestPipeline.java | 17 +- .../autopsy/ingest/FileIngestTask.java | 46 ++--- .../ingest/FileIngestTaskScheduler.java | 124 +++++------ .../sleuthkit/autopsy/ingest/IngestJob.java | 195 +++++++++--------- .../autopsy/ingest/IngestJobContext.java | 4 +- .../autopsy/ingest/IngestManager.java | 173 +++++++++------- .../autopsy/ingest/IngestServices.java | 4 +- .../hashdatabase/HashLookupSettingsPanel.java | 2 +- .../KeywordSearchEditListPanel.java | 2 +- .../KeywordSearchListsViewerPanel.java | 2 +- c:casesSamll2Againautopsy.db | 0 18 files changed, 329 insertions(+), 329 deletions(-) create mode 100755 c:casesSamll2Againautopsy.db diff --git a/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java b/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java index a03933abf6..f7067adf6e 100644 --- a/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java +++ b/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java @@ -131,7 +131,8 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat private void setListener() { Case.addPropertyChangeListener(this);// add this class to listen to any changes in the Case.java class this.em.addPropertyChangeListener(this); - IngestManager.addPropertyChangeListener(this); + IngestManager.getInstance().addIngestJobEventListener(this); + IngestManager.getInstance().addIngestModuleEventListener(this); } public void setDirectoryListingActive() { diff --git a/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties b/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties index 144c7359fd..253f4f390d 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties @@ -60,13 +60,9 @@ IngestMessagePanel.sortByComboBox.model.priority=Priority IngestMessagesToolbar.customizeButton.toolTipText=Ingest Messages IngestMessageTopComponent.initComponents.name=Ingest Inbox IngestManager.StartIngestJobsTask.run.startupErr.dlgMsg=Unable to start up one or more ingest modules, ingest job cancelled. -IngestManager.StartIngestJobsTask.run.startupErr.dlgSolution=Please disable the failed modules or fix the errors and then restart ingest\ +IngestManager.StartIngestJobsTask.run.startupErr.dlgSolution=Please disable the failed modules or fix the errors and then restart ingest \ by right clicking on the data source and selecting Run Ingest Modules. -IngestManager.StartIngestJobsTask.run.startupErr.dlgErrorList=Errors\:\ +IngestManager.StartIngestJobsTask.run.startupErr.dlgErrorList=Errors\: \ \ {0} IngestManager.StartIngestJobsTask.run.startupErr.dlgTitle=Ingest Failure -IngestManager.StartIngestJobsTask.run.progress.msg1=Data source ingest tasks for {0} -IngestManager.StartIngestJobsTask.run.progress.msg2=Data source ingest tasks for {0} -IngestManager.StartIngestJobsTask.run.progress.msg3=Data source ingest tasks for {0} -IngestManager.StartIngestJobsTask.run.progress.msg4=Data source ingest tasks for {0} diff --git a/Core/src/org/sleuthkit/autopsy/ingest/Bundle_ja.properties b/Core/src/org/sleuthkit/autopsy/ingest/Bundle_ja.properties index e3433daff3..c6a50ca915 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/Bundle_ja.properties +++ b/Core/src/org/sleuthkit/autopsy/ingest/Bundle_ja.properties @@ -56,10 +56,6 @@ IngestMonitor.mgrErrMsg.lowDiskSpace.msg=\u30c7\u30a3\u30b9\u30af{0}\u306e\u30c7 IngestMonitor.mgrErrMsg.lowDiskSpace.title=\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u304c\u4e2d\u6b62\u3055\u308c\u307e\u3057\u305f\u30fc{0}\u306e\u30c7\u30a3\u30b9\u30af\u9818\u57df\u4e0d\u8db3 IngestScheduler.DataSourceScheduler.toString.size=DataSourceQueue, \u30b5\u30a4\u30ba\uff1a OpenIDE-Module-Name=\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8 -IngestManager.StartIngestJobsTask.run.progress.msg1={0}\u306e\u30c7\u30fc\u30bf\u30bd\u30fc\u30b9\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u30bf\u30b9\u30af -IngestManager.StartIngestJobsTask.run.progress.msg2={0}\u306e\u30c7\u30fc\u30bf\u30bd\u30fc\u30b9\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u30bf\u30b9\u30af -IngestManager.StartIngestJobsTask.run.progress.msg3={0}\u306e\u30c7\u30fc\u30bf\u30bd\u30fc\u30b9\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u30bf\u30b9\u30af -IngestManager.StartIngestJobsTask.run.progress.msg4={0}\u306e\u30c7\u30fc\u30bf\u30bd\u30fc\u30b9\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u30bf\u30b9\u30af IngestManager.StartIngestJobsTask.run.startupErr.dlgErrorList=\u30a8\u30e9\u30fc\uff1a\ \ {0} diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleProgress.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleProgress.java index 39fbca98fe..db2c4b3e37 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleProgress.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleProgress.java @@ -18,16 +18,18 @@ */ package org.sleuthkit.autopsy.ingest; +import org.netbeans.api.progress.ProgressHandle; + /** * Used by data source ingest modules to report progress. */ public class DataSourceIngestModuleProgress { - private final IngestJob ingestJob; + private final ProgressHandle progress; private final String moduleDisplayName; - DataSourceIngestModuleProgress(IngestJob ingestJob, String moduleDisplayName) { - this.ingestJob = ingestJob; + DataSourceIngestModuleProgress(ProgressHandle progress, String moduleDisplayName) { + this.progress = progress; this.moduleDisplayName = moduleDisplayName; } @@ -40,7 +42,7 @@ public class DataSourceIngestModuleProgress { * data source. */ public void switchToDeterminate(int workUnits) { - ingestJob.getDataSourceTaskProgressBar().switchToDeterminate(workUnits); + progress.switchToDeterminate(workUnits); } /** @@ -48,7 +50,7 @@ public class DataSourceIngestModuleProgress { * the total work units to process the data source is unknown. */ public void switchToIndeterminate() { - ingestJob.getDataSourceTaskProgressBar().switchToIndeterminate(); + progress.switchToIndeterminate(); } /** @@ -58,6 +60,6 @@ public class DataSourceIngestModuleProgress { * @param workUnits Number of work units performed so far by the module. */ public void progress(int workUnits) { - ingestJob.getDataSourceTaskProgressBar().progress(this.moduleDisplayName, workUnits); + progress.progress(this.moduleDisplayName, workUnits); } } \ No newline at end of file diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java index b824ffe615..86d001f1f8 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java @@ -22,6 +22,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import org.netbeans.api.progress.ProgressHandle; import org.sleuthkit.datamodel.Content; /** @@ -30,12 +31,12 @@ import org.sleuthkit.datamodel.Content; */ final class DataSourceIngestPipeline { - private final IngestJob job; + private final IngestJobContext context; private final List moduleTemplates; private List modules = new ArrayList<>(); - DataSourceIngestPipeline(IngestJob job, List moduleTemplates) { - this.job = job; + DataSourceIngestPipeline(IngestJobContext context, List moduleTemplates) { + this.context = context; this.moduleTemplates = moduleTemplates; } @@ -50,7 +51,6 @@ final class DataSourceIngestPipeline { for (IngestModuleTemplate template : moduleTemplates) { if (template.isDataSourceIngestModuleTemplate()) { DataSourceIngestModuleDecorator module = new DataSourceIngestModuleDecorator(template.createDataSourceIngestModule(), template.getModuleName()); - IngestJobContext context = new IngestJobContext(job); try { module.startUp(context); modulesByClass.put(module.getClassName(), module); @@ -75,26 +75,26 @@ final class DataSourceIngestPipeline { return errors; } - List process() { + List process(Content dataSource, ProgressHandle progress) { List errors = new ArrayList<>(); for (DataSourceIngestModuleDecorator module : this.modules) { try { - module.process(job.getDataSource(), new DataSourceIngestModuleProgress(job, module.getDisplayName())); + module.process(dataSource, new DataSourceIngestModuleProgress(progress, module.getDisplayName())); } catch (Exception ex) { errors.add(new IngestModuleError(module.getDisplayName(), ex)); } - if (job.isCancelled()) { + if (context.isJobCancelled()) { break; } } return errors; } - List shutDown(boolean ingestJobCancelled) { + List shutDown() { List errors = new ArrayList<>(); for (DataSourceIngestModuleDecorator module : this.modules) { try { - module.shutDown(ingestJobCancelled); + module.shutDown(context.isJobCancelled()); } catch (Exception ex) { errors.add(new IngestModuleError(module.getDisplayName(), ex)); } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTask.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTask.java index af27d9b8c0..2ad3c644e3 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTask.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTask.java @@ -38,7 +38,7 @@ final class DataSourceIngestTask { return dataSource; } - void execute() { - ingestJob.process(); + void execute() throws InterruptedException { + ingestJob.process(dataSource); } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTaskScheduler.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTaskScheduler.java index 22a6124396..ca22a6267b 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTaskScheduler.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTaskScheduler.java @@ -32,26 +32,30 @@ final class DataSourceIngestTaskScheduler { private DataSourceIngestTaskScheduler() { } - synchronized void addTask(DataSourceIngestTask task) throws InterruptedException { - task.getIngestJob().notifyTaskPending(); - try { - tasks.put(task); - } - catch (InterruptedException ex) { - // RJCTOD: Need a safety notification to undo above + synchronized void addTask(DataSourceIngestTask task) { + // The capacity of the tasks queue is not bounded, so the call + // to put() should not block except for normal synchronized access. + // Still, notify the job that the task has been added first so that + // the take() of the task cannot occur before the notification. + task.getIngestJob().notifyTaskAdded(); + + // If the thread executing this code is ever interrupted, it is + // because the number of ingest threads has been decreased while + // ingest jobs are running. This thread will exit in an orderly fashion, + // but the task still needs to be enqueued rather than lost. + while (true) { + try { + tasks.put(task); + break; + } catch (InterruptedException ex) { + // Reset the interrupted status of the thread so the orderly + // exit can occur in the intended place. + Thread.currentThread().interrupt(); + } } } DataSourceIngestTask getNextTask() throws InterruptedException { - return tasks.take(); - } - - boolean hasTasksForIngestJob(long jobId) { - for (DataSourceIngestTask task : tasks) { - if (task.getIngestJobId() == jobId) { - return true; - } - } - return false; + return tasks.take(); } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java index 2cd7d838e4..1b3743e616 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java @@ -30,12 +30,12 @@ import org.sleuthkit.datamodel.AbstractFile; */ final class FileIngestPipeline { - private final IngestJob job; + private final IngestJobContext context; private final List moduleTemplates; private List modules = new ArrayList<>(); - FileIngestPipeline(IngestJob task, List moduleTemplates) { - this.job = task; + FileIngestPipeline(IngestJobContext context, List moduleTemplates) { + this.context = context; this.moduleTemplates = moduleTemplates; } @@ -50,7 +50,6 @@ final class FileIngestPipeline { for (IngestModuleTemplate template : moduleTemplates) { if (template.isFileIngestModuleTemplate()) { FileIngestModuleDecorator module = new FileIngestModuleDecorator(template.createFileIngestModule(), template.getModuleName()); - IngestJobContext context = new IngestJobContext(job); try { module.startUp(context); modulesByClass.put(module.getClassName(), module); @@ -83,22 +82,22 @@ final class FileIngestPipeline { } catch (Exception ex) { errors.add(new IngestModuleError(module.getDisplayName(), ex)); } - if (job.isCancelled()) { + if (context.isJobCancelled()) { break; } } file.close(); - if (!job.isCancelled()) { - IngestManager.fireFileIngestDone(file.getId()); + if (!context.isJobCancelled()) { + IngestManager.getInstance().fireFileIngestDone(file.getId()); } return errors; } - List shutDown(boolean ingestJobCancelled) { + List shutDown() { List errors = new ArrayList<>(); for (FileIngestModuleDecorator module : this.modules) { try { - module.shutDown(ingestJobCancelled); + module.shutDown(context.isJobCancelled()); } catch (Exception ex) { errors.add(new IngestModuleError(module.getDisplayName(), ex)); } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java index db0edf9edf..54ed9a8501 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java @@ -19,17 +19,16 @@ package org.sleuthkit.autopsy.ingest; import java.util.Objects; -import java.util.logging.Level; import org.sleuthkit.datamodel.AbstractFile; -import org.sleuthkit.datamodel.TskCoreException; final class FileIngestTask { - final AbstractFile file; - private final IngestJob ingestJob; - FileIngestTask(AbstractFile file, IngestJob task) { - this.file = file; + private final IngestJob ingestJob; + private final AbstractFile file; + + FileIngestTask(IngestJob task, AbstractFile file) { this.ingestJob = task; + this.file = file; } public IngestJob getIngestJob() { @@ -39,30 +38,11 @@ final class FileIngestTask { public AbstractFile getFile() { return file; } - - void execute(long threadId) { + + void execute() throws InterruptedException { ingestJob.process(file); } - @Override - public String toString() { //RJCTODO: May not keep this - try { - return "ProcessTask{" + "file=" + file.getId() + ": " + file.getUniquePath() + "}"; // + ", dataSourceTask=" + dataSourceTask + '}'; - } catch (TskCoreException ex) { - // RJCTODO -// FileIngestTaskScheduler.logger.log(Level.SEVERE, "Cound not get unique path of file in queue, ", ex); //NON-NLS - } - return "ProcessTask{" + "file=" + file.getId() + ": " + file.getName() + '}'; - } - - /** - * two process tasks are equal when the file/dir and modules are the - * same this enables are not to queue up the same file/dir, modules - * tuples into the root dir set - * - * @param obj - * @return - */ @Override public boolean equals(Object obj) { if (obj == null) { @@ -71,13 +51,11 @@ final class FileIngestTask { if (getClass() != obj.getClass()) { return false; } - final FileIngestTask other = (FileIngestTask) obj; - if (this.file != other.file && (this.file == null || !this.file.equals(other.file))) { + FileIngestTask other = (FileIngestTask) obj; + if (this.ingestJob != other.ingestJob && (this.ingestJob == null || !this.ingestJob.equals(other.ingestJob))) { return false; } - IngestJob thisTask = this.getIngestJob(); - IngestJob otherTask = other.getIngestJob(); - if (thisTask != otherTask && (thisTask == null || !thisTask.equals(otherTask))) { + if (this.file != other.file && (this.file == null || !this.file.equals(other.file))) { return false; } return true; @@ -86,8 +64,8 @@ final class FileIngestTask { @Override public int hashCode() { int hash = 5; - hash = 47 * hash + Objects.hashCode(this.file); hash = 47 * hash + Objects.hashCode(this.ingestJob); + hash = 47 * hash + Objects.hashCode(this.file); return hash; - } + } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTaskScheduler.java b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTaskScheduler.java index 280012ebf8..fcbceab6ec 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTaskScheduler.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/FileIngestTaskScheduler.java @@ -43,23 +43,20 @@ import org.sleuthkit.datamodel.VirtualDirectory; final class FileIngestTaskScheduler { private static final Logger logger = Logger.getLogger(FileIngestTaskScheduler.class.getName()); - private static FileIngestTaskScheduler instance; + private static FileIngestTaskScheduler instance = new FileIngestTaskScheduler(); private final TreeSet rootDirectoryTasks = new TreeSet<>(new RootDirectoryTaskComparator()); private final List directoryTasks = new ArrayList<>(); - private final LinkedBlockingQueue fileTasks = new LinkedBlockingQueue<>(); // Unlimited capacity + private final LinkedBlockingQueue fileTasks = new LinkedBlockingQueue<>(); private static final int FAT_NTFS_FLAGS = TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT12.getValue() | TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT16.getValue() | TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT32.getValue() | TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_NTFS.getValue(); - static synchronized FileIngestTaskScheduler getInstance() { - if (instance == null) { - instance = new FileIngestTaskScheduler(); - } + static FileIngestTaskScheduler getInstance() { return instance; } private FileIngestTaskScheduler() { } - synchronized void addTasks(IngestJob dataSourceTask, Content dataSource) { + synchronized void addTasks(IngestJob job, Content dataSource) throws InterruptedException { Collection rootObjects = dataSource.accept(new GetRootDirectoryVisitor()); List firstLevelFiles = new ArrayList<>(); if (rootObjects.isEmpty() && dataSource instanceof AbstractFile) { @@ -87,9 +84,9 @@ final class FileIngestTaskScheduler { } } for (AbstractFile firstLevelFile : firstLevelFiles) { - FileIngestTask fileTask = new FileIngestTask(firstLevelFile, dataSourceTask); + FileIngestTask fileTask = new FileIngestTask(job, firstLevelFile); if (shouldEnqueueTask(fileTask)) { - rootDirectoryTasks.add(fileTask); + addTaskToRootDirectoryQueue(fileTask); } } @@ -97,16 +94,9 @@ final class FileIngestTaskScheduler { updateQueues(); } - synchronized void addTask(IngestJob ingestJob, AbstractFile file) { - try { - FileIngestTask fileTask = new FileIngestTask(file, ingestJob); - if (shouldEnqueueTask(fileTask)) { - fileTask.getIngestJob().notifyTaskPending(); - fileTasks.put(fileTask); // Queue has unlimited capacity, does not block. - } - } catch (InterruptedException ex) { - // RJCTODO: Perhaps this is the convenience method? - // RJCTODO: Need undo + synchronized void addTask(FileIngestTask task) { + if (shouldEnqueueTask(task)) { + addTaskToFileQueue(task); } } @@ -116,29 +106,7 @@ final class FileIngestTaskScheduler { return task; } - synchronized boolean hasTasksForJob(long ingestJobId) { - for (FileIngestTask task : rootDirectoryTasks) { - if (task.getIngestJob().getJobId() == ingestJobId) { - return true; - } - } - - for (FileIngestTask task : directoryTasks) { - if (task.getIngestJob().getJobId() == ingestJobId) { - return true; - } - } - - for (FileIngestTask task : fileTasks) { - if (task.getIngestJob().getJobId() == ingestJobId) { - return true; - } - } - - return false; - } - - private void updateQueues() { + private void updateQueues() throws InterruptedException { // we loop because we could have a directory that has all files // that do not get enqueued while (true) { @@ -152,23 +120,15 @@ final class FileIngestTaskScheduler { if (rootDirectoryTasks.isEmpty()) { return; } - FileIngestTask rootTask = this.rootDirectoryTasks.pollFirst(); - directoryTasks.add(rootTask); + addTaskToDirectoryQueue(rootDirectoryTasks.pollFirst(), false); } //pop and push AbstractFile directory children if any //add the popped and its leaf children onto cur file list FileIngestTask parentTask = directoryTasks.remove(directoryTasks.size() - 1); - final AbstractFile parentFile = parentTask.file; + final AbstractFile parentFile = parentTask.getFile(); // add itself to the file list if (shouldEnqueueTask(parentTask)) { - // RJCTODO - try { - parentTask.getIngestJob().notifyTaskPending(); - fileTasks.put(parentTask); - } catch (InterruptedException ex) { - // RJCTODO: Maybe make a convenience method - // RJCTODO: Need undo - } + addTaskToFileQueue(parentTask); } // add its children to the file and directory lists try { @@ -176,18 +136,11 @@ final class FileIngestTaskScheduler { for (Content c : children) { if (c instanceof AbstractFile) { AbstractFile childFile = (AbstractFile) c; - FileIngestTask childTask = new FileIngestTask(childFile, parentTask.getIngestJob()); + FileIngestTask childTask = new FileIngestTask(parentTask.getIngestJob(), childFile); if (childFile.hasChildren()) { - this.directoryTasks.add(childTask); + addTaskToDirectoryQueue(childTask, true); } else if (shouldEnqueueTask(childTask)) { - // RJCTODO - try { - childTask.getIngestJob().notifyTaskPending(); - fileTasks.put(childTask); - } catch (InterruptedException ex) { - // RJCTODO: Maybe make a convenience method - // RJCTODO: Need undo - } + addTaskToFileQueue(childTask); } } } @@ -197,10 +150,39 @@ final class FileIngestTaskScheduler { } } - synchronized void emptyQueues() { // RJCTODO: Perhaps clear all... - this.rootDirectoryTasks.clear(); - this.directoryTasks.clear(); - this.fileTasks.clear(); + private void addTaskToRootDirectoryQueue(FileIngestTask task) { + directoryTasks.add(task); + task.getIngestJob().notifyTaskAdded(); + } + + private void addTaskToDirectoryQueue(FileIngestTask task, boolean isNewTask) { + if (isNewTask) { + directoryTasks.add(task); + } + task.getIngestJob().notifyTaskAdded(); + } + + private void addTaskToFileQueue(FileIngestTask task) { + // The capacity of the file tasks queue is not bounded, so the call + // to put() should not block except for normal synchronized access. + // Still, notify the job that the task has been added first so that + // the take() of the task cannot occur before the notification. + task.getIngestJob().notifyTaskAdded(); + + // If the thread executing this code is ever interrupted, it is + // because the number of ingest threads has been decreased while + // ingest jobs are running. This thread will exit in an orderly fashion, + // but the task still needs to be enqueued rather than lost. + while (true) { + try { + fileTasks.put(task); + break; + } catch (InterruptedException ex) { + // Reset the interrupted status of the thread so the orderly + // exit can occur in the intended place. + Thread.currentThread().interrupt(); + } + } } /** @@ -211,7 +193,7 @@ final class FileIngestTaskScheduler { * @return true if should be enqueued, false otherwise */ private static boolean shouldEnqueueTask(final FileIngestTask processTask) { - final AbstractFile aFile = processTask.file; + final AbstractFile aFile = processTask.getFile(); //if it's unalloc file, skip if so scheduled if (processTask.getIngestJob().shouldProcessUnallocatedSpace() == false && aFile.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS)) { return false; @@ -320,10 +302,10 @@ final class FileIngestTaskScheduler { @Override public int compare(FileIngestTask q1, FileIngestTask q2) { - AbstractFilePriority.Priority p1 = AbstractFilePriority.getPriority(q1.file); - AbstractFilePriority.Priority p2 = AbstractFilePriority.getPriority(q2.file); + AbstractFilePriority.Priority p1 = AbstractFilePriority.getPriority(q1.getFile()); + AbstractFilePriority.Priority p2 = AbstractFilePriority.getPriority(q2.getFile()); if (p1 == p2) { - return (int) (q2.file.getId() - q1.file.getId()); + return (int) (q2.getFile().getId() - q1.getFile().getId()); } else { return p2.ordinal() - p1.ordinal(); } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java index 8982be2061..a07c135e89 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java @@ -24,10 +24,12 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; +import java.util.logging.Level; import org.netbeans.api.progress.ProgressHandle; import org.netbeans.api.progress.ProgressHandleFactory; import org.openide.util.Cancellable; import org.openide.util.NbBundle; +import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.Content; @@ -37,33 +39,48 @@ import org.sleuthkit.datamodel.Content; */ final class IngestJob { + private static final Logger logger = Logger.getLogger(IngestManager.class.getName()); private static final AtomicLong nextIngestJobId = new AtomicLong(0L); - private static final ConcurrentHashMap ingestJobs = new ConcurrentHashMap<>(); // Maps job ids to jobs. - private final long jobId; - private final Content dataSource; + private static final ConcurrentHashMap ingestJobsById = new ConcurrentHashMap<>(); + private final long id; + private final Content rootDataSource; private final List ingestModuleTemplates; private final boolean processUnallocatedSpace; private final LinkedBlockingQueue dataSourceIngestPipelines = new LinkedBlockingQueue<>(); private final LinkedBlockingQueue fileIngestPipelines = new LinkedBlockingQueue<>(); private final AtomicInteger tasksInProgress = new AtomicInteger(0); private final AtomicLong processedFiles = new AtomicLong(0L); + private final AtomicLong filesToIngestEstimate = new AtomicLong(0L); private ProgressHandle dataSourceTasksProgress; private ProgressHandle fileTasksProgress; - private long filesToIngestEstimate = 0; private volatile boolean cancelled; - static List startIngestJob(Content dataSource, List ingestModuleTemplates, boolean processUnallocatedSpace) { // RJCTODO: return errors + /** + * Creates an ingest job for a data source and starts up the ingest + * pipelines for the job. + * + * @param rootDataSource The data source to ingest. + * @param ingestModuleTemplates The ingest module templates to use to create + * the ingest pipelines for the job. + * @param processUnallocatedSpace Whether or not the job should include + * processing of unallocated space. + * @return A collection of ingest module startUp up errors, empty on + * success. + * @throws InterruptedException + */ + static List startJob(Content dataSource, List ingestModuleTemplates, boolean processUnallocatedSpace) throws InterruptedException { long jobId = nextIngestJobId.incrementAndGet(); IngestJob ingestJob = new IngestJob(jobId, dataSource, ingestModuleTemplates, processUnallocatedSpace); - List errors = ingestJob.start(); + List errors = ingestJob.startUp(); if (errors.isEmpty()) { - ingestJobs.put(jobId, ingestJob); + ingestJobsById.put(jobId, ingestJob); + IngestManager.getInstance().fireIngestJobStarted(jobId); } return errors; } static boolean jobsAreRunning() { - for (IngestJob job : ingestJobs.values()) { + for (IngestJob job : ingestJobsById.values()) { if (!job.isCancelled()) { return true; } @@ -71,79 +88,72 @@ final class IngestJob { return false; } - static void addFileToIngestJob(long ingestJobId, AbstractFile file) { // RJCTODO: Move back to IngestManager - IngestJob job = ingestJobs.get(ingestJobId); + static void addFileToJob(long ingestJobId, AbstractFile file) { // RJCTODO: Just one at a time? + IngestJob job = ingestJobsById.get(ingestJobId); if (job != null) { - FileIngestTaskScheduler.getInstance().addTask(job, file); +// long adjustedFilesCount = job.filesToIngestEstimate.incrementAndGet(); // RJCTODO: Not the best name now? +// job.fileTasksProgress.switchToIndeterminate(); // RJCTODO: Comment this stuff +// job.fileTasksProgress.switchToDeterminate((int) adjustedFilesCount); +// job.fileTasksProgress.progress(job.processedFiles.intValue()); + FileIngestTaskScheduler.getInstance().addTask(new FileIngestTask(job, file)); } } - static void cancelAllIngestJobs() { - for (IngestJob job : ingestJobs.values()) { + static void cancelAllJobs() { + for (IngestJob job : ingestJobsById.values()) { job.cancel(); } } private IngestJob(long id, Content dataSource, List ingestModuleTemplates, boolean processUnallocatedSpace) { - this.jobId = id; - this.dataSource = dataSource; + this.id = id; + this.rootDataSource = dataSource; this.ingestModuleTemplates = ingestModuleTemplates; this.processUnallocatedSpace = processUnallocatedSpace; this.cancelled = false; } - long getJobId() { - return jobId; - } - - Content getDataSource() { - return dataSource; + long getId() { + return id; } boolean shouldProcessUnallocatedSpace() { return processUnallocatedSpace; } - List start() { + List startUp() throws InterruptedException { List errors = startUpIngestPipelines(); if (errors.isEmpty()) { - DataSourceIngestTaskScheduler.getInstance().addTask(new DataSourceIngestTask(this, dataSource)); - FileIngestTaskScheduler.getInstance().addTasks(this, dataSource); startDataSourceIngestProgressBar(); startFileIngestProgressBar(); + FileIngestTaskScheduler.getInstance().addTasks(this, rootDataSource); // RJCTODO: Think about this ordering "solution" for small images + DataSourceIngestTaskScheduler.getInstance().addTask(new DataSourceIngestTask(this, rootDataSource)); } return errors; } - private List startUpIngestPipelines() { + private List startUpIngestPipelines() throws InterruptedException { + IngestJobContext context = new IngestJobContext(this); List errors = new ArrayList<>(); int maxNumberOfPipelines = IngestManager.getMaxNumberOfDataSourceIngestThreads(); for (int i = 0; i < maxNumberOfPipelines; ++i) { - DataSourceIngestPipeline pipeline = new DataSourceIngestPipeline(this, ingestModuleTemplates); + DataSourceIngestPipeline pipeline = new DataSourceIngestPipeline(context, ingestModuleTemplates); errors.addAll(pipeline.startUp()); - try { - dataSourceIngestPipelines.put(pipeline); - } catch (InterruptedException ex) { - // RJCTODO: log unexpected block and interrupt, or throw - } - if (errors.isEmpty()) { - // No need to accumulate presumably redundant erros. + dataSourceIngestPipelines.put(pipeline); + if (!errors.isEmpty()) { + // No need to accumulate presumably redundant errors. break; } } maxNumberOfPipelines = IngestManager.getMaxNumberOfFileIngestThreads(); for (int i = 0; i < maxNumberOfPipelines; ++i) { - FileIngestPipeline pipeline = new FileIngestPipeline(this, ingestModuleTemplates); + FileIngestPipeline pipeline = new FileIngestPipeline(context, ingestModuleTemplates); errors.addAll(pipeline.startUp()); - try { - fileIngestPipelines.put(pipeline); - } catch (InterruptedException ex) { - // RJCTODO: log unexpected block and interrupt, or throw - } - if (errors.isEmpty()) { - // No need to accumulate presumably redundant erros. + fileIngestPipelines.put(pipeline); + if (!errors.isEmpty()) { + // No need to accumulate presumably redundant errors. break; } } @@ -154,7 +164,7 @@ final class IngestJob { private void startDataSourceIngestProgressBar() { final String displayName = NbBundle.getMessage(this.getClass(), "IngestJob.progress.dataSourceIngest.displayName", - dataSource.getName()); + rootDataSource.getName()); dataSourceTasksProgress = ProgressHandleFactory.createHandle(displayName, new Cancellable() { @Override public boolean cancel() { @@ -169,13 +179,13 @@ final class IngestJob { } }); dataSourceTasksProgress.start(); - dataSourceTasksProgress.switchToIndeterminate(); // RJCTODO: check out the logic in the pipleine class + dataSourceTasksProgress.switchToIndeterminate(); } private void startFileIngestProgressBar() { final String displayName = NbBundle.getMessage(this.getClass(), "IngestJob.progress.fileIngest.displayName", - dataSource.getName()); + rootDataSource.getName()); fileTasksProgress = ProgressHandleFactory.createHandle(displayName, new Cancellable() { @Override public boolean cancel() { @@ -188,77 +198,78 @@ final class IngestJob { return true; } }); - filesToIngestEstimate = dataSource.accept(new GetFilesCountVisitor()); + long initialFilesCount = rootDataSource.accept(new GetFilesCountVisitor()); + filesToIngestEstimate.getAndAdd(initialFilesCount); fileTasksProgress.start(); - fileTasksProgress.switchToDeterminate((int) filesToIngestEstimate); + fileTasksProgress.switchToDeterminate((int) initialFilesCount); // RJCTODO: This cast is troublesome, can use intValue } /** - * Called by the ingest task schedulers when an ingest task for this ingest - * job is added to the scheduler's task queue. + * Called by the ingest task schedulers when an ingest task is added to this + * ingest job. */ - void notifyTaskScheduled() { - // Increment the task counter when a task is scheduled so that there is - // a persistent record of the task's existence even after it is removed - // from the scheduler by an ingest thread. The task counter is used by - // the job to determine when it is done. + void notifyTaskAdded() { tasksInProgress.incrementAndGet(); } - /** - * Called by the ingest schedulers as an "undo" operation for - * notifyTaskScheduled(). - */ - void notifyTaskCompleted() { - // Decrement the task counter when a task is discarded by a scheduler. - // The task counter is used by the job to determine when it is done. - tasksInProgress.decrementAndGet(); + void process(Content dataSource) throws InterruptedException { + // If the job is not cancelled, complete the task, otherwise just flush + // it. In either case, the task counter needs to be decremented and the + // shut down check needs to occur. + if (!isCancelled()) { + List errors = new ArrayList<>(); + DataSourceIngestPipeline pipeline = dataSourceIngestPipelines.take(); + errors.addAll(pipeline.process(dataSource, dataSourceTasksProgress)); + if (!errors.isEmpty()) { + logIngestModuleErrors(errors); + } + dataSourceIngestPipelines.put(pipeline); + } + shutDownIfAllTasksCompleted(); } - void process() throws InterruptedException { + void process(AbstractFile file) throws InterruptedException { + // If the job is not cancelled, complete the task, otherwise just flush + // it. In either case, the task counter needs to be decremented and the + // shut down check needs to occur. if (!isCancelled()) { - try { - DataSourceIngestPipeline pipeline = dataSourceIngestPipelines.take(); - pipeline.process(); // RJCTODO: Pass data source through? - dataSourceIngestPipelines.put(pipeline); - } catch (InterruptedException ex) { - // RJCTODO: + List errors = new ArrayList<>(); + FileIngestPipeline pipeline = fileIngestPipelines.take(); +// fileTasksProgress.progress(file.getName(), (int) processedFiles.incrementAndGet()); RJCTODO + errors.addAll(pipeline.process(file)); + fileIngestPipelines.put(pipeline); + if (!errors.isEmpty()) { + logIngestModuleErrors(errors); } } - ifCompletedShutDown(); + shutDownIfAllTasksCompleted(); } - void process(AbstractFile file) { - if (!isCancelled()) { - try { - FileIngestPipeline pipeline = fileIngestPipelines.take(); - fileTasksProgress.progress(file.getName(), (int) processedFiles.incrementAndGet()); - pipeline.process(file); - fileIngestPipelines.put(pipeline); - } catch (InterruptedException ex) { - // RJCTODO: Log block and interrupt - } - } - ifCompletedShutDown(); - } - - void ifCompletedShutDown() { + private void shutDownIfAllTasksCompleted() { if (tasksInProgress.decrementAndGet() == 0) { + List errors = new ArrayList<>(); while (!dataSourceIngestPipelines.isEmpty()) { DataSourceIngestPipeline pipeline = dataSourceIngestPipelines.poll(); - pipeline.shutDown(cancelled); + errors.addAll(pipeline.shutDown()); } while (!fileIngestPipelines.isEmpty()) { FileIngestPipeline pipeline = fileIngestPipelines.poll(); - pipeline.shutDown(cancelled); + errors.addAll(pipeline.shutDown()); } - ingestJobs.remove(jobId); - IngestManager.getInstance().fireIngestJobCompleted(jobId); + fileTasksProgress.finish(); + dataSourceTasksProgress.finish(); + ingestJobsById.remove(id); + if (!errors.isEmpty()) { + logIngestModuleErrors(errors); + } + IngestManager.getInstance().fireIngestJobCompleted(id); } } - ProgressHandle getDataSourceTaskProgressBar() { - return dataSourceTasksProgress; // RJCTODO: Should just pass the progress handle or the object to the pipeline + private void logIngestModuleErrors(List errors) { + for (IngestModuleError error : errors) { + logger.log(Level.SEVERE, error.getModuleDisplayName() + " experienced an error", error.getModuleError()); + } } boolean isCancelled() { @@ -267,7 +278,7 @@ final class IngestJob { void cancel() { cancelled = true; - fileTasksProgress.finish(); - IngestManager.getInstance().fireIngestJobCancelled(jobId); + fileTasksProgress.finish(); // RJCTODO: What about the other progress bar? + IngestManager.getInstance().fireIngestJobCancelled(id); } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java index 6a9b4a9454..b8d4493de4 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java @@ -39,7 +39,7 @@ public final class IngestJobContext { * @return The ingest job identifier. */ public long getJobId() { - return this.ingestJob.getJobId(); + return this.ingestJob.getId(); } /** @@ -60,7 +60,7 @@ public final class IngestJobContext { */ public void addFiles(List files) { for (AbstractFile file : files) { - IngestJob.addFileToIngestJob(ingestJob.getJobId(), file); + IngestJob.addFileToJob(ingestJob.getId(), file); } } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java index a5b0da296c..36cc73fefb 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java @@ -21,7 +21,9 @@ package org.sleuthkit.autopsy.ingest; import java.beans.PropertyChangeListener; import java.beans.PropertyChangeSupport; import java.util.List; +import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; @@ -51,14 +53,15 @@ public class IngestManager { private static final Logger logger = Logger.getLogger(IngestManager.class.getName()); private static final Preferences userPreferences = NbPreferences.forModule(IngestManager.class); private static final IngestManager instance = new IngestManager(); - private final PropertyChangeSupport pcs = new PropertyChangeSupport(IngestManager.class); + private final PropertyChangeSupport ingestJobEventPublisher = new PropertyChangeSupport(IngestManager.class); + private final PropertyChangeSupport ingestModuleEventPublisher = new PropertyChangeSupport(IngestManager.class); private final IngestMonitor ingestMonitor = new IngestMonitor(); private final ExecutorService startIngestJobsThreadPool = Executors.newSingleThreadExecutor(); - private final ConcurrentHashMap> startIngestJobThreads = new ConcurrentHashMap<>(); // Maps thread ids to cancellation handles. private final ExecutorService dataSourceIngestThreadPool = Executors.newSingleThreadExecutor(); - private final ConcurrentHashMap> dataSourceIngestThreads = new ConcurrentHashMap<>(); // Maps thread ids to cancellation handles. private final ExecutorService fileIngestThreadPool = Executors.newFixedThreadPool(MAX_NUMBER_OF_FILE_INGEST_THREADS); - private final ExecutorService fireIngestJobEventsThreadPool = Executors.newSingleThreadExecutor(); + private final ExecutorService fireIngestEventsThreadPool = Executors.newSingleThreadExecutor(); + private final ConcurrentHashMap> startIngestJobThreads = new ConcurrentHashMap<>(); // Maps thread ids to cancellation handles. + private final ConcurrentHashMap> dataSourceIngestThreads = new ConcurrentHashMap<>(); // Maps thread ids to cancellation handles. private final ConcurrentHashMap> fileIngestThreads = new ConcurrentHashMap<>(); // Maps thread ids to cancellation handles. private final AtomicLong nextThreadId = new AtomicLong(0L); private volatile IngestMessageTopComponent ingestMessageBox; @@ -154,7 +157,7 @@ public class IngestManager { */ private void startDataSourceIngestThread() { long threadId = nextThreadId.incrementAndGet(); - Future handle = dataSourceIngestThreadPool.submit(new DataSourceIngestThread(threadId)); + Future handle = dataSourceIngestThreadPool.submit(new DataSourceIngestThread()); dataSourceIngestThreads.put(threadId, handle); } @@ -164,7 +167,7 @@ public class IngestManager { */ private void startFileIngestThread() { long threadId = nextThreadId.incrementAndGet(); - Future handle = fileIngestThreadPool.submit(new FileIngestThread(threadId)); + Future handle = fileIngestThreadPool.submit(new FileIngestThread()); fileIngestThreads.put(threadId, handle); } @@ -200,35 +203,48 @@ public class IngestManager { } public void cancelAllIngestJobs() { - cancelStartIngestJobsTasks(); - IngestJob.cancelAllIngestJobs(); - } - - private void cancelStartIngestJobsTasks() { - for (Future future : startIngestJobThreads.values()) { - future.cancel(true); + // Stop creating new ingest jobs. + for (Future handle : startIngestJobThreads.values()) { + handle.cancel(true); + try { + // Blocks until the job starting thread responds. The thread + // removes itself from this collection, which does not disrupt + // this loop since the collection is a ConcurrentHashMap. + handle.get(); + } catch (InterruptedException | ExecutionException ex) { + // This should never happen, something is awry, but everything + // should be o.k. anyway. + logger.log(Level.SEVERE, "Unexpected thread interrupt", ex); + } } - startIngestJobThreads.clear(); + startIngestJobThreads.clear(); // Make sure. + + // Cancel all the jobs already created. This will make the the ingest + // threads flush out any lingering ingest tasks without processing them. + IngestJob.cancelAllJobs(); } /** * Ingest events. */ - public enum IngestEvent { // RJCTODO: Update comments if time permits + public enum IngestEvent { /** * Property change event fired when an ingest job is started. The old - * and new values of the PropertyChangeEvent object are set to null. + * value of the PropertyChangeEvent object is set to the ingest job id, + * and the new value is set to null. */ INGEST_JOB_STARTED, /** * Property change event fired when an ingest job is completed. The old - * and new values of the PropertyChangeEvent object are set to null. + * value of the PropertyChangeEvent object is set to the ingest job id, + * and the new value is set to null. */ INGEST_JOB_COMPLETED, /** * Property change event fired when an ingest job is canceled. The old - * and new values of the PropertyChangeEvent object are set to null. + * value of the PropertyChangeEvent object is set to the ingest job id, + * and the new value is set to null. */ INGEST_JOB_CANCELLED, /** @@ -253,21 +269,59 @@ public class IngestManager { }; /** - * Add an ingest event property change listener. + * Add an ingest job event property change listener. * * @param listener The PropertyChangeListener to register. */ - public void addPropertyChangeListener(final PropertyChangeListener listener) { - pcs.addPropertyChangeListener(listener); + public void addIngestJobEventListener(final PropertyChangeListener listener) { + ingestJobEventPublisher.addPropertyChangeListener(listener); } /** - * Remove an ingest event property change listener. + * Remove an ingest job event property change listener. * * @param listener The PropertyChangeListener to unregister. */ - public void removePropertyChangeListener(final PropertyChangeListener listener) { - pcs.removePropertyChangeListener(listener); + public void removeIngestJobEventListener(final PropertyChangeListener listener) { + ingestJobEventPublisher.removePropertyChangeListener(listener); + } + + /** + * Add an ingest module event property change listener. + * + * @param listener The PropertyChangeListener to register. + */ + public void addIngestModuleEventListener(final PropertyChangeListener listener) { + ingestModuleEventPublisher.addPropertyChangeListener(listener); + } + + /** + * Remove an ingest module event property change listener. + * + * @param listener The PropertyChangeListener to unregister. + */ + public void removeIngestModuleEventListener(final PropertyChangeListener listener) { + ingestModuleEventPublisher.removePropertyChangeListener(listener); + } + + /** + * Add an ingest module event property change listener. + * + * @deprecated + * @param listener The PropertyChangeListener to register. + */ + public static void addPropertyChangeListener(final PropertyChangeListener listener) { + instance.ingestModuleEventPublisher.addPropertyChangeListener(listener); + } + + /** + * Remove an ingest module event property change listener. + * + * @deprecated + * @param listener The PropertyChangeListener to unregister. + */ + public static void removePropertyChangeListener(final PropertyChangeListener listener) { + instance.ingestModuleEventPublisher.removePropertyChangeListener(listener); } /** @@ -276,7 +330,7 @@ public class IngestManager { * @param ingestJobId The ingest job id. */ void fireIngestJobStarted(long ingestJobId) { - fireIngestJobEventsThreadPool.submit(new FireIngestEventThread(IngestEvent.INGEST_JOB_STARTED, ingestJobId, null)); + fireIngestEventsThreadPool.submit(new FireIngestEventThread(ingestJobEventPublisher, IngestEvent.INGEST_JOB_STARTED, ingestJobId, null)); } /** @@ -285,7 +339,7 @@ public class IngestManager { * @param ingestJobId The ingest job id. */ void fireIngestJobCompleted(long ingestJobId) { - fireIngestJobEventsThreadPool.submit(new FireIngestEventThread(IngestEvent.INGEST_JOB_COMPLETED, ingestJobId, null)); + fireIngestEventsThreadPool.submit(new FireIngestEventThread(ingestJobEventPublisher, IngestEvent.INGEST_JOB_COMPLETED, ingestJobId, null)); } /** @@ -294,7 +348,7 @@ public class IngestManager { * @param ingestJobId The ingest job id. */ void fireIngestJobCancelled(long ingestJobId) { - fireIngestJobEventsThreadPool.submit(new FireIngestEventThread(IngestEvent.INGEST_JOB_CANCELLED, ingestJobId, null)); + fireIngestEventsThreadPool.submit(new FireIngestEventThread(ingestJobEventPublisher, IngestEvent.INGEST_JOB_CANCELLED, ingestJobId, null)); } /** @@ -303,7 +357,7 @@ public class IngestManager { * @param fileId The object id of file. */ void fireFileIngestDone(long fileId) { - fireIngestJobEventsThreadPool.submit(new FireIngestEventThread(IngestEvent.FILE_DONE, fileId, null)); + fireIngestEventsThreadPool.submit(new FireIngestEventThread(ingestModuleEventPublisher, IngestEvent.FILE_DONE, fileId, null)); } /** @@ -312,7 +366,7 @@ public class IngestManager { * @param moduleDataEvent A ModuleDataEvent with the details of the posting. */ void fireIngestModuleDataEvent(ModuleDataEvent moduleDataEvent) { - fireIngestJobEventsThreadPool.submit(new FireIngestEventThread(IngestEvent.DATA, moduleDataEvent, null)); + fireIngestEventsThreadPool.submit(new FireIngestEventThread(ingestModuleEventPublisher, IngestEvent.DATA, moduleDataEvent, null)); } /** @@ -323,7 +377,7 @@ public class IngestManager { * content. */ void fireIngestModuleContentEvent(ModuleContentEvent moduleContentEvent) { - fireIngestJobEventsThreadPool.submit(new FireIngestEventThread(IngestEvent.CONTENT_CHANGED, moduleContentEvent, null)); + fireIngestEventsThreadPool.submit(new FireIngestEventThread(ingestModuleEventPublisher, IngestEvent.CONTENT_CHANGED, moduleContentEvent, null)); } /** @@ -341,7 +395,7 @@ public class IngestManager { * Get the free disk space of the drive where to which ingest data is being * written, as reported by the ingest monitor. * - * @return Free disk space, -1 if unknown // RJCTODO: What units? + * @return Free disk space, -1 if unknown */ long getFreeDiskSpace() { if (ingestMonitor != null) { @@ -352,10 +406,10 @@ public class IngestManager { } /** - * A Runnable that creates ingest jobs and submits the initial data source + * A Callable that creates ingest jobs and submits the initial data source * and file ingest tasks to the task schedulers. */ - private class StartIngestJobsThread implements Runnable { + private class StartIngestJobsThread implements Callable { private final long threadId; private final List dataSources; @@ -371,7 +425,7 @@ public class IngestManager { } @Override - public void run() { + public Void call() { try { final String displayName = NbBundle.getMessage(this.getClass(), "IngestManager.StartIngestJobsTask.run.displayName"); @@ -387,26 +441,25 @@ public class IngestManager { return true; } }); - progress.start(dataSources.size() * 2); + progress.start(dataSources.size()); if (!ingestMonitor.isRunning()) { ingestMonitor.start(); } - int workUnitsCompleted = 0; + int dataSourceProcessed = 0; for (Content dataSource : dataSources) { if (Thread.currentThread().isInterrupted()) { break; } - // Create an ingest job. - List errors = IngestJob.startIngestJob(dataSource, moduleTemplates, processUnallocatedSpace); + // Start an ingest job for the data source. + List errors = IngestJob.startJob(dataSource, moduleTemplates, processUnallocatedSpace); if (!errors.isEmpty()) { - // Report the error to the user. + // Report the errors to the user. They have already been logged. StringBuilder moduleStartUpErrors = new StringBuilder(); for (IngestModuleError error : errors) { String moduleName = error.getModuleDisplayName(); - logger.log(Level.SEVERE, "The " + moduleName + " module failed to start up", error.getModuleError()); //NON-NLS moduleStartUpErrors.append(moduleName); moduleStartUpErrors.append(": "); moduleStartUpErrors.append(error.getModuleError().getLocalizedMessage()); @@ -427,40 +480,16 @@ public class IngestManager { NbBundle.getMessage(this.getClass(), "IngestManager.StartIngestJobsTask.run.startupErr.dlgTitle"), JOptionPane.ERROR_MESSAGE); } - - fireIngestJobEventsThreadPool.submit(new FireIngestEventThread(IngestEvent.INGEST_JOB_STARTED)); - - // Queue a data source ingest task for the ingest job. - final String inputName = dataSource.getName(); - progress.progress( - NbBundle.getMessage(this.getClass(), "IngestManager.StartIngestJobsThread.run.progress.msg1", - inputName), workUnitsCompleted); - DataSourceIngestTaskScheduler.getInstance().addTask(new DataSourceIngestTask(ingestJob, ingestJob.getDataSource())); - progress.progress( - NbBundle.getMessage(this.getClass(), "IngestManager.StartIngestJobsThread.run.progress.msg2", - inputName), ++workUnitsCompleted); - - // Queue the file ingest tasks for the ingest job. - progress.progress( - NbBundle.getMessage(this.getClass(), "IngestManager.StartIngestJobsThread.run.progress.msg3", - inputName), workUnitsCompleted); - FileIngestTaskScheduler.getInstance().addTasks(ingestJob, ingestJob.getDataSource()); - progress.progress( - NbBundle.getMessage(this.getClass(), "IngestManager.StartIngestJobsThread.run.progress.msg4", - inputName), ++workUnitsCompleted); + progress.progress(++dataSourceProcessed); if (!Thread.currentThread().isInterrupted()) { break; } } - } catch (Exception ex) { - String message = String.format("StartIngestJobsTask (id=%d) caught exception", threadId); //NON-NLS - logger.log(Level.SEVERE, message, ex); - MessageNotifyUtil.Message.error( - NbBundle.getMessage(this.getClass(), "IngestManager.StartIngestJobsTask.run.catchException.msg")); } finally { progress.finish(); startIngestJobThreads.remove(threadId); + return null; } } } @@ -512,16 +541,18 @@ public class IngestManager { } /** - * A Runnable that fire ingest events to ingest manager property change + * A Runnable that fires ingest events to ingest manager property change * listeners. */ - private class FireIngestEventThread implements Runnable { + private static class FireIngestEventThread implements Runnable { + private final PropertyChangeSupport publisher; private final IngestEvent event; private final Object oldValue; private final Object newValue; - FireIngestEventThread(IngestEvent event, Object oldValue, Object newValue) { + FireIngestEventThread(PropertyChangeSupport publisher, IngestEvent event, Object oldValue, Object newValue) { + this.publisher = publisher; this.event = event; this.oldValue = oldValue; this.newValue = newValue; @@ -530,10 +561,10 @@ public class IngestManager { @Override public void run() { try { - pcs.firePropertyChange(event.toString(), oldValue, newValue); + publisher.firePropertyChange(event.toString(), oldValue, newValue); } catch (Exception e) { logger.log(Level.SEVERE, "Ingest manager listener threw exception", e); //NON-NLS - MessageNotifyUtil.Notify.show(NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr"), // RJCTODO: Oddly named strings + MessageNotifyUtil.Notify.show(NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr"), NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr.errListenToUpdates.msg"), MessageNotifyUtil.MessageType.ERROR); } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestServices.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestServices.java index c573753281..5bf4f182a7 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestServices.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestServices.java @@ -95,7 +95,7 @@ public final class IngestServices { * artifact data */ public void fireModuleDataEvent(ModuleDataEvent moduleDataEvent) { - IngestManager.fireIngestModuleDataEvent(moduleDataEvent); + IngestManager.getInstance().fireIngestModuleDataEvent(moduleDataEvent); } /** @@ -107,7 +107,7 @@ public final class IngestServices { * changed */ public void fireModuleContentEvent(ModuleContentEvent moduleContentEvent) { - IngestManager.fireIngestModuleContentEvent(moduleContentEvent); + IngestManager.getInstance().fireIngestModuleContentEvent(moduleContentEvent); } /** diff --git a/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashLookupSettingsPanel.java b/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashLookupSettingsPanel.java index 7e901fbc71..c414731036 100644 --- a/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashLookupSettingsPanel.java +++ b/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashLookupSettingsPanel.java @@ -69,7 +69,7 @@ public final class HashLookupSettingsPanel extends IngestModuleGlobalSettingsPan // Listen to the ingest modules to refresh the enabled/disabled state of // the components in sync with file ingest. - IngestManager.addPropertyChangeListener(new PropertyChangeListener() { + IngestManager.getInstance().addIngestJobEventListener(new PropertyChangeListener() { @Override public void propertyChange(PropertyChangeEvent evt) { if (isIngestJobEvent(evt)) { diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchEditListPanel.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchEditListPanel.java index 90e7f197d1..4ba2d39eb6 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchEditListPanel.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchEditListPanel.java @@ -126,7 +126,7 @@ class KeywordSearchEditListPanel extends javax.swing.JPanel implements ListSelec setButtonStates(); - IngestManager.addPropertyChangeListener(new PropertyChangeListener() { + IngestManager.getInstance().addIngestJobEventListener(new PropertyChangeListener() { @Override public void propertyChange(PropertyChangeEvent evt) { String changed = evt.getPropertyName(); diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchListsViewerPanel.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchListsViewerPanel.java index 898b21bfc5..0e8c099aac 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchListsViewerPanel.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchListsViewerPanel.java @@ -118,7 +118,7 @@ class KeywordSearchListsViewerPanel extends AbstractKeywordSearchPerformer { ingestRunning = IngestManager.getInstance().isIngestRunning(); updateComponents(); - IngestManager.addPropertyChangeListener(new PropertyChangeListener() { + IngestManager.getInstance().addIngestJobEventListener(new PropertyChangeListener() { @Override public void propertyChange(PropertyChangeEvent evt) { String changed = evt.getPropertyName(); diff --git a/c:casesSamll2Againautopsy.db b/c:casesSamll2Againautopsy.db new file mode 100755 index 0000000000..e69de29bb2