mstoeck3/autopsy-flatpak
1
0
Fork 0
mirror of https://github.com/overcuriousity/autopsy-flatpak.git synced 2025-07-12 07:56:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

Preserve intermediate state of improved ingest framework

Browse Source
This commit is contained in:
Richard Cordovano 2014-05-01 09:41:01 -04:00
parent ae1793bfd3
commit a77d566a82
18 changed files with 329 additions and 329 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java
View File

@ -131,7 +131,8 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat
private void setListener() {
Case.addPropertyChangeListener(this);// add this class to listen to any changes in the Case.java class
this.em.addPropertyChangeListener(this);
IngestManager.addPropertyChangeListener(this);
IngestManager.getInstance().addIngestJobEventListener(this);
IngestManager.getInstance().addIngestModuleEventListener(this);
}
public void setDirectoryListingActive() {

4
Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties
View File

@ -66,7 +66,3 @@ IngestManager.StartIngestJobsTask.run.startupErr.dlgErrorList=Errors\:\
\
{0}
IngestManager.StartIngestJobsTask.run.startupErr.dlgTitle=Ingest Failure
IngestManager.StartIngestJobsTask.run.progress.msg1=Data source ingest tasks for {0}
IngestManager.StartIngestJobsTask.run.progress.msg2=Data source ingest tasks for {0}
IngestManager.StartIngestJobsTask.run.progress.msg3=Data source ingest tasks for {0}
IngestManager.StartIngestJobsTask.run.progress.msg4=Data source ingest tasks for {0}

4
Core/src/org/sleuthkit/autopsy/ingest/Bundle_ja.properties
View File

@ -56,10 +56,6 @@ IngestMonitor.mgrErrMsg.lowDiskSpace.msg=\u30c7\u30a3\u30b9\u30af{0}\u306e\u30c7
IngestMonitor.mgrErrMsg.lowDiskSpace.title=\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u304c\u4e2d\u6b62\u3055\u308c\u307e\u3057\u305f\u30fc{0}\u306e\u30c7\u30a3\u30b9\u30af\u9818\u57df\u4e0d\u8db3
IngestScheduler.DataSourceScheduler.toString.size=DataSourceQueue, \u30b5\u30a4\u30ba\uff1a
OpenIDE-Module-Name=\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8
IngestManager.StartIngestJobsTask.run.progress.msg1={0}\u306e\u30c7\u30fc\u30bf\u30bd\u30fc\u30b9\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u30bf\u30b9\u30af
IngestManager.StartIngestJobsTask.run.progress.msg2={0}\u306e\u30c7\u30fc\u30bf\u30bd\u30fc\u30b9\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u30bf\u30b9\u30af
IngestManager.StartIngestJobsTask.run.progress.msg3={0}\u306e\u30c7\u30fc\u30bf\u30bd\u30fc\u30b9\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u30bf\u30b9\u30af
IngestManager.StartIngestJobsTask.run.progress.msg4={0}\u306e\u30c7\u30fc\u30bf\u30bd\u30fc\u30b9\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u30bf\u30b9\u30af
IngestManager.StartIngestJobsTask.run.startupErr.dlgErrorList=\u30a8\u30e9\u30fc\uff1a\
\
{0}

14
Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestModuleProgress.java
View File

@ -18,16 +18,18 @@
*/
package org.sleuthkit.autopsy.ingest;
import org.netbeans.api.progress.ProgressHandle;
/**
* Used by data source ingest modules to report progress.
*/
public class DataSourceIngestModuleProgress {
private final IngestJob ingestJob;
private final ProgressHandle progress;
private final String moduleDisplayName;
DataSourceIngestModuleProgress(IngestJob ingestJob, String moduleDisplayName) {
this.ingestJob = ingestJob;
DataSourceIngestModuleProgress(ProgressHandle progress, String moduleDisplayName) {
this.progress = progress;
this.moduleDisplayName = moduleDisplayName;
}
@ -40,7 +42,7 @@ public class DataSourceIngestModuleProgress {
* data source.
*/
public void switchToDeterminate(int workUnits) {
ingestJob.getDataSourceTaskProgressBar().switchToDeterminate(workUnits);
progress.switchToDeterminate(workUnits);
}
/**
@ -48,7 +50,7 @@ public class DataSourceIngestModuleProgress {
* the total work units to process the data source is unknown.
*/
public void switchToIndeterminate() {
ingestJob.getDataSourceTaskProgressBar().switchToIndeterminate();
progress.switchToIndeterminate();
}
/**
@ -58,6 +60,6 @@ public class DataSourceIngestModuleProgress {
* @param workUnits Number of work units performed so far by the module.
*/
public void progress(int workUnits) {
ingestJob.getDataSourceTaskProgressBar().progress(this.moduleDisplayName, workUnits);
progress.progress(this.moduleDisplayName, workUnits);
}
}

18
Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java
View File

@ -22,6 +22,7 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.netbeans.api.progress.ProgressHandle;
import org.sleuthkit.datamodel.Content;
/**
@ -30,12 +31,12 @@ import org.sleuthkit.datamodel.Content;
*/
final class DataSourceIngestPipeline {
private final IngestJob job;
private final IngestJobContext context;
private final List<IngestModuleTemplate> moduleTemplates;
private List<DataSourceIngestModuleDecorator> modules = new ArrayList<>();
DataSourceIngestPipeline(IngestJob job, List<IngestModuleTemplate> moduleTemplates) {
this.job = job;
DataSourceIngestPipeline(IngestJobContext context, List<IngestModuleTemplate> moduleTemplates) {
this.context = context;
this.moduleTemplates = moduleTemplates;
}
@ -50,7 +51,6 @@ final class DataSourceIngestPipeline {
for (IngestModuleTemplate template : moduleTemplates) {
if (template.isDataSourceIngestModuleTemplate()) {
DataSourceIngestModuleDecorator module = new DataSourceIngestModuleDecorator(template.createDataSourceIngestModule(), template.getModuleName());
IngestJobContext context = new IngestJobContext(job);
try {
module.startUp(context);
modulesByClass.put(module.getClassName(), module);
@ -75,26 +75,26 @@ final class DataSourceIngestPipeline {
return errors;
}
List<IngestModuleError> process() {
List<IngestModuleError> process(Content dataSource, ProgressHandle progress) {
List<IngestModuleError> errors = new ArrayList<>();
for (DataSourceIngestModuleDecorator module : this.modules) {
try {
module.process(job.getDataSource(), new DataSourceIngestModuleProgress(job, module.getDisplayName()));
module.process(dataSource, new DataSourceIngestModuleProgress(progress, module.getDisplayName()));
} catch (Exception ex) {
errors.add(new IngestModuleError(module.getDisplayName(), ex));
}
if (job.isCancelled()) {
if (context.isJobCancelled()) {
break;
}
}
return errors;
}
List<IngestModuleError> shutDown(boolean ingestJobCancelled) {
List<IngestModuleError> shutDown() {
List<IngestModuleError> errors = new ArrayList<>();
for (DataSourceIngestModuleDecorator module : this.modules) {
try {
module.shutDown(ingestJobCancelled);
module.shutDown(context.isJobCancelled());
} catch (Exception ex) {
errors.add(new IngestModuleError(module.getDisplayName(), ex));
}

4
Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTask.java
View File

@ -38,7 +38,7 @@ final class DataSourceIngestTask {
return dataSource;
}
void execute() {
ingestJob.process();
void execute() throws InterruptedException {
ingestJob.process(dataSource);
}
}

30
Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestTaskScheduler.java
View File

@ -32,26 +32,30 @@ final class DataSourceIngestTaskScheduler {
private DataSourceIngestTaskScheduler() {
}
synchronized void addTask(DataSourceIngestTask task) throws InterruptedException {
task.getIngestJob().notifyTaskPending();
synchronized void addTask(DataSourceIngestTask task) {
// The capacity of the tasks queue is not bounded, so the call
// to put() should not block except for normal synchronized access.
// Still, notify the job that the task has been added first so that
// the take() of the task cannot occur before the notification.
task.getIngestJob().notifyTaskAdded();
// If the thread executing this code is ever interrupted, it is
// because the number of ingest threads has been decreased while
// ingest jobs are running. This thread will exit in an orderly fashion,
// but the task still needs to be enqueued rather than lost.
while (true) {
try {
tasks.put(task);
break;
} catch (InterruptedException ex) {
// Reset the interrupted status of the thread so the orderly
// exit can occur in the intended place.
Thread.currentThread().interrupt();
}
catch (InterruptedException ex) {
// RJCTOD: Need a safety notification to undo above
}
}
DataSourceIngestTask getNextTask() throws InterruptedException {
return tasks.take();
}
boolean hasTasksForIngestJob(long jobId) {
for (DataSourceIngestTask task : tasks) {
if (task.getIngestJobId() == jobId) {
return true;
}
}
return false;
}
}

17
Core/src/org/sleuthkit/autopsy/ingest/FileIngestPipeline.java
View File

@ -30,12 +30,12 @@ import org.sleuthkit.datamodel.AbstractFile;
*/
final class FileIngestPipeline {
private final IngestJob job;
private final IngestJobContext context;
private final List<IngestModuleTemplate> moduleTemplates;
private List<FileIngestModuleDecorator> modules = new ArrayList<>();
FileIngestPipeline(IngestJob task, List<IngestModuleTemplate> moduleTemplates) {
this.job = task;
FileIngestPipeline(IngestJobContext context, List<IngestModuleTemplate> moduleTemplates) {
this.context = context;
this.moduleTemplates = moduleTemplates;
}
@ -50,7 +50,6 @@ final class FileIngestPipeline {
for (IngestModuleTemplate template : moduleTemplates) {
if (template.isFileIngestModuleTemplate()) {
FileIngestModuleDecorator module = new FileIngestModuleDecorator(template.createFileIngestModule(), template.getModuleName());
IngestJobContext context = new IngestJobContext(job);
try {
module.startUp(context);
modulesByClass.put(module.getClassName(), module);
@ -83,22 +82,22 @@ final class FileIngestPipeline {
} catch (Exception ex) {
errors.add(new IngestModuleError(module.getDisplayName(), ex));
}
if (job.isCancelled()) {
if (context.isJobCancelled()) {
break;
}
}
file.close();
if (!job.isCancelled()) {
IngestManager.fireFileIngestDone(file.getId());
if (!context.isJobCancelled()) {
IngestManager.getInstance().fireFileIngestDone(file.getId());
}
return errors;
}
List<IngestModuleError> shutDown(boolean ingestJobCancelled) {
List<IngestModuleError> shutDown() {
List<IngestModuleError> errors = new ArrayList<>();
for (FileIngestModuleDecorator module : this.modules) {
try {
module.shutDown(ingestJobCancelled);
module.shutDown(context.isJobCancelled());
} catch (Exception ex) {
errors.add(new IngestModuleError(module.getDisplayName(), ex));
}

42
Core/src/org/sleuthkit/autopsy/ingest/FileIngestTask.java
View File

@ -19,17 +19,16 @@
package org.sleuthkit.autopsy.ingest;
import java.util.Objects;
import java.util.logging.Level;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.TskCoreException;
final class FileIngestTask {
final AbstractFile file;
private final IngestJob ingestJob;
FileIngestTask(AbstractFile file, IngestJob task) {
this.file = file;
private final IngestJob ingestJob;
private final AbstractFile file;
FileIngestTask(IngestJob task, AbstractFile file) {
this.ingestJob = task;
this.file = file;
}
public IngestJob getIngestJob() {
@ -40,29 +39,10 @@ final class FileIngestTask {
return file;
}
void execute(long threadId) {
void execute() throws InterruptedException {
ingestJob.process(file);
}
@Override
public String toString() { //RJCTODO: May not keep this
try {
return "ProcessTask{" + "file=" + file.getId() + ": " + file.getUniquePath() + "}"; // + ", dataSourceTask=" + dataSourceTask + '}';
} catch (TskCoreException ex) {
// RJCTODO
// FileIngestTaskScheduler.logger.log(Level.SEVERE, "Cound not get unique path of file in queue, ", ex); //NON-NLS
}
return "ProcessTask{" + "file=" + file.getId() + ": " + file.getName() + '}';
}
/**
* two process tasks are equal when the file/dir and modules are the
* same this enables are not to queue up the same file/dir, modules
* tuples into the root dir set
*
* @param obj
* @return
*/
@Override
public boolean equals(Object obj) {
if (obj == null) {
@ -71,13 +51,11 @@ final class FileIngestTask {
if (getClass() != obj.getClass()) {
return false;
}
final FileIngestTask other = (FileIngestTask) obj;
if (this.file != other.file && (this.file == null || !this.file.equals(other.file))) {
FileIngestTask other = (FileIngestTask) obj;
if (this.ingestJob != other.ingestJob && (this.ingestJob == null || !this.ingestJob.equals(other.ingestJob))) {
return false;
}
IngestJob thisTask = this.getIngestJob();
IngestJob otherTask = other.getIngestJob();
if (thisTask != otherTask && (thisTask == null || !thisTask.equals(otherTask))) {
if (this.file != other.file && (this.file == null || !this.file.equals(other.file))) {
return false;
}
return true;
@ -86,8 +64,8 @@ final class FileIngestTask {
@Override
public int hashCode() {
int hash = 5;
hash = 47 * hash + Objects.hashCode(this.file);
hash = 47 * hash + Objects.hashCode(this.ingestJob);
hash = 47 * hash + Objects.hashCode(this.file);
return hash;
}
}

124
Core/src/org/sleuthkit/autopsy/ingest/FileIngestTaskScheduler.java
View File

@ -43,23 +43,20 @@ import org.sleuthkit.datamodel.VirtualDirectory;
final class FileIngestTaskScheduler {
private static final Logger logger = Logger.getLogger(FileIngestTaskScheduler.class.getName());
private static FileIngestTaskScheduler instance;
private static FileIngestTaskScheduler instance = new FileIngestTaskScheduler();
private final TreeSet<FileIngestTask> rootDirectoryTasks = new TreeSet<>(new RootDirectoryTaskComparator());
private final List<FileIngestTask> directoryTasks = new ArrayList<>();
private final LinkedBlockingQueue<FileIngestTask> fileTasks = new LinkedBlockingQueue<>(); // Unlimited capacity
private final LinkedBlockingQueue<FileIngestTask> fileTasks = new LinkedBlockingQueue<>();
private static final int FAT_NTFS_FLAGS = TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT12.getValue() | TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT16.getValue() | TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT32.getValue() | TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_NTFS.getValue();
static synchronized FileIngestTaskScheduler getInstance() {
if (instance == null) {
instance = new FileIngestTaskScheduler();
}
static FileIngestTaskScheduler getInstance() {
return instance;
}
private FileIngestTaskScheduler() {
}
synchronized void addTasks(IngestJob dataSourceTask, Content dataSource) {
synchronized void addTasks(IngestJob job, Content dataSource) throws InterruptedException {
Collection<AbstractFile> rootObjects = dataSource.accept(new GetRootDirectoryVisitor());
List<AbstractFile> firstLevelFiles = new ArrayList<>();
if (rootObjects.isEmpty() && dataSource instanceof AbstractFile) {
@ -87,9 +84,9 @@ final class FileIngestTaskScheduler {
}
}
for (AbstractFile firstLevelFile : firstLevelFiles) {
FileIngestTask fileTask = new FileIngestTask(firstLevelFile, dataSourceTask);
FileIngestTask fileTask = new FileIngestTask(job, firstLevelFile);
if (shouldEnqueueTask(fileTask)) {
rootDirectoryTasks.add(fileTask);
addTaskToRootDirectoryQueue(fileTask);
}
}
@ -97,16 +94,9 @@ final class FileIngestTaskScheduler {
updateQueues();
}
synchronized void addTask(IngestJob ingestJob, AbstractFile file) {
try {
FileIngestTask fileTask = new FileIngestTask(file, ingestJob);
if (shouldEnqueueTask(fileTask)) {
fileTask.getIngestJob().notifyTaskPending();
fileTasks.put(fileTask); // Queue has unlimited capacity, does not block.
}
} catch (InterruptedException ex) {
// RJCTODO: Perhaps this is the convenience method?
// RJCTODO: Need undo
synchronized void addTask(FileIngestTask task) {
if (shouldEnqueueTask(task)) {
addTaskToFileQueue(task);
}
}
@ -116,29 +106,7 @@ final class FileIngestTaskScheduler {
return task;
}
synchronized boolean hasTasksForJob(long ingestJobId) {
for (FileIngestTask task : rootDirectoryTasks) {
if (task.getIngestJob().getJobId() == ingestJobId) {
return true;
}
}
for (FileIngestTask task : directoryTasks) {
if (task.getIngestJob().getJobId() == ingestJobId) {
return true;
}
}
for (FileIngestTask task : fileTasks) {
if (task.getIngestJob().getJobId() == ingestJobId) {
return true;
}
}
return false;
}
private void updateQueues() {
private void updateQueues() throws InterruptedException {
// we loop because we could have a directory that has all files
// that do not get enqueued
while (true) {
@ -152,23 +120,15 @@ final class FileIngestTaskScheduler {
if (rootDirectoryTasks.isEmpty()) {
return;
}
FileIngestTask rootTask = this.rootDirectoryTasks.pollFirst();
directoryTasks.add(rootTask);
addTaskToDirectoryQueue(rootDirectoryTasks.pollFirst(), false);
}
//pop and push AbstractFile directory children if any
//add the popped and its leaf children onto cur file list
FileIngestTask parentTask = directoryTasks.remove(directoryTasks.size() - 1);
final AbstractFile parentFile = parentTask.file;
final AbstractFile parentFile = parentTask.getFile();
// add itself to the file list
if (shouldEnqueueTask(parentTask)) {
// RJCTODO
try {
parentTask.getIngestJob().notifyTaskPending();
fileTasks.put(parentTask);
} catch (InterruptedException ex) {
// RJCTODO: Maybe make a convenience method
// RJCTODO: Need undo
}
addTaskToFileQueue(parentTask);
}
// add its children to the file and directory lists
try {
@ -176,18 +136,11 @@ final class FileIngestTaskScheduler {
for (Content c : children) {
if (c instanceof AbstractFile) {
AbstractFile childFile = (AbstractFile) c;
FileIngestTask childTask = new FileIngestTask(childFile, parentTask.getIngestJob());
FileIngestTask childTask = new FileIngestTask(parentTask.getIngestJob(), childFile);
if (childFile.hasChildren()) {
this.directoryTasks.add(childTask);
addTaskToDirectoryQueue(childTask, true);
} else if (shouldEnqueueTask(childTask)) {
// RJCTODO
try {
childTask.getIngestJob().notifyTaskPending();
fileTasks.put(childTask);
} catch (InterruptedException ex) {
// RJCTODO: Maybe make a convenience method
// RJCTODO: Need undo
}
addTaskToFileQueue(childTask);
}
}
}
@ -197,10 +150,39 @@ final class FileIngestTaskScheduler {
}
}
synchronized void emptyQueues() { // RJCTODO: Perhaps clear all...
this.rootDirectoryTasks.clear();
this.directoryTasks.clear();
this.fileTasks.clear();
private void addTaskToRootDirectoryQueue(FileIngestTask task) {
directoryTasks.add(task);
task.getIngestJob().notifyTaskAdded();
}
private void addTaskToDirectoryQueue(FileIngestTask task, boolean isNewTask) {
if (isNewTask) {
directoryTasks.add(task);
}
task.getIngestJob().notifyTaskAdded();
}
private void addTaskToFileQueue(FileIngestTask task) {
// The capacity of the file tasks queue is not bounded, so the call
// to put() should not block except for normal synchronized access.
// Still, notify the job that the task has been added first so that
// the take() of the task cannot occur before the notification.
task.getIngestJob().notifyTaskAdded();
// If the thread executing this code is ever interrupted, it is
// because the number of ingest threads has been decreased while
// ingest jobs are running. This thread will exit in an orderly fashion,
// but the task still needs to be enqueued rather than lost.
while (true) {
try {
fileTasks.put(task);
break;
} catch (InterruptedException ex) {
// Reset the interrupted status of the thread so the orderly
// exit can occur in the intended place.
Thread.currentThread().interrupt();
}
}
}
/**
@ -211,7 +193,7 @@ final class FileIngestTaskScheduler {
* @return true if should be enqueued, false otherwise
*/
private static boolean shouldEnqueueTask(final FileIngestTask processTask) {
final AbstractFile aFile = processTask.file;
final AbstractFile aFile = processTask.getFile();
//if it's unalloc file, skip if so scheduled
if (processTask.getIngestJob().shouldProcessUnallocatedSpace() == false && aFile.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS)) {
return false;
@ -320,10 +302,10 @@ final class FileIngestTaskScheduler {
@Override
public int compare(FileIngestTask q1, FileIngestTask q2) {
AbstractFilePriority.Priority p1 = AbstractFilePriority.getPriority(q1.file);
AbstractFilePriority.Priority p2 = AbstractFilePriority.getPriority(q2.file);
AbstractFilePriority.Priority p1 = AbstractFilePriority.getPriority(q1.getFile());
AbstractFilePriority.Priority p2 = AbstractFilePriority.getPriority(q2.getFile());
if (p1 == p2) {
return (int) (q2.file.getId() - q1.file.getId());
return (int) (q2.getFile().getId() - q1.getFile().getId());
} else {
return p2.ordinal() - p1.ordinal();
}

179
Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java
View File

@ -24,10 +24,12 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.logging.Level;
import org.netbeans.api.progress.ProgressHandle;
import org.netbeans.api.progress.ProgressHandleFactory;
import org.openide.util.Cancellable;
import org.openide.util.NbBundle;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.Content;
@ -37,33 +39,48 @@ import org.sleuthkit.datamodel.Content;
*/
final class IngestJob {
private static final Logger logger = Logger.getLogger(IngestManager.class.getName());
private static final AtomicLong nextIngestJobId = new AtomicLong(0L);
private static final ConcurrentHashMap<Long, IngestJob> ingestJobs = new ConcurrentHashMap<>(); // Maps job ids to jobs.
private final long jobId;
private final Content dataSource;
private static final ConcurrentHashMap<Long, IngestJob> ingestJobsById = new ConcurrentHashMap<>();
private final long id;
private final Content rootDataSource;
private final List<IngestModuleTemplate> ingestModuleTemplates;
private final boolean processUnallocatedSpace;
private final LinkedBlockingQueue<DataSourceIngestPipeline> dataSourceIngestPipelines = new LinkedBlockingQueue<>();
private final LinkedBlockingQueue<FileIngestPipeline> fileIngestPipelines = new LinkedBlockingQueue<>();
private final AtomicInteger tasksInProgress = new AtomicInteger(0);
private final AtomicLong processedFiles = new AtomicLong(0L);
private final AtomicLong filesToIngestEstimate = new AtomicLong(0L);
private ProgressHandle dataSourceTasksProgress;
private ProgressHandle fileTasksProgress;
private long filesToIngestEstimate = 0;
private volatile boolean cancelled;
static List<IngestModuleError> startIngestJob(Content dataSource, List<IngestModuleTemplate> ingestModuleTemplates, boolean processUnallocatedSpace) { // RJCTODO: return errors
/**
* Creates an ingest job for a data source and starts up the ingest
* pipelines for the job.
*
* @param rootDataSource The data source to ingest.
* @param ingestModuleTemplates The ingest module templates to use to create
* the ingest pipelines for the job.
* @param processUnallocatedSpace Whether or not the job should include
* processing of unallocated space.
* @return A collection of ingest module startUp up errors, empty on
* success.
* @throws InterruptedException
*/
static List<IngestModuleError> startJob(Content dataSource, List<IngestModuleTemplate> ingestModuleTemplates, boolean processUnallocatedSpace) throws InterruptedException {
long jobId = nextIngestJobId.incrementAndGet();
IngestJob ingestJob = new IngestJob(jobId, dataSource, ingestModuleTemplates, processUnallocatedSpace);
List<IngestModuleError> errors = ingestJob.start();
List<IngestModuleError> errors = ingestJob.startUp();
if (errors.isEmpty()) {
ingestJobs.put(jobId, ingestJob);
ingestJobsById.put(jobId, ingestJob);
IngestManager.getInstance().fireIngestJobStarted(jobId);
}
return errors;
}
static boolean jobsAreRunning() {
for (IngestJob job : ingestJobs.values()) {
for (IngestJob job : ingestJobsById.values()) {
if (!job.isCancelled()) {
return true;
}
@ -71,79 +88,72 @@ final class IngestJob {
return false;
}
static void addFileToIngestJob(long ingestJobId, AbstractFile file) { // RJCTODO: Move back to IngestManager
IngestJob job = ingestJobs.get(ingestJobId);
static void addFileToJob(long ingestJobId, AbstractFile file) { // RJCTODO: Just one at a time?
IngestJob job = ingestJobsById.get(ingestJobId);
if (job != null) {
FileIngestTaskScheduler.getInstance().addTask(job, file);
// long adjustedFilesCount = job.filesToIngestEstimate.incrementAndGet(); // RJCTODO: Not the best name now?
// job.fileTasksProgress.switchToIndeterminate(); // RJCTODO: Comment this stuff
// job.fileTasksProgress.switchToDeterminate((int) adjustedFilesCount);
// job.fileTasksProgress.progress(job.processedFiles.intValue());
FileIngestTaskScheduler.getInstance().addTask(new FileIngestTask(job, file));
}
}
static void cancelAllIngestJobs() {
for (IngestJob job : ingestJobs.values()) {
static void cancelAllJobs() {
for (IngestJob job : ingestJobsById.values()) {
job.cancel();
}
}
private IngestJob(long id, Content dataSource, List<IngestModuleTemplate> ingestModuleTemplates, boolean processUnallocatedSpace) {
this.jobId = id;
this.dataSource = dataSource;
this.id = id;
this.rootDataSource = dataSource;
this.ingestModuleTemplates = ingestModuleTemplates;
this.processUnallocatedSpace = processUnallocatedSpace;
this.cancelled = false;
}
long getJobId() {
return jobId;
}
Content getDataSource() {
return dataSource;
long getId() {
return id;
}
boolean shouldProcessUnallocatedSpace() {
return processUnallocatedSpace;
}
List<IngestModuleError> start() {
List<IngestModuleError> startUp() throws InterruptedException {
List<IngestModuleError> errors = startUpIngestPipelines();
if (errors.isEmpty()) {
DataSourceIngestTaskScheduler.getInstance().addTask(new DataSourceIngestTask(this, dataSource));
FileIngestTaskScheduler.getInstance().addTasks(this, dataSource);
startDataSourceIngestProgressBar();
startFileIngestProgressBar();
FileIngestTaskScheduler.getInstance().addTasks(this, rootDataSource); // RJCTODO: Think about this ordering "solution" for small images
DataSourceIngestTaskScheduler.getInstance().addTask(new DataSourceIngestTask(this, rootDataSource));
}
return errors;
}
private List<IngestModuleError> startUpIngestPipelines() {
private List<IngestModuleError> startUpIngestPipelines() throws InterruptedException {
IngestJobContext context = new IngestJobContext(this);
List<IngestModuleError> errors = new ArrayList<>();
int maxNumberOfPipelines = IngestManager.getMaxNumberOfDataSourceIngestThreads();
for (int i = 0; i < maxNumberOfPipelines; ++i) {
DataSourceIngestPipeline pipeline = new DataSourceIngestPipeline(this, ingestModuleTemplates);
DataSourceIngestPipeline pipeline = new DataSourceIngestPipeline(context, ingestModuleTemplates);
errors.addAll(pipeline.startUp());
try {
dataSourceIngestPipelines.put(pipeline);
} catch (InterruptedException ex) {
// RJCTODO: log unexpected block and interrupt, or throw
}
if (errors.isEmpty()) {
// No need to accumulate presumably redundant erros.
if (!errors.isEmpty()) {
// No need to accumulate presumably redundant errors.
break;
}
}
maxNumberOfPipelines = IngestManager.getMaxNumberOfFileIngestThreads();
for (int i = 0; i < maxNumberOfPipelines; ++i) {
FileIngestPipeline pipeline = new FileIngestPipeline(this, ingestModuleTemplates);
FileIngestPipeline pipeline = new FileIngestPipeline(context, ingestModuleTemplates);
errors.addAll(pipeline.startUp());
try {
fileIngestPipelines.put(pipeline);
} catch (InterruptedException ex) {
// RJCTODO: log unexpected block and interrupt, or throw
}
if (errors.isEmpty()) {
// No need to accumulate presumably redundant erros.
if (!errors.isEmpty()) {
// No need to accumulate presumably redundant errors.
break;
}
}
@ -154,7 +164,7 @@ final class IngestJob {
private void startDataSourceIngestProgressBar() {
final String displayName = NbBundle.getMessage(this.getClass(),
"IngestJob.progress.dataSourceIngest.displayName",
dataSource.getName());
rootDataSource.getName());
dataSourceTasksProgress = ProgressHandleFactory.createHandle(displayName, new Cancellable() {
@Override
public boolean cancel() {
@ -169,13 +179,13 @@ final class IngestJob {
}
});
dataSourceTasksProgress.start();
dataSourceTasksProgress.switchToIndeterminate(); // RJCTODO: check out the logic in the pipleine class
dataSourceTasksProgress.switchToIndeterminate();
}
private void startFileIngestProgressBar() {
final String displayName = NbBundle.getMessage(this.getClass(),
"IngestJob.progress.fileIngest.displayName",
dataSource.getName());
rootDataSource.getName());
fileTasksProgress = ProgressHandleFactory.createHandle(displayName, new Cancellable() {
@Override
public boolean cancel() {
@ -188,77 +198,78 @@ final class IngestJob {
return true;
}
});
filesToIngestEstimate = dataSource.accept(new GetFilesCountVisitor());
long initialFilesCount = rootDataSource.accept(new GetFilesCountVisitor());
filesToIngestEstimate.getAndAdd(initialFilesCount);
fileTasksProgress.start();
fileTasksProgress.switchToDeterminate((int) filesToIngestEstimate);
fileTasksProgress.switchToDeterminate((int) initialFilesCount); // RJCTODO: This cast is troublesome, can use intValue
}
/**
* Called by the ingest task schedulers when an ingest task for this ingest
* job is added to the scheduler's task queue.
* Called by the ingest task schedulers when an ingest task is added to this
* ingest job.
*/
void notifyTaskScheduled() {
// Increment the task counter when a task is scheduled so that there is
// a persistent record of the task's existence even after it is removed
// from the scheduler by an ingest thread. The task counter is used by
// the job to determine when it is done.
void notifyTaskAdded() {
tasksInProgress.incrementAndGet();
}
/**
* Called by the ingest schedulers as an "undo" operation for
* notifyTaskScheduled().
*/
void notifyTaskCompleted() {
// Decrement the task counter when a task is discarded by a scheduler.
// The task counter is used by the job to determine when it is done.
tasksInProgress.decrementAndGet();
}
void process() throws InterruptedException {
void process(Content dataSource) throws InterruptedException {
// If the job is not cancelled, complete the task, otherwise just flush
// it. In either case, the task counter needs to be decremented and the
// shut down check needs to occur.
if (!isCancelled()) {
try {
List<IngestModuleError> errors = new ArrayList<>();
DataSourceIngestPipeline pipeline = dataSourceIngestPipelines.take();
pipeline.process(); // RJCTODO: Pass data source through?
errors.addAll(pipeline.process(dataSource, dataSourceTasksProgress));
if (!errors.isEmpty()) {
logIngestModuleErrors(errors);
}
dataSourceIngestPipelines.put(pipeline);
} catch (InterruptedException ex) {
// RJCTODO:
}
}
ifCompletedShutDown();
shutDownIfAllTasksCompleted();
}
void process(AbstractFile file) {
void process(AbstractFile file) throws InterruptedException {
// If the job is not cancelled, complete the task, otherwise just flush
// it. In either case, the task counter needs to be decremented and the
// shut down check needs to occur.
if (!isCancelled()) {
try {
List<IngestModuleError> errors = new ArrayList<>();
FileIngestPipeline pipeline = fileIngestPipelines.take();
fileTasksProgress.progress(file.getName(), (int) processedFiles.incrementAndGet());
pipeline.process(file);
// fileTasksProgress.progress(file.getName(), (int) processedFiles.incrementAndGet()); RJCTODO
errors.addAll(pipeline.process(file));
fileIngestPipelines.put(pipeline);
} catch (InterruptedException ex) {
// RJCTODO: Log block and interrupt
if (!errors.isEmpty()) {
logIngestModuleErrors(errors);
}
}
ifCompletedShutDown();
shutDownIfAllTasksCompleted();
}
void ifCompletedShutDown() {
private void shutDownIfAllTasksCompleted() {
if (tasksInProgress.decrementAndGet() == 0) {
List<IngestModuleError> errors = new ArrayList<>();
while (!dataSourceIngestPipelines.isEmpty()) {
DataSourceIngestPipeline pipeline = dataSourceIngestPipelines.poll();
pipeline.shutDown(cancelled);
errors.addAll(pipeline.shutDown());
}
while (!fileIngestPipelines.isEmpty()) {
FileIngestPipeline pipeline = fileIngestPipelines.poll();
pipeline.shutDown(cancelled);
errors.addAll(pipeline.shutDown());
}
ingestJobs.remove(jobId);
IngestManager.getInstance().fireIngestJobCompleted(jobId);
fileTasksProgress.finish();
dataSourceTasksProgress.finish();
ingestJobsById.remove(id);
if (!errors.isEmpty()) {
logIngestModuleErrors(errors);
}
IngestManager.getInstance().fireIngestJobCompleted(id);
}
}
ProgressHandle getDataSourceTaskProgressBar() {
return dataSourceTasksProgress; // RJCTODO: Should just pass the progress handle or the object to the pipeline
private void logIngestModuleErrors(List<IngestModuleError> errors) {
for (IngestModuleError error : errors) {
logger.log(Level.SEVERE, error.getModuleDisplayName() + " experienced an error", error.getModuleError());
}
}
boolean isCancelled() {
@ -267,7 +278,7 @@ final class IngestJob {
void cancel() {
cancelled = true;
fileTasksProgress.finish();
IngestManager.getInstance().fireIngestJobCancelled(jobId);
fileTasksProgress.finish(); // RJCTODO: What about the other progress bar?
IngestManager.getInstance().fireIngestJobCancelled(id);
}
}

4
Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java
View File

@ -39,7 +39,7 @@ public final class IngestJobContext {
* @return The ingest job identifier.
*/
public long getJobId() {
return this.ingestJob.getJobId();
return this.ingestJob.getId();
}
/**
@ -60,7 +60,7 @@ public final class IngestJobContext {
*/
public void addFiles(List<AbstractFile> files) {
for (AbstractFile file : files) {
IngestJob.addFileToIngestJob(ingestJob.getJobId(), file);
IngestJob.addFileToJob(ingestJob.getId(), file);
}
}
}

171
Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java
View File

@ -21,7 +21,9 @@ package org.sleuthkit.autopsy.ingest;
import java.beans.PropertyChangeListener;
import java.beans.PropertyChangeSupport;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
@ -51,14 +53,15 @@ public class IngestManager {
private static final Logger logger = Logger.getLogger(IngestManager.class.getName());
private static final Preferences userPreferences = NbPreferences.forModule(IngestManager.class);
private static final IngestManager instance = new IngestManager();
private final PropertyChangeSupport pcs = new PropertyChangeSupport(IngestManager.class);
private final PropertyChangeSupport ingestJobEventPublisher = new PropertyChangeSupport(IngestManager.class);
private final PropertyChangeSupport ingestModuleEventPublisher = new PropertyChangeSupport(IngestManager.class);
private final IngestMonitor ingestMonitor = new IngestMonitor();
private final ExecutorService startIngestJobsThreadPool = Executors.newSingleThreadExecutor();
private final ConcurrentHashMap<Long, Future<?>> startIngestJobThreads = new ConcurrentHashMap<>(); // Maps thread ids to cancellation handles.
private final ExecutorService dataSourceIngestThreadPool = Executors.newSingleThreadExecutor();
private final ConcurrentHashMap<Long, Future<?>> dataSourceIngestThreads = new ConcurrentHashMap<>(); // Maps thread ids to cancellation handles.
private final ExecutorService fileIngestThreadPool = Executors.newFixedThreadPool(MAX_NUMBER_OF_FILE_INGEST_THREADS);
private final ExecutorService fireIngestJobEventsThreadPool = Executors.newSingleThreadExecutor();
private final ExecutorService fireIngestEventsThreadPool = Executors.newSingleThreadExecutor();
private final ConcurrentHashMap<Long, Future<Void>> startIngestJobThreads = new ConcurrentHashMap<>(); // Maps thread ids to cancellation handles.
private final ConcurrentHashMap<Long, Future<?>> dataSourceIngestThreads = new ConcurrentHashMap<>(); // Maps thread ids to cancellation handles.
private final ConcurrentHashMap<Long, Future<?>> fileIngestThreads = new ConcurrentHashMap<>(); // Maps thread ids to cancellation handles.
private final AtomicLong nextThreadId = new AtomicLong(0L);
private volatile IngestMessageTopComponent ingestMessageBox;
@ -154,7 +157,7 @@ public class IngestManager {
*/
private void startDataSourceIngestThread() {
long threadId = nextThreadId.incrementAndGet();
Future<?> handle = dataSourceIngestThreadPool.submit(new DataSourceIngestThread(threadId));
Future<?> handle = dataSourceIngestThreadPool.submit(new DataSourceIngestThread());
dataSourceIngestThreads.put(threadId, handle);
}
@ -164,7 +167,7 @@ public class IngestManager {
*/
private void startFileIngestThread() {
long threadId = nextThreadId.incrementAndGet();
Future<?> handle = fileIngestThreadPool.submit(new FileIngestThread(threadId));
Future<?> handle = fileIngestThreadPool.submit(new FileIngestThread());
fileIngestThreads.put(threadId, handle);
}
@ -200,35 +203,48 @@ public class IngestManager {
}
public void cancelAllIngestJobs() {
cancelStartIngestJobsTasks();
IngestJob.cancelAllIngestJobs();
// Stop creating new ingest jobs.
for (Future<Void> handle : startIngestJobThreads.values()) {
handle.cancel(true);
try {
// Blocks until the job starting thread responds. The thread
// removes itself from this collection, which does not disrupt
// this loop since the collection is a ConcurrentHashMap.
handle.get();
} catch (InterruptedException | ExecutionException ex) {
// This should never happen, something is awry, but everything
// should be o.k. anyway.
logger.log(Level.SEVERE, "Unexpected thread interrupt", ex);
}
}
startIngestJobThreads.clear(); // Make sure.
private void cancelStartIngestJobsTasks() {
for (Future<?> future : startIngestJobThreads.values()) {
future.cancel(true);
}
startIngestJobThreads.clear();
// Cancel all the jobs already created. This will make the the ingest
// threads flush out any lingering ingest tasks without processing them.
IngestJob.cancelAllJobs();
}
/**
* Ingest events.
*/
public enum IngestEvent { // RJCTODO: Update comments if time permits
public enum IngestEvent {
/**
* Property change event fired when an ingest job is started. The old
* and new values of the PropertyChangeEvent object are set to null.
* value of the PropertyChangeEvent object is set to the ingest job id,
* and the new value is set to null.
*/
INGEST_JOB_STARTED,
/**
* Property change event fired when an ingest job is completed. The old
* and new values of the PropertyChangeEvent object are set to null.
* value of the PropertyChangeEvent object is set to the ingest job id,
* and the new value is set to null.
*/
INGEST_JOB_COMPLETED,
/**
* Property change event fired when an ingest job is canceled. The old
* and new values of the PropertyChangeEvent object are set to null.
* value of the PropertyChangeEvent object is set to the ingest job id,
* and the new value is set to null.
*/
INGEST_JOB_CANCELLED,
/**
@ -253,21 +269,59 @@ public class IngestManager {
};
/**
* Add an ingest event property change listener.
* Add an ingest job event property change listener.
*
* @param listener The PropertyChangeListener to register.
*/
public void addPropertyChangeListener(final PropertyChangeListener listener) {
pcs.addPropertyChangeListener(listener);
public void addIngestJobEventListener(final PropertyChangeListener listener) {
ingestJobEventPublisher.addPropertyChangeListener(listener);
}
/**
* Remove an ingest event property change listener.
* Remove an ingest job event property change listener.
*
* @param listener The PropertyChangeListener to unregister.
*/
public void removePropertyChangeListener(final PropertyChangeListener listener) {
pcs.removePropertyChangeListener(listener);
public void removeIngestJobEventListener(final PropertyChangeListener listener) {
ingestJobEventPublisher.removePropertyChangeListener(listener);
}
/**
* Add an ingest module event property change listener.
*
* @param listener The PropertyChangeListener to register.
*/
public void addIngestModuleEventListener(final PropertyChangeListener listener) {
ingestModuleEventPublisher.addPropertyChangeListener(listener);
}
/**
* Remove an ingest module event property change listener.
*
* @param listener The PropertyChangeListener to unregister.
*/
public void removeIngestModuleEventListener(final PropertyChangeListener listener) {
ingestModuleEventPublisher.removePropertyChangeListener(listener);
}
/**
* Add an ingest module event property change listener.
*
* @deprecated
* @param listener The PropertyChangeListener to register.
*/
public static void addPropertyChangeListener(final PropertyChangeListener listener) {
instance.ingestModuleEventPublisher.addPropertyChangeListener(listener);
}
/**
* Remove an ingest module event property change listener.
*
* @deprecated
* @param listener The PropertyChangeListener to unregister.
*/
public static void removePropertyChangeListener(final PropertyChangeListener listener) {
instance.ingestModuleEventPublisher.removePropertyChangeListener(listener);
}
/**
@ -276,7 +330,7 @@ public class IngestManager {
* @param ingestJobId The ingest job id.
*/
void fireIngestJobStarted(long ingestJobId) {
fireIngestJobEventsThreadPool.submit(new FireIngestEventThread(IngestEvent.INGEST_JOB_STARTED, ingestJobId, null));
fireIngestEventsThreadPool.submit(new FireIngestEventThread(ingestJobEventPublisher, IngestEvent.INGEST_JOB_STARTED, ingestJobId, null));
}
/**
@ -285,7 +339,7 @@ public class IngestManager {
* @param ingestJobId The ingest job id.
*/
void fireIngestJobCompleted(long ingestJobId) {
fireIngestJobEventsThreadPool.submit(new FireIngestEventThread(IngestEvent.INGEST_JOB_COMPLETED, ingestJobId, null));
fireIngestEventsThreadPool.submit(new FireIngestEventThread(ingestJobEventPublisher, IngestEvent.INGEST_JOB_COMPLETED, ingestJobId, null));
}
/**
@ -294,7 +348,7 @@ public class IngestManager {
* @param ingestJobId The ingest job id.
*/
void fireIngestJobCancelled(long ingestJobId) {
fireIngestJobEventsThreadPool.submit(new FireIngestEventThread(IngestEvent.INGEST_JOB_CANCELLED, ingestJobId, null));
fireIngestEventsThreadPool.submit(new FireIngestEventThread(ingestJobEventPublisher, IngestEvent.INGEST_JOB_CANCELLED, ingestJobId, null));
}
/**
@ -303,7 +357,7 @@ public class IngestManager {
* @param fileId The object id of file.
*/
void fireFileIngestDone(long fileId) {
fireIngestJobEventsThreadPool.submit(new FireIngestEventThread(IngestEvent.FILE_DONE, fileId, null));
fireIngestEventsThreadPool.submit(new FireIngestEventThread(ingestModuleEventPublisher, IngestEvent.FILE_DONE, fileId, null));
}
/**
@ -312,7 +366,7 @@ public class IngestManager {
* @param moduleDataEvent A ModuleDataEvent with the details of the posting.
*/
void fireIngestModuleDataEvent(ModuleDataEvent moduleDataEvent) {
fireIngestJobEventsThreadPool.submit(new FireIngestEventThread(IngestEvent.DATA, moduleDataEvent, null));
fireIngestEventsThreadPool.submit(new FireIngestEventThread(ingestModuleEventPublisher, IngestEvent.DATA, moduleDataEvent, null));
}
/**
@ -323,7 +377,7 @@ public class IngestManager {
* content.
*/
void fireIngestModuleContentEvent(ModuleContentEvent moduleContentEvent) {
fireIngestJobEventsThreadPool.submit(new FireIngestEventThread(IngestEvent.CONTENT_CHANGED, moduleContentEvent, null));
fireIngestEventsThreadPool.submit(new FireIngestEventThread(ingestModuleEventPublisher, IngestEvent.CONTENT_CHANGED, moduleContentEvent, null));
}
/**
@ -341,7 +395,7 @@ public class IngestManager {
* Get the free disk space of the drive where to which ingest data is being
* written, as reported by the ingest monitor.
*
* @return Free disk space, -1 if unknown // RJCTODO: What units?
* @return Free disk space, -1 if unknown
*/
long getFreeDiskSpace() {
if (ingestMonitor != null) {
@ -352,10 +406,10 @@ public class IngestManager {
}
/**
* A Runnable that creates ingest jobs and submits the initial data source
* A Callable that creates ingest jobs and submits the initial data source
* and file ingest tasks to the task schedulers.
*/
private class StartIngestJobsThread implements Runnable {
private class StartIngestJobsThread implements Callable<Void> {
private final long threadId;
private final List<Content> dataSources;
@ -371,7 +425,7 @@ public class IngestManager {
}
@Override
public void run() {
public Void call() {
try {
final String displayName = NbBundle.getMessage(this.getClass(),
"IngestManager.StartIngestJobsTask.run.displayName");
@ -387,26 +441,25 @@ public class IngestManager {
return true;
}
});
progress.start(dataSources.size() * 2);
progress.start(dataSources.size());
if (!ingestMonitor.isRunning()) {
ingestMonitor.start();
}
int workUnitsCompleted = 0;
int dataSourceProcessed = 0;
for (Content dataSource : dataSources) {
if (Thread.currentThread().isInterrupted()) {
break;
}
// Create an ingest job.
List<IngestModuleError> errors = IngestJob.startIngestJob(dataSource, moduleTemplates, processUnallocatedSpace);
// Start an ingest job for the data source.
List<IngestModuleError> errors = IngestJob.startJob(dataSource, moduleTemplates, processUnallocatedSpace);
if (!errors.isEmpty()) {
// Report the error to the user.
// Report the errors to the user. They have already been logged.
StringBuilder moduleStartUpErrors = new StringBuilder();
for (IngestModuleError error : errors) {
String moduleName = error.getModuleDisplayName();
logger.log(Level.SEVERE, "The " + moduleName + " module failed to start up", error.getModuleError()); //NON-NLS
moduleStartUpErrors.append(moduleName);
moduleStartUpErrors.append(": ");
moduleStartUpErrors.append(error.getModuleError().getLocalizedMessage());
@ -427,40 +480,16 @@ public class IngestManager {
NbBundle.getMessage(this.getClass(),
"IngestManager.StartIngestJobsTask.run.startupErr.dlgTitle"), JOptionPane.ERROR_MESSAGE);
}
fireIngestJobEventsThreadPool.submit(new FireIngestEventThread(IngestEvent.INGEST_JOB_STARTED));
// Queue a data source ingest task for the ingest job.
final String inputName = dataSource.getName();
progress.progress(
NbBundle.getMessage(this.getClass(), "IngestManager.StartIngestJobsThread.run.progress.msg1",
inputName), workUnitsCompleted);
DataSourceIngestTaskScheduler.getInstance().addTask(new DataSourceIngestTask(ingestJob, ingestJob.getDataSource()));
progress.progress(
NbBundle.getMessage(this.getClass(), "IngestManager.StartIngestJobsThread.run.progress.msg2",
inputName), ++workUnitsCompleted);
// Queue the file ingest tasks for the ingest job.
progress.progress(
NbBundle.getMessage(this.getClass(), "IngestManager.StartIngestJobsThread.run.progress.msg3",
inputName), workUnitsCompleted);
FileIngestTaskScheduler.getInstance().addTasks(ingestJob, ingestJob.getDataSource());
progress.progress(
NbBundle.getMessage(this.getClass(), "IngestManager.StartIngestJobsThread.run.progress.msg4",
inputName), ++workUnitsCompleted);
progress.progress(++dataSourceProcessed);
if (!Thread.currentThread().isInterrupted()) {
break;
}
}
} catch (Exception ex) {
String message = String.format("StartIngestJobsTask (id=%d) caught exception", threadId); //NON-NLS
logger.log(Level.SEVERE, message, ex);
MessageNotifyUtil.Message.error(
NbBundle.getMessage(this.getClass(), "IngestManager.StartIngestJobsTask.run.catchException.msg"));
} finally {
progress.finish();
startIngestJobThreads.remove(threadId);
return null;
}
}
}
@ -512,16 +541,18 @@ public class IngestManager {
}
/**
* A Runnable that fire ingest events to ingest manager property change
* A Runnable that fires ingest events to ingest manager property change
* listeners.
*/
private class FireIngestEventThread implements Runnable {
private static class FireIngestEventThread implements Runnable {
private final PropertyChangeSupport publisher;
private final IngestEvent event;
private final Object oldValue;
private final Object newValue;
FireIngestEventThread(IngestEvent event, Object oldValue, Object newValue) {
FireIngestEventThread(PropertyChangeSupport publisher, IngestEvent event, Object oldValue, Object newValue) {
this.publisher = publisher;
this.event = event;
this.oldValue = oldValue;
this.newValue = newValue;
@ -530,10 +561,10 @@ public class IngestManager {
@Override
public void run() {
try {
pcs.firePropertyChange(event.toString(), oldValue, newValue);
publisher.firePropertyChange(event.toString(), oldValue, newValue);
} catch (Exception e) {
logger.log(Level.SEVERE, "Ingest manager listener threw exception", e); //NON-NLS
MessageNotifyUtil.Notify.show(NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr"), // RJCTODO: Oddly named strings
MessageNotifyUtil.Notify.show(NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr"),
NbBundle.getMessage(IngestManager.class, "IngestManager.moduleErr.errListenToUpdates.msg"),
MessageNotifyUtil.MessageType.ERROR);
}

4
Core/src/org/sleuthkit/autopsy/ingest/IngestServices.java
View File

@ -95,7 +95,7 @@ public final class IngestServices {
* artifact data
*/
public void fireModuleDataEvent(ModuleDataEvent moduleDataEvent) {
IngestManager.fireIngestModuleDataEvent(moduleDataEvent);
IngestManager.getInstance().fireIngestModuleDataEvent(moduleDataEvent);
}
/**
@ -107,7 +107,7 @@ public final class IngestServices {
* changed
*/
public void fireModuleContentEvent(ModuleContentEvent moduleContentEvent) {
IngestManager.fireIngestModuleContentEvent(moduleContentEvent);
IngestManager.getInstance().fireIngestModuleContentEvent(moduleContentEvent);
}
/**

2
HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashLookupSettingsPanel.java
View File

@ -69,7 +69,7 @@ public final class HashLookupSettingsPanel extends IngestModuleGlobalSettingsPan
// Listen to the ingest modules to refresh the enabled/disabled state of
// the components in sync with file ingest.
IngestManager.addPropertyChangeListener(new PropertyChangeListener() {
IngestManager.getInstance().addIngestJobEventListener(new PropertyChangeListener() {
@Override
public void propertyChange(PropertyChangeEvent evt) {
if (isIngestJobEvent(evt)) {

2
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchEditListPanel.java
View File

@ -126,7 +126,7 @@ class KeywordSearchEditListPanel extends javax.swing.JPanel implements ListSelec
setButtonStates();
IngestManager.addPropertyChangeListener(new PropertyChangeListener() {
IngestManager.getInstance().addIngestJobEventListener(new PropertyChangeListener() {
@Override
public void propertyChange(PropertyChangeEvent evt) {
String changed = evt.getPropertyName();

2
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchListsViewerPanel.java
View File

@ -118,7 +118,7 @@ class KeywordSearchListsViewerPanel extends AbstractKeywordSearchPerformer {
ingestRunning = IngestManager.getInstance().isIngestRunning();
updateComponents();
IngestManager.addPropertyChangeListener(new PropertyChangeListener() {
IngestManager.getInstance().addIngestJobEventListener(new PropertyChangeListener() {
@Override
public void propertyChange(PropertyChangeEvent evt) {
String changed = evt.getPropertyName();

0
c:casesSamll2Againautopsy.db Executable file
View File