Partial revision of IngestScheduler

This commit is contained in:
Richard Cordovano 2014-05-19 13:07:44 -04:00
parent f8d26589e0
commit b75adfa8f2
7 changed files with 221 additions and 115 deletions

View File

@ -33,37 +33,28 @@ import org.sleuthkit.datamodel.Content;
final class DataSourceIngestPipeline { final class DataSourceIngestPipeline {
private final IngestJobContext context; private final IngestJobContext context;
private final List<IngestModuleTemplate> moduleTemplates;
private List<DataSourceIngestModuleDecorator> modules = new ArrayList<>(); private List<DataSourceIngestModuleDecorator> modules = new ArrayList<>();
DataSourceIngestPipeline(IngestJobContext context, List<IngestModuleTemplate> moduleTemplates) { DataSourceIngestPipeline(IngestJobContext context, List<IngestModuleTemplate> moduleTemplates) {
this.context = context; this.context = context;
this.moduleTemplates = moduleTemplates;
}
List<IngestModuleError> startUp() {
List<IngestModuleError> errors = new ArrayList<>();
// Create an ingest module instance from each ingest module template // Create an ingest module instance from each ingest module template
// that has an ingest module factory capable of making data source // that has an ingest module factory capable of making data source
// ingest modules. Map the module class names to the module instance // ingest modules. Map the module class names to the module instances
// to allow the modules to be put in the sequence indicated by the // to allow the modules to be put in the sequence indicated by the
// ingest pipelines configuration. // ingest pipelines configuration.
Map<String, DataSourceIngestModuleDecorator> modulesByClass = new HashMap<>(); Map<String, DataSourceIngestModuleDecorator> modulesByClass = new HashMap<>();
for (IngestModuleTemplate template : moduleTemplates) { for (IngestModuleTemplate template : moduleTemplates) {
if (template.isDataSourceIngestModuleTemplate()) { if (template.isDataSourceIngestModuleTemplate()) {
DataSourceIngestModuleDecorator module = new DataSourceIngestModuleDecorator(template.createDataSourceIngestModule(), template.getModuleName()); DataSourceIngestModuleDecorator module = new DataSourceIngestModuleDecorator(template.createDataSourceIngestModule(), template.getModuleName());
try { modulesByClass.put(module.getClassName(), module);
module.startUp(context);
modulesByClass.put(module.getClassName(), module);
} catch (Exception ex) {
errors.add(new IngestModuleError(module.getDisplayName(), ex));
}
} }
} }
// Establish the module sequence of the core ingest modules
// indicated by the ingest pipeline configuration, adding any // Add the ingest modules to the pipeline in the order indicated by the
// additional modules found in the global lookup to the end of the // data source ingest pipeline configuration, adding any additional
// pipeline in arbitrary order. // modules found in the global lookup but not mentioned in the
// configuration to the end of the pipeline in arbitrary order.
List<String> pipelineConfig = IngestPipelinesConfiguration.getInstance().getDataSourceIngestPipelineConfig(); List<String> pipelineConfig = IngestPipelinesConfiguration.getInstance().getDataSourceIngestPipelineConfig();
for (String moduleClassName : pipelineConfig) { for (String moduleClassName : pipelineConfig) {
if (modulesByClass.containsKey(moduleClassName)) { if (modulesByClass.containsKey(moduleClassName)) {
@ -73,6 +64,21 @@ final class DataSourceIngestPipeline {
for (DataSourceIngestModuleDecorator module : modulesByClass.values()) { for (DataSourceIngestModuleDecorator module : modulesByClass.values()) {
modules.add(module); modules.add(module);
} }
}
boolean isEmpty() {
return modules.isEmpty();
}
List<IngestModuleError> startUp() {
List<IngestModuleError> errors = new ArrayList<>();
for (DataSourceIngestModuleDecorator module : this.modules) {
try {
module.startUp(context);
} catch (Exception ex) {
errors.add(new IngestModuleError(module.getDisplayName(), ex));
}
}
return errors; return errors;
} }

View File

@ -0,0 +1,56 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2012-2014 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.ingest;
import java.util.HashSet;
import java.util.Set;
import java.util.concurrent.LinkedBlockingQueue;
import org.sleuthkit.datamodel.Content;
final class DataSourceIngestTaskScheduler implements IngestTaskQueue{
private static final DataSourceIngestTaskScheduler instance = new DataSourceIngestTaskScheduler();
private final Set<Long> tasksInProgress = new HashSet<>(); // Guarded by this
private final LinkedBlockingQueue<DataSourceIngestTask> dataSourceTasks = new LinkedBlockingQueue<>();
static DataSourceIngestTaskScheduler getInstance() {
return instance;
}
private DataSourceIngestTaskScheduler() {
}
synchronized void addDataSourceTask(IngestJob job, Content dataSource) throws InterruptedException {
tasksInProgress.add(job.getId());
dataSourceTasks.put(new DataSourceIngestTask(job, dataSource));
}
@Override
public IngestTask getNextTask() throws InterruptedException {
return dataSourceTasks.take();
}
synchronized void taskIsCompleted(DataSourceIngestTask task) {
tasksInProgress.remove(task.getIngestJob().getId());
}
synchronized boolean hasIncompleteTasks(IngestJob job) {
return tasksInProgress.contains(job.getId());
}
}

View File

@ -31,37 +31,28 @@ import org.sleuthkit.datamodel.AbstractFile;
final class FileIngestPipeline { final class FileIngestPipeline {
private final IngestJobContext context; private final IngestJobContext context;
private final List<IngestModuleTemplate> moduleTemplates;
private List<FileIngestModuleDecorator> modules = new ArrayList<>(); private List<FileIngestModuleDecorator> modules = new ArrayList<>();
FileIngestPipeline(IngestJobContext context, List<IngestModuleTemplate> moduleTemplates) { FileIngestPipeline(IngestJobContext context, List<IngestModuleTemplate> moduleTemplates) {
this.context = context; this.context = context;
this.moduleTemplates = moduleTemplates;
}
List<IngestModuleError> startUp() {
List<IngestModuleError> errors = new ArrayList<>();
// Create an ingest module instance from each ingest module template // Create an ingest module instance from each ingest module template
// that has an ingest module factory capable of making data source // that has an ingest module factory capable of making file ingest
// ingest modules. Map the module class names to the module instance // modules. Map the module class names to the module instances to allow
// to allow the modules to be put in the sequence indicated by the // the modules to be put in the sequence indicated by the ingest
// ingest pipelines configuration. // pipelines configuration.
Map<String, FileIngestModuleDecorator> modulesByClass = new HashMap<>(); Map<String, FileIngestModuleDecorator> modulesByClass = new HashMap<>();
for (IngestModuleTemplate template : moduleTemplates) { for (IngestModuleTemplate template : moduleTemplates) {
if (template.isFileIngestModuleTemplate()) { if (template.isFileIngestModuleTemplate()) {
FileIngestModuleDecorator module = new FileIngestModuleDecorator(template.createFileIngestModule(), template.getModuleName()); FileIngestModuleDecorator module = new FileIngestModuleDecorator(template.createFileIngestModule(), template.getModuleName());
try { modulesByClass.put(module.getClassName(), module);
module.startUp(context);
modulesByClass.put(module.getClassName(), module);
} catch (Exception ex) {
errors.add(new IngestModuleError(module.getDisplayName(), ex));
}
} }
} }
// Establish the module sequence of the core ingest modules
// indicated by the ingest pipeline configuration, adding any // Add the ingest modules to the pipeline in the order indicated by the
// additional modules found in the global lookup to the end of the // file ingest pipeline configuration, adding any additional modules
// pipeline in arbitrary order. // found in the global lookup but not mentioned in the configuration to
// the end of the pipeline in arbitrary order.
List<String> pipelineConfig = IngestPipelinesConfiguration.getInstance().getFileIngestPipelineConfig(); List<String> pipelineConfig = IngestPipelinesConfiguration.getInstance().getFileIngestPipelineConfig();
for (String moduleClassName : pipelineConfig) { for (String moduleClassName : pipelineConfig) {
if (modulesByClass.containsKey(moduleClassName)) { if (modulesByClass.containsKey(moduleClassName)) {
@ -71,12 +62,27 @@ final class FileIngestPipeline {
for (FileIngestModuleDecorator module : modulesByClass.values()) { for (FileIngestModuleDecorator module : modulesByClass.values()) {
modules.add(module); modules.add(module);
} }
}
boolean isEmpty() {
return modules.isEmpty();
}
List<IngestModuleError> startUp() {
List<IngestModuleError> errors = new ArrayList<>();
for (FileIngestModuleDecorator module : modules) {
try {
module.startUp(context);
} catch (Exception ex) {
errors.add(new IngestModuleError(module.getDisplayName(), ex));
}
}
return errors; return errors;
} }
List<IngestModuleError> process(AbstractFile file) { List<IngestModuleError> process(AbstractFile file) {
List<IngestModuleError> errors = new ArrayList<>(); List<IngestModuleError> errors = new ArrayList<>();
for (FileIngestModuleDecorator module : this.modules) { for (FileIngestModuleDecorator module : modules) {
try { try {
module.process(file); module.process(file);
} catch (Exception ex) { } catch (Exception ex) {
@ -95,7 +101,7 @@ final class FileIngestPipeline {
List<IngestModuleError> shutDown() { List<IngestModuleError> shutDown() {
List<IngestModuleError> errors = new ArrayList<>(); List<IngestModuleError> errors = new ArrayList<>();
for (FileIngestModuleDecorator module : this.modules) { for (FileIngestModuleDecorator module : modules) {
try { try {
module.shutDown(); module.shutDown();
} catch (Exception ex) { } catch (Exception ex) {
@ -105,7 +111,7 @@ final class FileIngestPipeline {
return errors; return errors;
} }
private static class FileIngestModuleDecorator implements FileIngestModule { private static final class FileIngestModuleDecorator implements FileIngestModule {
private final FileIngestModule module; private final FileIngestModule module;
private final String displayName; private final String displayName;

View File

@ -46,8 +46,8 @@ final class IngestJob {
private long estimatedFilesToProcess = 0L; // Guarded by this private long estimatedFilesToProcess = 0L; // Guarded by this
private long processedFiles = 0L; // Guarded by this private long processedFiles = 0L; // Guarded by this
private DataSourceIngestPipeline dataSourceIngestPipeline; private DataSourceIngestPipeline dataSourceIngestPipeline;
private ProgressHandle dataSourceTasksProgress; private ProgressHandle dataSourceIngestProgress;
private ProgressHandle fileTasksProgress; private ProgressHandle fileIngestProgress;
private volatile boolean cancelled = false; private volatile boolean cancelled = false;
/** /**
@ -62,15 +62,18 @@ final class IngestJob {
* @throws InterruptedException * @throws InterruptedException
*/ */
static List<IngestModuleError> startIngestJob(Content dataSource, List<IngestModuleTemplate> ingestModuleTemplates, boolean processUnallocatedSpace) throws InterruptedException { static List<IngestModuleError> startIngestJob(Content dataSource, List<IngestModuleTemplate> ingestModuleTemplates, boolean processUnallocatedSpace) throws InterruptedException {
List<IngestModuleError> errors = new ArrayList<>();
long jobId = nextIngestJobId.incrementAndGet(); long jobId = nextIngestJobId.incrementAndGet();
IngestJob job = new IngestJob(jobId, dataSource, ingestModuleTemplates, processUnallocatedSpace); IngestJob job = new IngestJob(jobId, dataSource, ingestModuleTemplates, processUnallocatedSpace);
ingestJobsById.put(jobId, job); job.createIngestPipelines();
List<IngestModuleError> errors = job.start(); if (job.canBeStarted()) {
if (errors.isEmpty()) { ingestJobsById.put(jobId, job);
IngestManager.getInstance().fireIngestJobStarted(jobId); errors = job.start();
taskScheduler.addTasksForIngestJob(job, dataSource); if (errors.isEmpty()) {
} else { IngestManager.getInstance().fireIngestJobStarted(jobId);
ingestJobsById.remove(jobId); } else {
ingestJobsById.remove(jobId);
}
} }
return errors; return errors;
} }
@ -105,11 +108,36 @@ final class IngestJob {
return processUnallocatedSpace; return processUnallocatedSpace;
} }
private void createIngestPipelines() throws InterruptedException {
IngestJobContext context = new IngestJobContext(this);
dataSourceIngestPipeline = new DataSourceIngestPipeline(context, ingestModuleTemplates);
int numberOfPipelines = IngestManager.getInstance().getNumberOfFileIngestThreads();
for (int i = 0; i < numberOfPipelines; ++i) {
fileIngestPipelines.put(new FileIngestPipeline(context, ingestModuleTemplates));
}
}
private boolean canBeStarted() {
if (!dataSourceIngestPipeline.isEmpty()) {
return true;
}
for (FileIngestPipeline pipeline : fileIngestPipelines) {
if (!pipeline.isEmpty()) {
return true;
}
}
return false;
}
private List<IngestModuleError> start() throws InterruptedException { private List<IngestModuleError> start() throws InterruptedException {
List<IngestModuleError> errors = startUpIngestPipelines(); List<IngestModuleError> errors = startUpIngestPipelines();
if (errors.isEmpty()) { if (errors.isEmpty()) {
startFileIngestProgressBar();
startDataSourceIngestProgressBar(); startDataSourceIngestProgressBar();
taskScheduler.addDataSourceTask(this, dataSource);
startFileIngestProgressBar();
if (!taskScheduler.addFileTasks(this, dataSource)) {
finishFileIngestProgressBar();
}
} }
return errors; return errors;
} }
@ -140,11 +168,11 @@ final class IngestJob {
final String displayName = NbBundle.getMessage(this.getClass(), final String displayName = NbBundle.getMessage(this.getClass(),
"IngestJob.progress.dataSourceIngest.initialDisplayName", "IngestJob.progress.dataSourceIngest.initialDisplayName",
dataSource.getName()); dataSource.getName());
dataSourceTasksProgress = ProgressHandleFactory.createHandle(displayName, new Cancellable() { dataSourceIngestProgress = ProgressHandleFactory.createHandle(displayName, new Cancellable() {
@Override @Override
public boolean cancel() { public boolean cancel() {
if (dataSourceTasksProgress != null) { if (dataSourceIngestProgress != null) {
dataSourceTasksProgress.setDisplayName( dataSourceIngestProgress.setDisplayName(
NbBundle.getMessage(this.getClass(), NbBundle.getMessage(this.getClass(),
"IngestJob.progress.cancelling", "IngestJob.progress.cancelling",
displayName)); displayName));
@ -153,19 +181,26 @@ final class IngestJob {
return true; return true;
} }
}); });
dataSourceTasksProgress.start(); dataSourceIngestProgress.start();
dataSourceTasksProgress.switchToIndeterminate(); dataSourceIngestProgress.switchToIndeterminate();
}
private synchronized void finishDataSourceIngestProgressBar() {
if (dataSourceIngestProgress != null) {
dataSourceIngestProgress.finish();
dataSourceIngestProgress = null;
}
} }
private void startFileIngestProgressBar() { private void startFileIngestProgressBar() {
final String displayName = NbBundle.getMessage(this.getClass(), final String displayName = NbBundle.getMessage(this.getClass(),
"IngestJob.progress.fileIngest.displayName", "IngestJob.progress.fileIngest.displayName",
dataSource.getName()); dataSource.getName());
fileTasksProgress = ProgressHandleFactory.createHandle(displayName, new Cancellable() { fileIngestProgress = ProgressHandleFactory.createHandle(displayName, new Cancellable() {
@Override @Override
public boolean cancel() { public boolean cancel() {
if (fileTasksProgress != null) { if (fileIngestProgress != null) {
fileTasksProgress.setDisplayName( fileIngestProgress.setDisplayName(
NbBundle.getMessage(this.getClass(), "IngestJob.progress.cancelling", NbBundle.getMessage(this.getClass(), "IngestJob.progress.cancelling",
displayName)); displayName));
} }
@ -174,30 +209,33 @@ final class IngestJob {
} }
}); });
estimatedFilesToProcess = dataSource.accept(new GetFilesCountVisitor()); estimatedFilesToProcess = dataSource.accept(new GetFilesCountVisitor());
fileTasksProgress.start(); fileIngestProgress.start();
fileTasksProgress.switchToDeterminate((int) estimatedFilesToProcess); fileIngestProgress.switchToDeterminate((int) estimatedFilesToProcess);
}
private synchronized void finishFileIngestProgressBar() {
if (fileIngestProgress != null) {
fileIngestProgress.finish();
fileIngestProgress = null;
}
} }
void process(DataSourceIngestTask task) throws InterruptedException { void process(DataSourceIngestTask task) throws InterruptedException {
if (!isCancelled()) { if (!isCancelled()) {
List<IngestModuleError> errors = new ArrayList<>(); List<IngestModuleError> errors = new ArrayList<>();
errors.addAll(dataSourceIngestPipeline.process(task.getDataSource(), dataSourceTasksProgress)); errors.addAll(dataSourceIngestPipeline.process(task.getDataSource(), dataSourceIngestProgress));
if (!errors.isEmpty()) { if (!errors.isEmpty()) {
logIngestModuleErrors(errors); logIngestModuleErrors(errors);
} }
} else { } else {
taskScheduler.removeTasksForIngestJob(id); taskScheduler.removeQueuedTasksForIngestJob(id);
} }
// taskScheduler.taskIsCompleted(task);
// Because there is only one data source task per job, it is o.k. to dataSourceIngestProgress.finish();
// call ProgressHandle.finish() now that the data source ingest modules // if (!taskScheduler.hasFileIngestTasksForIngestJob()) {
// are through using the progress bar via the DataSourceIngestModuleProgress wrapper. // finish();
// Calling ProgressHandle.finish() again in finish() will be harmless. // }
dataSourceTasksProgress.finish();
if (taskScheduler.isLastTaskForIngestJob(task)) {
finish();
}
} }
void process(FileIngestTask task) throws InterruptedException { void process(FileIngestTask task) throws InterruptedException {
@ -206,9 +244,9 @@ final class IngestJob {
synchronized (this) { synchronized (this) {
++processedFiles; ++processedFiles;
if (processedFiles <= estimatedFilesToProcess) { if (processedFiles <= estimatedFilesToProcess) {
fileTasksProgress.progress(file.getName(), (int) processedFiles); fileIngestProgress.progress(file.getName(), (int) processedFiles);
} else { } else {
fileTasksProgress.progress(file.getName(), (int) estimatedFilesToProcess); fileIngestProgress.progress(file.getName(), (int) estimatedFilesToProcess);
} }
} }
FileIngestPipeline pipeline = fileIngestPipelines.take(); FileIngestPipeline pipeline = fileIngestPipelines.take();
@ -219,12 +257,16 @@ final class IngestJob {
logIngestModuleErrors(errors); logIngestModuleErrors(errors);
} }
} else { } else {
taskScheduler.removeTasksForIngestJob(id); taskScheduler.removeQueuedTasksForIngestJob(id);
} }
// taskScheduler.taskIsCompleted(task);
if (taskScheduler.isLastTaskForIngestJob(task)) { // if (!taskScheduler.hasFileIngestTasksForIngestJob()) {
finish(); // fileIngestProgress.finish();
} // if (!taskScheduler.hasDataSourceTasksForIngestJob()) {
// finish();
// }
// }
} }
private void finish() { private void finish() {
@ -236,8 +278,6 @@ final class IngestJob {
if (!errors.isEmpty()) { if (!errors.isEmpty()) {
logIngestModuleErrors(errors); logIngestModuleErrors(errors);
} }
dataSourceTasksProgress.finish();
fileTasksProgress.finish();
ingestJobsById.remove(id); ingestJobsById.remove(id);
if (!isCancelled()) { if (!isCancelled()) {
IngestManager.getInstance().fireIngestJobCompleted(id); IngestManager.getInstance().fireIngestJobCompleted(id);

View File

@ -60,7 +60,7 @@ public final class IngestJobContext {
*/ */
public void addFiles(List<AbstractFile> files) { public void addFiles(List<AbstractFile> files) {
for (AbstractFile file : files) { for (AbstractFile file : files) {
IngestScheduler.getInstance().addFileTaskToIngestJob(ingestJob, file); IngestScheduler.getInstance().addFileTask(ingestJob, file);
} }
} }
} }

View File

@ -45,7 +45,8 @@ final class IngestScheduler {
private final TreeSet<FileIngestTask> rootDirectoryTasks = new TreeSet<>(new RootDirectoryTaskComparator()); // Guarded by this private final TreeSet<FileIngestTask> rootDirectoryTasks = new TreeSet<>(new RootDirectoryTaskComparator()); // Guarded by this
private final List<FileIngestTask> directoryTasks = new ArrayList<>(); // Guarded by this private final List<FileIngestTask> directoryTasks = new ArrayList<>(); // Guarded by this
private final LinkedBlockingQueue<FileIngestTask> fileTasks = new LinkedBlockingQueue<>(); // Guarded by this private final LinkedBlockingQueue<FileIngestTask> fileTasks = new LinkedBlockingQueue<>(); // Guarded by this
private final List<IngestTask> tasksInProgress = new ArrayList<>(); // Guarded by this private final List<IngestTask> dataSourceTasksInProgress = new ArrayList<>(); // Guarded by this
private final List<IngestTask> fileTasksInProgress = new ArrayList<>(); // Guarded by this
private final DataSourceIngestTaskQueue dataSourceTaskDispenser = new DataSourceIngestTaskQueue(); private final DataSourceIngestTaskQueue dataSourceTaskDispenser = new DataSourceIngestTaskQueue();
private final FileIngestTaskQueue fileTaskDispenser = new FileIngestTaskQueue(); private final FileIngestTaskQueue fileTaskDispenser = new FileIngestTaskQueue();
@ -56,23 +57,17 @@ final class IngestScheduler {
private IngestScheduler() { private IngestScheduler() {
} }
synchronized void addTasksForIngestJob(IngestJob job, Content dataSource) throws InterruptedException { void addDataSourceTask(IngestJob job, Content dataSource) throws InterruptedException {
// Enqueue a data source ingest task for the data source. dataSourceTasks.put(new DataSourceIngestTask(job, dataSource));
DataSourceIngestTask task = new DataSourceIngestTask(job, dataSource); }
try {
dataSourceTasks.put(task);
} catch (InterruptedException ex) {
Thread.currentThread().interrupt();
logger.log(Level.FINE, "Task scheduling for ingest job interrupted", ex); //NON-NLS
return;
}
synchronized boolean addFileTasks(IngestJob job, Content dataSource) throws InterruptedException {
// Get the top level files of the data source. // Get the top level files of the data source.
Collection<AbstractFile> rootObjects = dataSource.accept(new GetRootDirectoryVisitor()); Collection<AbstractFile> rootObjects = dataSource.accept(new GetRootDirectoryVisitor());
List<AbstractFile> toptLevelFiles = new ArrayList<>(); List<AbstractFile> topLevelFiles = new ArrayList<>();
if (rootObjects.isEmpty() && dataSource instanceof AbstractFile) { if (rootObjects.isEmpty() && dataSource instanceof AbstractFile) {
// The data source is itself a file. // The data source is itself a file.
toptLevelFiles.add((AbstractFile) dataSource); topLevelFiles.add((AbstractFile) dataSource);
} else { } else {
for (AbstractFile root : rootObjects) { for (AbstractFile root : rootObjects) {
List<Content> children; List<Content> children;
@ -81,13 +76,13 @@ final class IngestScheduler {
if (children.isEmpty()) { if (children.isEmpty()) {
// Add the root object itself, it could be an unallocated space // Add the root object itself, it could be an unallocated space
// file, or a child of a volume or an image. // file, or a child of a volume or an image.
toptLevelFiles.add(root); topLevelFiles.add(root);
} else { } else {
// The root object is a file system root directory, get // The root object is a file system root directory, get
// the files within it. // the files within it.
for (Content child : children) { for (Content child : children) {
if (child instanceof AbstractFile) { if (child instanceof AbstractFile) {
toptLevelFiles.add((AbstractFile) child); topLevelFiles.add((AbstractFile) child);
} }
} }
} }
@ -97,18 +92,22 @@ final class IngestScheduler {
} }
} }
// Enqueue file ingest tasks for the top level files. if (!topLevelFiles.isEmpty()) {
for (AbstractFile firstLevelFile : toptLevelFiles) { // Enqueue file ingest tasks for the top level files.
FileIngestTask fileTask = new FileIngestTask(job, firstLevelFile); for (AbstractFile firstLevelFile : topLevelFiles) {
if (shouldEnqueueFileTask(fileTask)) { FileIngestTask fileTask = new FileIngestTask(job, firstLevelFile);
rootDirectoryTasks.add(fileTask); if (shouldEnqueueFileTask(fileTask)) {
rootDirectoryTasks.add(fileTask);
}
} }
} updateFileTaskQueues();
return true;
} else {
return false;
}
}
updateFileTaskQueues(null); void addFileTask(IngestJob job, AbstractFile file) {
}
void addFileTaskToIngestJob(IngestJob job, AbstractFile file) {
FileIngestTask task = new FileIngestTask(job, file); FileIngestTask task = new FileIngestTask(job, file);
if (shouldEnqueueFileTask(task)) { if (shouldEnqueueFileTask(task)) {
try { try {
@ -120,7 +119,7 @@ final class IngestScheduler {
} }
} }
synchronized void removeTasksForIngestJob(long ingestJobId) { synchronized void removeQueuedTasksForIngestJob(long ingestJobId) {
// Remove all tasks for this ingest job that are not in progress. // Remove all tasks for this ingest job that are not in progress.
Iterator<FileIngestTask> fileTasksIterator = fileTasks.iterator(); Iterator<FileIngestTask> fileTasksIterator = fileTasks.iterator();
while (fileTasksIterator.hasNext()) { while (fileTasksIterator.hasNext()) {
@ -148,11 +147,7 @@ final class IngestScheduler {
} }
} }
private synchronized void updateFileTaskQueues(FileIngestTask taskInProgress) throws InterruptedException { private synchronized void updateFileTaskQueues() throws InterruptedException {
if (taskInProgress != null) {
tasksInProgress.add(taskInProgress);
}
// we loop because we could have a directory that has all files // we loop because we could have a directory that has all files
// that do not get enqueued // that do not get enqueued
while (true) { while (true) {
@ -262,7 +257,7 @@ final class IngestScheduler {
return fileTaskDispenser; return fileTaskDispenser;
} }
synchronized boolean isLastTaskForIngestJob(IngestTask completedTask) { synchronized boolean wasLastTaskForIngestJob(IngestTask completedTask) {
tasksInProgress.remove(completedTask); tasksInProgress.remove(completedTask);
IngestJob job = completedTask.getIngestJob(); IngestJob job = completedTask.getIngestJob();
long jobId = job.getId(); long jobId = job.getId();
@ -393,17 +388,20 @@ final class IngestScheduler {
private class DataSourceIngestTaskQueue implements IngestTaskQueue { private class DataSourceIngestTaskQueue implements IngestTaskQueue {
@Override @Override
public IngestTask getNextTask() throws InterruptedException { public IngestTask getNextTask() throws InterruptedException { // RJCTODO: Does this need to be synchronized?
return dataSourceTasks.take(); DataSourceIngestTask task = dataSourceTasks.take();
dataSourceTasksInProgress.add(task);
return task;
} }
} }
private class FileIngestTaskQueue implements IngestTaskQueue { private class FileIngestTaskQueue implements IngestTaskQueue {
@Override @Override
public IngestTask getNextTask() throws InterruptedException { public IngestTask getNextTask() throws InterruptedException { // RJCTODO: Does this need to be synchronized?
FileIngestTask task = fileTasks.take(); FileIngestTask task = fileTasks.take();
updateFileTaskQueues(task); fileTasksInProgress.add(task);
updateFileTaskQueues();
return task; return task;
} }
} }

View File

@ -18,6 +18,6 @@
*/ */
package org.sleuthkit.autopsy.ingest; package org.sleuthkit.autopsy.ingest;
interface IngestTaskQueue { interface IngestTaskQueue { // RJCTODO: Renmae to IngestTaskScheduler
IngestTask getNextTask() throws InterruptedException; IngestTask getNextTask() throws InterruptedException;
} }