From 719305b06730d885f7bb2238ea852d204febeb41 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Thu, 17 Apr 2014 15:33:05 -0400 Subject: [PATCH 1/5] Made getting tasks from ingest schedulers thread-safe --- .../sleuthkit/autopsy/ingest/IngestJob.java | 2 +- .../autopsy/ingest/IngestManager.java | 21 +- .../autopsy/ingest/IngestScheduler.java | 312 ++++++------------ 3 files changed, 113 insertions(+), 222 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java index af5e9b8f3a..4aa180469e 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java @@ -39,7 +39,7 @@ final class IngestJob { private final boolean processUnallocatedSpace; private final HashMap fileIngestPipelines = new HashMap<>(); private final HashMap dataSourceIngestPipelines = new HashMap<>(); - private final IngestScheduler.FileScheduler fileScheduler = IngestScheduler.getInstance().getFileScheduler(); + private final IngestScheduler.FileIngestScheduler fileScheduler = IngestScheduler.getInstance().getFileScheduler(); private FileIngestPipeline initialFileIngestPipeline = null; private DataSourceIngestPipeline initialDataSourceIngestPipeline = null; private ProgressHandle dataSourceTaskProgress; diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java index 6dadb8c5fb..ccd44c4e56 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java @@ -40,6 +40,7 @@ import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.Content; import java.util.prefs.Preferences; import javax.swing.SwingWorker; +import org.sleuthkit.autopsy.ingest.IngestScheduler.FileIngestScheduler.FileIngestTask; /** * Manages the execution of ingest jobs. @@ -315,8 +316,8 @@ public class IngestManager { } // Jettision the remaining data source and file ingest tasks. - scheduler.getFileScheduler().empty(); - scheduler.getDataSourceScheduler().empty(); + scheduler.getFileScheduler().emptyQueues(); + scheduler.getDataSourceIngestScheduler().emptyQueues(); } synchronized void reportStartIngestJobsTaskDone(long taskId) { @@ -408,7 +409,7 @@ public class IngestManager { // Queue the data source ingest tasks for the ingest job. final String inputName = dataSource.getName(); progress.progress("Data source ingest tasks for " + inputName, workUnitsCompleted); // RJCTODO: Improve - scheduler.getDataSourceScheduler().schedule(ingestJob); + scheduler.getDataSourceIngestScheduler().queueForIngest(ingestJob); progress.progress("Data source ingest tasks for " + inputName, ++workUnitsCompleted); // Queue the file ingest tasks for the ingest job. @@ -443,13 +444,14 @@ public class IngestManager { @Override public void run() { try { - IngestScheduler.DataSourceScheduler scheduler = IngestScheduler.getInstance().getDataSourceScheduler(); - while (scheduler.hasNext()) { + IngestScheduler.DataSourceIngestScheduler scheduler = IngestScheduler.getInstance().getDataSourceIngestScheduler(); + IngestJob job = scheduler.getNextTask(); + while (job != null) { if (Thread.currentThread().isInterrupted()) { break; } - IngestJob job = scheduler.next(); job.getDataSourceIngestPipelineForThread(id).process(); + job = scheduler.getNextTask(); } } catch (Exception ex) { String message = String.format("RunDataSourceIngestModulesTask (id=%d) caught exception", id); @@ -471,15 +473,16 @@ public class IngestManager { @Override public void run() { try { - IngestScheduler.FileScheduler fileScheduler = IngestScheduler.getInstance().getFileScheduler(); - while (fileScheduler.hasNext()) { + IngestScheduler.FileIngestScheduler fileScheduler = IngestScheduler.getInstance().getFileScheduler(); + FileIngestTask task = fileScheduler.getNextTask(); + while (task != null) { if (Thread.currentThread().isInterrupted()) { break; } - IngestScheduler.FileScheduler.FileTask task = fileScheduler.next(); IngestJob job = task.getJob(); job.updateFileTasksProgressBar(task.getFile().getName()); job.getFileIngestPipelineForThread(id).process(task.getFile()); + task = fileScheduler.getNextTask(); } } catch (Exception ex) { String message = String.format("RunFileSourceIngestModulesTask (id=%d) caught exception", id); diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestScheduler.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestScheduler.java index 6e9003b4cc..1e282eb6cf 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestScheduler.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestScheduler.java @@ -22,9 +22,9 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Comparator; import java.util.HashSet; -import java.util.Iterator; import java.util.LinkedList; import java.util.List; +import java.util.Objects; import java.util.Set; import java.util.TreeSet; import java.util.logging.Level; @@ -32,7 +32,7 @@ import java.util.logging.Logger; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.sleuthkit.autopsy.casemodule.Case; -import org.sleuthkit.autopsy.ingest.IngestScheduler.FileScheduler.FileTask; +import org.sleuthkit.autopsy.ingest.IngestScheduler.FileIngestScheduler.FileIngestTask; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.ContentVisitor; @@ -50,29 +50,18 @@ import org.sleuthkit.datamodel.TskData.TSK_DB_FILES_TYPE_ENUM; import org.sleuthkit.datamodel.TskData.TSK_FS_META_TYPE_ENUM; /** - * Schedules data source (images, file-sets, etc) and files with their - * associated modules for ingest, and manage queues of the scheduled tasks. - * - * Currently a singleton object only (as there is one pipeline at a time) - * - * Contains internal schedulers for content objects into data source and and - * file ingest pipelines. - * + * Enqueues data source ingest and file ingest tasks for processing. */ final class IngestScheduler { + private static IngestScheduler instance; private static final Logger logger = Logger.getLogger(IngestScheduler.class.getName()); - private final DataSourceScheduler dataSourceScheduler = new DataSourceScheduler(); - private final FileScheduler fileScheduler = new FileScheduler(); - + private final DataSourceIngestScheduler dataSourceScheduler = new DataSourceIngestScheduler(); + private final FileIngestScheduler fileScheduler = new FileIngestScheduler(); + private IngestScheduler() { } - /** - * Get ingest scheduler singleton instance - * - * @return - */ static synchronized IngestScheduler getInstance() { if (instance == null) { instance = new IngestScheduler(); @@ -81,36 +70,19 @@ final class IngestScheduler { return instance; } - DataSourceScheduler getDataSourceScheduler() { + DataSourceIngestScheduler getDataSourceIngestScheduler() { return dataSourceScheduler; } - FileScheduler getFileScheduler() { + FileIngestScheduler getFileScheduler() { return fileScheduler; } - /** - * FileScheduler ingest scheduler - * - * Supports addition ScheduledTasks - tuples of (data-source, modules) - * - * Enqueues files and modules, and sorts the files by priority. Maintains - * only top level directories in memory (not all children files of the scheduled container content objects) - * - * getNext() will return next FileTask - tuple of (file, modules) - * - */ - static class FileScheduler implements Iterator { - //root folders enqueued - private TreeSet rootDirectoryTasks; - - //stack of current dirs to be processed recursively - private List directoryTasks; - - //list of files being processed in the currently processed directory - private LinkedList fileTasks; //need to add to start and end quickly - - //estimated total files to be enqueued for currently scheduled content objects + static class FileIngestScheduler { + + private TreeSet rootDirectoryTasks; + private List directoryTasks; + private LinkedList fileTasks; //need to add to start and end quickly private int filesEnqueuedEst = 0; private int filesDequeued = 0; private final static int FAT_NTFS_FLAGS = TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT12.getValue() @@ -118,34 +90,16 @@ final class IngestScheduler { | TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT32.getValue() | TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_NTFS.getValue(); - private FileScheduler() { + private FileIngestScheduler() { rootDirectoryTasks = new TreeSet<>(new RootTaskComparator()); directoryTasks = new ArrayList<>(); fileTasks = new LinkedList<>(); - resetCounters(); + resetCounters(); } - + private void resetCounters() { filesEnqueuedEst = 0; filesDequeued = 0; - } - - @Override - public synchronized String toString() { - StringBuilder sb = new StringBuilder(); - sb.append("\nRootDirs(sorted), size: ").append(rootDirectoryTasks.size()); - for (FileTask task : rootDirectoryTasks) { - sb.append(task.toString()).append(" "); - } - sb.append("\nCurDirs(stack), size: ").append(directoryTasks.size()); - for (FileTask task : directoryTasks) { - sb.append(task.toString()).append(" "); - } - sb.append("\nCurFiles, size: ").append(fileTasks.size()); - for (FileTask task : fileTasks) { - sb.append(task.toString()).append(" "); - } - return sb.toString(); } synchronized void scheduleIngestOfFiles(IngestJob dataSourceTask) { @@ -154,9 +108,8 @@ final class IngestScheduler { List firstLevelFiles = new ArrayList<>(); if (rootObjects.isEmpty() && dataSource instanceof AbstractFile) { // The data source is file. - firstLevelFiles.add((AbstractFile)dataSource); - } - else { + firstLevelFiles.add((AbstractFile) dataSource); + } else { for (AbstractFile root : rootObjects) { List children; try { @@ -164,8 +117,7 @@ final class IngestScheduler { if (children.isEmpty()) { //add the root itself, could be unalloc file, child of volume or image firstLevelFiles.add(root); - } - else { + } else { //root for fs root dir, schedule children dirs/files for (Content child : children) { if (child instanceof AbstractFile) { @@ -173,15 +125,14 @@ final class IngestScheduler { } } } - } - catch (TskCoreException ex) { + } catch (TskCoreException ex) { logger.log(Level.WARNING, "Could not get children of root to enqueue: " + root.getId() + ": " + root.getName(), ex); } } } for (AbstractFile firstLevelFile : firstLevelFiles) { - FileTask fileTask = new FileTask(firstLevelFile, dataSourceTask); + FileIngestTask fileTask = new FileIngestTask(firstLevelFile, dataSourceTask); if (shouldEnqueueTask(fileTask)) { rootDirectoryTasks.add(fileTask); } @@ -193,7 +144,7 @@ final class IngestScheduler { // Reshuffle/update the dir and file level queues if needed updateQueues(); } - + /** * Schedule a file to the file ingest, with associated modules. This * will add the file to beginning of the file queue. The method is @@ -203,17 +154,18 @@ final class IngestScheduler { * as the parent origin file. * * @param file file to be scheduled - * @param originalContext original content schedule context that was used - * to schedule the parent origin content, with the modules, settings, etc. + * @param originalContext original content schedule context that was + * used to schedule the parent origin content, with the modules, + * settings, etc. */ synchronized void scheduleFile(IngestJob ingestJob, AbstractFile file) { - FileTask fileTask = new FileTask(file, ingestJob); + FileIngestTask fileTask = new FileIngestTask(file, ingestJob); if (shouldEnqueueTask(fileTask)) { fileTasks.addFirst(fileTask); ++filesEnqueuedEst; - } - } - + } + } + float getPercentageDone() { if (filesEnqueuedEst == 0) { return 0; @@ -238,7 +190,7 @@ final class IngestScheduler { totalFiles += content.accept(countVisitor); } - logger.log(Level.INFO, "Total files to queue up: " + totalFiles); + logger.log(Level.INFO, "Total files to queue up: {0}", totalFiles); return totalFiles; } @@ -262,37 +214,21 @@ final class IngestScheduler { int getFilesDequeued() { return filesDequeued; } - - @Override - public synchronized boolean hasNext() { - if (fileTasks.isEmpty()) { - filesEnqueuedEst = 0; - filesDequeued = 0; - return false; - } - return true; - } - @Override - public synchronized FileTask next() { - if (!hasNext()) { - throw new IllegalStateException("No next ProcessTask, check hasNext() first!"); - } - - //dequeue the last in the list - final FileTask task = fileTasks.pollLast(); + synchronized FileIngestTask getNextTask() { + final FileIngestTask task = fileTasks.pollLast(); filesDequeued++; updateQueues(); - return task; } - /** + /** * Shuffle the queues so that there are files in the files queue. + * * @returns true if no more data in queue */ private synchronized void updateQueues() { - + // we loop because we could have a directory that has all files // that do not get enqueued while (true) { @@ -300,20 +236,20 @@ final class IngestScheduler { if (this.fileTasks.isEmpty() == false) { return; } - + // fill in the directory queue if it is empty. if (this.directoryTasks.isEmpty()) { // bail out if root is also empty -- we are done if (rootDirectoryTasks.isEmpty()) { return; } - FileTask rootTask = this.rootDirectoryTasks.pollFirst(); + FileIngestTask rootTask = this.rootDirectoryTasks.pollFirst(); directoryTasks.add(rootTask); } //pop and push AbstractFile directory children if any //add the popped and its leaf children onto cur file list - FileTask parentTask = directoryTasks.remove(directoryTasks.size() - 1); + FileIngestTask parentTask = directoryTasks.remove(directoryTasks.size() - 1); final AbstractFile parentFile = parentTask.file; // add itself to the file list @@ -327,12 +263,11 @@ final class IngestScheduler { for (Content c : children) { if (c instanceof AbstractFile) { AbstractFile childFile = (AbstractFile) c; - FileTask childTask = new FileTask(childFile, parentTask.getJob()); + FileIngestTask childTask = new FileIngestTask(childFile, parentTask.getJob()); if (childFile.hasChildren()) { this.directoryTasks.add(childTask); - } - else if (shouldEnqueueTask(childTask)) { + } else if (shouldEnqueueTask(childTask)) { this.fileTasks.addLast(childTask); } } @@ -344,13 +279,8 @@ final class IngestScheduler { } } - @Override - public void remove() { - throw new UnsupportedOperationException("Not supported."); - } - /** - * Return list of content objects that are in the queue to be processed. + * Return list of content objects that are in the queue to be processed. * * Helpful to determine whether ingest for particular input Content is * active @@ -361,20 +291,20 @@ final class IngestScheduler { synchronized List getSourceContent() { final Set contentSet = new HashSet<>(); - for (FileTask task : rootDirectoryTasks) { + for (FileIngestTask task : rootDirectoryTasks) { contentSet.add(task.getJob().getDataSource()); } - for (FileTask task : directoryTasks) { + for (FileIngestTask task : directoryTasks) { contentSet.add(task.getJob().getDataSource()); } - for (FileTask task : fileTasks) { + for (FileIngestTask task : fileTasks) { contentSet.add(task.getJob().getDataSource()); } return new ArrayList<>(contentSet); } - synchronized void empty() { + synchronized void emptyQueues() { this.rootDirectoryTasks.clear(); this.directoryTasks.clear(); this.fileTasks.clear(); @@ -387,7 +317,7 @@ final class IngestScheduler { * skipped * @return true if should be enqueued, false otherwise */ - private static boolean shouldEnqueueTask(final FileTask processTask) { + private static boolean shouldEnqueueTask(final FileIngestTask processTask) { final AbstractFile aFile = processTask.file; //if it's unalloc file, skip if so scheduled @@ -445,23 +375,20 @@ final class IngestScheduler { return true; } - /** - * Task for a specific file to process. More specific than the - * higher-level DataSourceTask. - */ - static class FileTask { + static class FileIngestTask { + private final AbstractFile file; private final IngestJob task; - public FileTask(AbstractFile file, IngestJob task) { + private FileIngestTask(AbstractFile file, IngestJob task) { this.file = file; this.task = task; } - + public IngestJob getJob() { return task; } - + public AbstractFile getFile() { return file; } @@ -494,7 +421,7 @@ final class IngestScheduler { if (getClass() != obj.getClass()) { return false; } - final FileTask other = (FileTask) obj; + final FileIngestTask other = (FileIngestTask) obj; if (this.file != other.file && (this.file == null || !this.file.equals(other.file))) { return false; } @@ -507,17 +434,25 @@ final class IngestScheduler { } return true; } - } - + + @Override + public int hashCode() { + int hash = 5; + hash = 47 * hash + Objects.hashCode(this.file); + hash = 47 * hash + Objects.hashCode(this.task); + return hash; + } + } + /** * Root dir sorter */ - private static class RootTaskComparator implements Comparator { + private static class RootTaskComparator implements Comparator { @Override - public int compare(FileTask q1, FileTask q2) { - AbstractFilePriotity.Priority p1 = AbstractFilePriotity.getPriority(q1.file); - AbstractFilePriotity.Priority p2 = AbstractFilePriotity.getPriority(q2.file); + public int compare(FileIngestTask q1, FileIngestTask q2) { + AbstractFilePriority.Priority p1 = AbstractFilePriority.getPriority(q1.file); + AbstractFilePriority.Priority p2 = AbstractFilePriority.getPriority(q2.file); if (p1 == p2) { return (int) (q2.file.getId() - q1.file.getId()); } else { @@ -530,16 +465,16 @@ final class IngestScheduler { * Priority determination for sorted AbstractFile, used by * RootDirComparator */ - private static class AbstractFilePriotity { + private static class AbstractFilePriority { enum Priority { LAST, LOW, MEDIUM, HIGH }; - static final List LAST_PRI_PATHS = new ArrayList(); - static final List LOW_PRI_PATHS = new ArrayList(); - static final List MEDIUM_PRI_PATHS = new ArrayList(); - static final List HIGH_PRI_PATHS = new ArrayList(); + static final List LAST_PRI_PATHS = new ArrayList<>(); + static final List LOW_PRI_PATHS = new ArrayList<>(); + static final List MEDIUM_PRI_PATHS = new ArrayList<>(); + static final List HIGH_PRI_PATHS = new ArrayList<>(); /* prioritize root directory folders based on the assumption that we are * looking for user content. Other types of investigations may want different @@ -568,54 +503,55 @@ final class IngestScheduler { } /** - * Get the scheduling priority for a given file. + * Get the scheduling priority for a given file. + * * @param abstractFile - * @return + * @return */ - static AbstractFilePriotity.Priority getPriority(final AbstractFile abstractFile) { + static AbstractFilePriority.Priority getPriority(final AbstractFile abstractFile) { if (!abstractFile.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.FS)) { //quickly filter out unstructured content //non-fs virtual files and dirs, such as representing unalloc space - return AbstractFilePriotity.Priority.LAST; + return AbstractFilePriority.Priority.LAST; } //determine the fs files priority by name final String path = abstractFile.getName(); if (path == null) { - return AbstractFilePriotity.Priority.MEDIUM; + return AbstractFilePriority.Priority.MEDIUM; } for (Pattern p : HIGH_PRI_PATHS) { Matcher m = p.matcher(path); if (m.find()) { - return AbstractFilePriotity.Priority.HIGH; + return AbstractFilePriority.Priority.HIGH; } } for (Pattern p : MEDIUM_PRI_PATHS) { Matcher m = p.matcher(path); if (m.find()) { - return AbstractFilePriotity.Priority.MEDIUM; + return AbstractFilePriority.Priority.MEDIUM; } } for (Pattern p : LOW_PRI_PATHS) { Matcher m = p.matcher(path); if (m.find()) { - return AbstractFilePriotity.Priority.LOW; + return AbstractFilePriority.Priority.LOW; } } for (Pattern p : LAST_PRI_PATHS) { Matcher m = p.matcher(path); if (m.find()) { - return AbstractFilePriotity.Priority.LAST; + return AbstractFilePriority.Priority.LAST; } } //default is medium - return AbstractFilePriotity.Priority.MEDIUM; + return AbstractFilePriority.Priority.MEDIUM; } } } @@ -623,8 +559,8 @@ final class IngestScheduler { /** * Get counts of ingestable files/dirs for the content input source. * - * Note, also includes counts of all unalloc children files (for the fs, image, volume) even - * if ingest didn't ask for them + * Note, also includes counts of all unalloc children files (for the fs, + * image, volume) even if ingest didn't ask for them */ static class GetFilesCountVisitor extends ContentVisitor.Default { @@ -649,7 +585,7 @@ final class IngestScheduler { //queryB.append(")"); try { final String query = queryB.toString(); - logger.log(Level.INFO, "Executing count files query: " + query); + logger.log(Level.INFO, "Executing count files query: {0}", query); return sc.countFilesWhere(query); } catch (TskCoreException ex) { logger.log(Level.SEVERE, "Couldn't get count of all files in FileSystem", ex); @@ -700,7 +636,7 @@ final class IngestScheduler { public Collection visit(VirtualDirectory ld) { //case when we hit a layout directoryor local file container, not under a real FS //or when root virt dir is scheduled - Collection ret = new ArrayList(); + Collection ret = new ArrayList<>(); ret.add(ld); return ret; } @@ -708,7 +644,7 @@ final class IngestScheduler { @Override public Collection visit(LayoutFile lf) { //case when we hit a layout file, not under a real FS - Collection ret = new ArrayList(); + Collection ret = new ArrayList<>(); ret.add(lf); return ret; } @@ -717,7 +653,7 @@ final class IngestScheduler { public Collection visit(Directory drctr) { //we hit a real directory, a child of real FS - Collection ret = new ArrayList(); + Collection ret = new ArrayList<>(); ret.add(drctr); @@ -752,81 +688,33 @@ final class IngestScheduler { } } - /** - * DataSourceScheduler ingest scheduler - */ - static class DataSourceScheduler implements Iterator { + static class DataSourceIngestScheduler { - private LinkedList tasks; + private final LinkedList tasks = new LinkedList<>(); - DataSourceScheduler() { - tasks = new LinkedList<>(); + private DataSourceIngestScheduler() { } - synchronized void schedule(IngestJob task) { + synchronized void queueForIngest(IngestJob job) { try { - if (task.getDataSource().getParent() != null) { - //only accepting parent-less content objects (Image, parentless VirtualDirectory) - logger.log(Level.SEVERE, "Only parent-less Content (data sources) can be scheduled for DataSource ingest, skipping: {0}", task.getDataSource()); + if (job.getDataSource().getParent() != null) { + logger.log(Level.SEVERE, "Only parent-less Content (data sources) can be scheduled for DataSource ingest, skipping: {0}", job.getDataSource()); return; } } catch (TskCoreException e) { - logger.log(Level.SEVERE, "Error validating data source to be scheduled for DataSource ingest" + task.getDataSource(), e); + logger.log(Level.SEVERE, "Error validating data source to be scheduled for DataSource ingest" + job.getDataSource(), e); return; } - tasks.addLast(task); + tasks.addLast(job); } - @Override - public synchronized IngestJob next() throws IllegalStateException { - if (!hasNext()) { - throw new IllegalStateException("There is no data source tasks in the queue, check hasNext()"); - } - - final IngestJob ret = tasks.pollFirst(); - return ret; + public synchronized IngestJob getNextTask() { + return tasks.pollFirst(); } - /** - * get all data source that are scheduled to process - * - * @return list of data sources in the queue scheduled to process - */ - synchronized List getContents() { - List contents = new ArrayList(); - for (IngestJob task : tasks) { - contents.add(task.getDataSource()); - } - return contents; - } - - @Override - public synchronized boolean hasNext() { - return !tasks.isEmpty(); - } - - @Override - public void remove() { - throw new UnsupportedOperationException("Removing of scheduled data source ingest tasks is not supported. "); - } - - synchronized void empty() { + synchronized void emptyQueues() { tasks.clear(); } - - synchronized int getCount() { - return tasks.size(); - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append("DataSourceQueue, size: ").append(getCount()); - for (IngestJob task : tasks) { - sb.append(task.toString()).append(" "); - } - return sb.toString(); - } } } \ No newline at end of file From 2b62498997cfdd6341e6b442ab7805e033a122f9 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Fri, 18 Apr 2014 09:09:05 -0400 Subject: [PATCH 2/5] Renamed file ingest scheduler method --- Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java | 2 +- .../org/sleuthkit/autopsy/ingest/IngestManager.java | 8 ++++---- .../org/sleuthkit/autopsy/ingest/IngestScheduler.java | 10 +++++----- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java index 4aa180469e..6d41cac96a 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java @@ -39,7 +39,7 @@ final class IngestJob { private final boolean processUnallocatedSpace; private final HashMap fileIngestPipelines = new HashMap<>(); private final HashMap dataSourceIngestPipelines = new HashMap<>(); - private final IngestScheduler.FileIngestScheduler fileScheduler = IngestScheduler.getInstance().getFileScheduler(); + private final IngestScheduler.FileIngestScheduler fileScheduler = IngestScheduler.getInstance().getFileIngestScheduler(); private FileIngestPipeline initialFileIngestPipeline = null; private DataSourceIngestPipeline initialDataSourceIngestPipeline = null; private ProgressHandle dataSourceTaskProgress; diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java index ccd44c4e56..c3829e7c5b 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java @@ -130,7 +130,7 @@ public class IngestManager { synchronized void addFileToIngestJob(long ingestJobId, AbstractFile file) { IngestJob job = ingestJobs.get(ingestJobId); if (job != null) { - scheduler.getFileScheduler().scheduleFile(job, file); + scheduler.getFileIngestScheduler().scheduleFile(job, file); } } @@ -316,7 +316,7 @@ public class IngestManager { } // Jettision the remaining data source and file ingest tasks. - scheduler.getFileScheduler().emptyQueues(); + scheduler.getFileIngestScheduler().emptyQueues(); scheduler.getDataSourceIngestScheduler().emptyQueues(); } @@ -414,7 +414,7 @@ public class IngestManager { // Queue the file ingest tasks for the ingest job. progress.progress("Data source ingest tasks for " + inputName, workUnitsCompleted); - scheduler.getFileScheduler().scheduleIngestOfFiles(ingestJob); + scheduler.getFileIngestScheduler().scheduleIngestOfFiles(ingestJob); progress.progress("Data source ingest tasks for " + inputName, ++workUnitsCompleted); if (!Thread.currentThread().isInterrupted()) { @@ -473,7 +473,7 @@ public class IngestManager { @Override public void run() { try { - IngestScheduler.FileIngestScheduler fileScheduler = IngestScheduler.getInstance().getFileScheduler(); + IngestScheduler.FileIngestScheduler fileScheduler = IngestScheduler.getInstance().getFileIngestScheduler(); FileIngestTask task = fileScheduler.getNextTask(); while (task != null) { if (Thread.currentThread().isInterrupted()) { diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestScheduler.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestScheduler.java index 1e282eb6cf..32005d66e6 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestScheduler.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestScheduler.java @@ -56,8 +56,8 @@ final class IngestScheduler { private static IngestScheduler instance; private static final Logger logger = Logger.getLogger(IngestScheduler.class.getName()); - private final DataSourceIngestScheduler dataSourceScheduler = new DataSourceIngestScheduler(); - private final FileIngestScheduler fileScheduler = new FileIngestScheduler(); + private final DataSourceIngestScheduler dataSourceIngestScheduler = new DataSourceIngestScheduler(); + private final FileIngestScheduler fileIngestScheduler = new FileIngestScheduler(); private IngestScheduler() { } @@ -71,11 +71,11 @@ final class IngestScheduler { } DataSourceIngestScheduler getDataSourceIngestScheduler() { - return dataSourceScheduler; + return dataSourceIngestScheduler; } - FileIngestScheduler getFileScheduler() { - return fileScheduler; + FileIngestScheduler getFileIngestScheduler() { + return fileIngestScheduler; } static class FileIngestScheduler { From 0b5524e705224a5c6f898321f851a397e2e9d4d6 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Mon, 21 Apr 2014 09:27:42 -0400 Subject: [PATCH 3/5] Renamed IngestScheduler.FileIngestScheduler method --- Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java | 2 +- Core/src/org/sleuthkit/autopsy/ingest/IngestScheduler.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java index c4a6f0202b..210a839749 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java @@ -417,7 +417,7 @@ public class IngestManager { // Queue the file ingest tasks for the ingest job. progress.progress("Data source ingest tasks for " + inputName, workUnitsCompleted); - scheduler.getFileIngestScheduler().scheduleIngestOfFiles(ingestJob); + scheduler.getFileIngestScheduler().queueForIngest(ingestJob); progress.progress("Data source ingest tasks for " + inputName, ++workUnitsCompleted); if (!Thread.currentThread().isInterrupted()) { diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestScheduler.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestScheduler.java index 971570a581..8fd228fc04 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestScheduler.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestScheduler.java @@ -122,7 +122,7 @@ final class IngestScheduler { return sb.toString(); } - synchronized void scheduleIngestOfFiles(IngestJob dataSourceTask) { + synchronized void queueForIngest(IngestJob dataSourceTask) { Content dataSource = dataSourceTask.getDataSource(); Collection rootObjects = dataSource.accept(new GetRootDirVisitor()); List firstLevelFiles = new ArrayList<>(); From bd6f90c8954126e5964abe7b28490f6df4ddb0d5 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Mon, 21 Apr 2014 09:31:35 -0400 Subject: [PATCH 4/5] Renamed another IngestScheduler.FileIngestScheduler method --- .../sleuthkit/autopsy/ingest/IngestManager.java | 2 +- .../sleuthkit/autopsy/ingest/IngestScheduler.java | 15 +-------------- 2 files changed, 2 insertions(+), 15 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java index 210a839749..c5fa91a01a 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java @@ -130,7 +130,7 @@ public class IngestManager { synchronized void addFileToIngestJob(long ingestJobId, AbstractFile file) { IngestJob job = ingestJobs.get(ingestJobId); if (job != null) { - scheduler.getFileIngestScheduler().scheduleFile(job, file); + scheduler.getFileIngestScheduler().queueFile(job, file); } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestScheduler.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestScheduler.java index 8fd228fc04..3c4d3d0cad 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestScheduler.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestScheduler.java @@ -165,20 +165,7 @@ final class IngestScheduler { updateQueues(); } - /** - * Schedule a file to the file ingest, with associated modules. This - * will add the file to beginning of the file queue. The method is - * intended for rescheduling a file that is a derivative of another - * content object that has already ingested and produced this file. As a - * result, the derivative file will be scheduled with the same priority - * as the parent origin file. - * - * @param file file to be scheduled - * @param originalContext original content schedule context that was - * used to schedule the parent origin content, with the modules, - * settings, etc. - */ - synchronized void scheduleFile(IngestJob ingestJob, AbstractFile file) { + synchronized void queueFile(IngestJob ingestJob, AbstractFile file) { FileIngestTask fileTask = new FileIngestTask(file, ingestJob); if (shouldEnqueueTask(fileTask)) { fileTasks.addFirst(fileTask); From 8ce3af30e21cdcdaa06c774e9e77aa1c2b72b900 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Mon, 21 Apr 2014 10:00:01 -0400 Subject: [PATCH 5/5] Added null cehck to IngestScheduler.FileIngestScheduler --- Core/src/org/sleuthkit/autopsy/ingest/IngestScheduler.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestScheduler.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestScheduler.java index 3c4d3d0cad..94583a8b42 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestScheduler.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestScheduler.java @@ -224,8 +224,10 @@ final class IngestScheduler { synchronized FileIngestTask getNextTask() { final FileIngestTask task = fileTasks.pollLast(); - filesDequeued++; - updateQueues(); + if (task != null) { + filesDequeued++; + updateQueues(); + } return task; }