Merge remote-tracking branch 'upstream/develop' into develop

Core/src/org/sleuthkit/autopsy/coreutils/ExecUtil.java

@@ -18,11 +18,13 @@
  */
 package org.sleuthkit.autopsy.coreutils;
 
+import com.sun.javafx.PlatformUtil;
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.io.Writer;
+import java.util.List;
 import java.util.concurrent.TimeUnit;
 import java.util.logging.Level;
 
@@ -108,12 +110,12 @@ public final class ExecUtil {
             do {
                 process.waitFor(timeOut, units);
                 if (process.isAlive() && terminator.shouldTerminateProcess()) {
-                    process.destroyForcibly();
+                    killProcess(process);
                 }
             } while (process.isAlive());
         } catch (InterruptedException ex) {
             if (process.isAlive()) {
-                process.destroyForcibly();
+                killProcess(process);
             }
             Logger.getLogger(ExecUtil.class.getName()).log(Level.INFO, "Thread interrupted while running {0}", processBuilder.command().get(0));
             Thread.currentThread().interrupt();
@@ -121,6 +123,33 @@ public final class ExecUtil {
         return process.exitValue();
     }
 
+    /**
+     * Kill a process and its children
+     * @param process The parent process to kill
+     */
+    public static void killProcess(Process process) {
+        if (process == null)
+            return;
+        
+        try {
+            if (PlatformUtil.isWindows()) {
+                Win32Process parentProcess = new Win32Process(process);
+                List<Win32Process> children = parentProcess.getChildren();
+
+                children.stream().forEach((child) -> {
+                    child.terminate();
+                });
+                parentProcess.terminate();
+            }
+            else {
+                process.destroyForcibly();
+            }
+        }
+        catch (Exception ex) {
+            logger.log(Level.WARNING, "Error occurred when attempting to kill process: {0}", ex.getMessage()); // NON-NLS
+        }
+    }
+            
     private static final Logger logger = Logger.getLogger(ExecUtil.class.getName());
     private Process proc = null;
     private ExecUtil.StreamToStringRedirect errorStringRedirect = null;

Core/src/org/sleuthkit/autopsy/coreutils/Win32Process.java

@@ -0,0 +1,115 @@
+/*
+ * Autopsy Forensic Browser
+ *
+ * Copyright 2012-2014 Basis Technology Corp.
+ * Contact: carrier <at> sleuthkit <dot> org
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.sleuthkit.autopsy.coreutils;
+
+import com.sun.jna.Pointer;
+import com.sun.jna.platform.win32.Kernel32;
+import com.sun.jna.platform.win32.Kernel32Util;
+import com.sun.jna.platform.win32.Tlhelp32;
+import com.sun.jna.platform.win32.WinDef.DWORD;
+import com.sun.jna.platform.win32.WinNT;
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Class that represents a Windows process.
+ * It uses JNA to access the Win32 API.
+ * This code is based on http://stackoverflow.com/questions/10124299/how-do-i-terminate-a-process-tree-from-java
+ */
+public class Win32Process {
+    WinNT.HANDLE handle;
+    int pid;
+
+    /**
+     * Create a Win32Process object for the given Process object.
+     * Reflection is used to construct a Windows process handle.
+     * @param process A Java Process object
+     * @throws Exception 
+     */
+    Win32Process (Process process) throws Exception
+    {
+        if (process.getClass().getName().equals("java.lang.Win32Process") || // NON-NLS
+                process.getClass().getName().equals("java.lang.ProcessImpl")) { // NON-NLS
+            try {
+                Field f = process.getClass().getDeclaredField("handle"); // NON-NLS
+                f.setAccessible(true);
+                long handleVal = f.getLong(process);
+                handle = new WinNT.HANDLE(Pointer.createConstant(handleVal));
+            }
+            catch (NoSuchFieldException | SecurityException | IllegalArgumentException | IllegalAccessException ex) {
+                throw new Exception(ex.getMessage()); // NON-NLS
+            }
+        }
+        this.pid = Kernel32.INSTANCE.GetProcessId(handle);
+    }
+    
+    /**
+     * Create a Win32Process object for the given process id.
+     * @param pid Process Id
+     * @throws Exception 
+     */
+    Win32Process (int pid) throws Exception
+    {
+        handle = Kernel32.INSTANCE.OpenProcess ( 
+                0x0400| /* PROCESS_QUERY_INFORMATION */
+                0x0800| /* PROCESS_SUSPEND_RESUME */
+                0x0001| /* PROCESS_TERMINATE */
+                0x00100000 /* SYNCHRONIZE */,
+                false,
+                pid);
+        if (handle == null) 
+            throw new Exception (Kernel32Util.formatMessageFromLastErrorCode (Kernel32.INSTANCE.GetLastError ()));
+        this.pid = Kernel32.INSTANCE.GetProcessId(handle);
+    }
+
+    @Override
+    protected void finalize () throws Throwable
+    {
+        Kernel32.INSTANCE.CloseHandle (handle);
+        super.finalize();
+    }
+
+    /**
+     * Kill the process. Note that this does not kill children.
+     */
+    public void terminate ()
+    {
+        Kernel32.INSTANCE.TerminateProcess (handle, 0);
+    }
+
+    /**
+     * Get children of current process object.
+     * @return list of child processes
+     * @throws IOException 
+     */
+    public List<Win32Process> getChildren () throws Exception
+    {
+        ArrayList<Win32Process> result = new ArrayList<> ();
+        WinNT.HANDLE hSnap = Kernel32.INSTANCE.CreateToolhelp32Snapshot(Tlhelp32.TH32CS_SNAPPROCESS, new DWORD(0));
+        Tlhelp32.PROCESSENTRY32.ByReference ent = new Tlhelp32.PROCESSENTRY32.ByReference ();
+        if (!Kernel32.INSTANCE.Process32First (hSnap, ent)) return result;
+        do {
+            if (ent.th32ParentProcessID.intValue () == pid) result.add (new Win32Process (ent.th32ProcessID.intValue ()));
+        } while (Kernel32.INSTANCE.Process32Next (hSnap, ent));
+        Kernel32.INSTANCE.CloseHandle (hSnap);
+        return result;
+    }    
+}

Core/src/org/sleuthkit/autopsy/examples/SampleExecutableDataSourceIngestModule.java

@@ -61,7 +61,6 @@ import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE;
 import org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE;
 import org.sleuthkit.datamodel.Content;
 import org.sleuthkit.datamodel.Image;
-import org.sleuthkit.datamodel.TskCoreException;
 import org.w3c.dom.Document;
 import org.w3c.dom.Element;
 

Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java

@@ -43,7 +43,7 @@ final class DataSourceIngestPipeline {
                 DataSourceIngestModuleDecorator module = new DataSourceIngestModuleDecorator(template.createDataSourceIngestModule(), template.getModuleName());
                 modules.add(module);
             }
-        }
+        }        
     }
 
     boolean isEmpty() {

Core/src/org/sleuthkit/autopsy/ingest/IngestJobContext.java

@@ -19,8 +19,6 @@
 package org.sleuthkit.autopsy.ingest;
 
 import java.util.List;
-import java.util.logging.Level;
-import org.sleuthkit.autopsy.coreutils.Logger;
 import org.sleuthkit.datamodel.AbstractFile;
 import org.sleuthkit.datamodel.Content;
 
@@ -30,8 +28,6 @@ import org.sleuthkit.datamodel.Content;
  */
 public final class IngestJobContext {
 
-    private static final Logger logger = Logger.getLogger(IngestJobContext.class.getName());
-    private static final IngestScheduler scheduler = IngestScheduler.getInstance();
     private final IngestJob ingestJob;
 
     IngestJobContext(IngestJob ingestJob) {
@@ -103,24 +99,25 @@ public final class IngestJobContext {
     }
 
     /**
-     * Adds one or more files to the files to be passed through the file ingest
-     * pipeline of the ingest job associated with this context.
+     * Adds one or more files, i.e., extracted or carved files, to the ingest
+     * job associated with this context.
      *
-     * @param files The files to be processed by the file ingest pipeline.
+     * @param files The files to be added.
+     * @deprecated use addFilesToJob() instead
      */
+    @Deprecated
     public void scheduleFiles(List<AbstractFile> files) {
-        for (AbstractFile file : files) {
-            try {
-                IngestJobContext.scheduler.scheduleAdditionalFileIngestTask(this.ingestJob, file);
-            } catch (InterruptedException ex) {
-                // Handle the unexpected interrupt here rather than make ingest 
-                // module writers responsible for writing this exception handler. 
-                // The interrupt flag of the thread is reset for detection by 
-                // the thread task code.  
-                Thread.currentThread().interrupt();
-                IngestJobContext.logger.log(Level.SEVERE, "File task scheduling unexpectedly interrupted", ex); //NON-NLS
-            }
-        }
+        this.addFilesToJob(files);
     }
-    
+
+    /**
+     * Adds one or more files, i.e., extracted or carved files, to the ingest
+     * job associated with this context.
+     *
+     * @param files The files to be added.
+     */
+    public void addFilesToJob(List<AbstractFile> files) {
+        this.ingestJob.addFiles(files);
+    }
+
 }

Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java

@@ -134,7 +134,7 @@ public class IngestManager {
      */
     private void startDataSourceIngestTask() {
         long threadId = nextThreadId.incrementAndGet();
-        dataSourceIngestThreadPool.submit(new ExecuteIngestTasksTask(threadId, IngestScheduler.getInstance().getDataSourceIngestTaskQueue()));
+        dataSourceIngestThreadPool.submit(new ExecuteIngestTasksRunnable(threadId, IngestTasksScheduler.getInstance().getDataSourceIngestTaskQueue()));
         ingestThreadActivitySnapshots.put(threadId, new IngestThreadActivitySnapshot(threadId));
     }
 
@@ -144,7 +144,7 @@ public class IngestManager {
      */
     private void startFileIngestTask() {
         long threadId = nextThreadId.incrementAndGet();
-        fileIngestThreadPool.submit(new ExecuteIngestTasksTask(threadId, IngestScheduler.getInstance().getFileIngestTaskQueue()));
+        fileIngestThreadPool.submit(new ExecuteIngestTasksRunnable(threadId, IngestTasksScheduler.getInstance().getFileIngestTaskQueue()));
         ingestThreadActivitySnapshots.put(threadId, new IngestThreadActivitySnapshot(threadId));
     }
 
@@ -154,7 +154,7 @@ public class IngestManager {
         }
 
         long taskId = nextThreadId.incrementAndGet();
-        Future<Void> task = startIngestJobsThreadPool.submit(new StartIngestJobsTask(taskId, dataSources, moduleTemplates, processUnallocatedSpace));
+        Future<Void> task = startIngestJobsThreadPool.submit(new StartIngestJobsCallable(taskId, dataSources, moduleTemplates, processUnallocatedSpace));
         startIngestJobsTasks.put(taskId, task);
     }
 
@@ -174,12 +174,12 @@ public class IngestManager {
     }
 
     void handleCaseOpened() {
-        IngestScheduler.getInstance().setEnabled(true);
+        IngestJob.jobCreationEnabled(true);
         clearIngestMessageBox();
     }
 
     void handleCaseClosed() {
-        IngestScheduler.getInstance().setEnabled(false);
+        IngestJob.jobCreationEnabled(false);
         cancelAllIngestJobs();
         clearIngestMessageBox();
     }
@@ -197,14 +197,14 @@ public class IngestManager {
      * @return True if any ingest jobs are in progress, false otherwise.
      */
     public boolean isIngestRunning() {
-        return IngestScheduler.getInstance().ingestJobsAreRunning();
+        return IngestJob.ingestJobsAreRunning();
     }
 
-    
     /**
      * Called each time a module in a data source pipeline starts
+     *
      * @param task
-     * @param ingestModuleDisplayName 
+     * @param ingestModuleDisplayName
      */
     void setIngestTaskProgress(DataSourceIngestTask task, String ingestModuleDisplayName) {
         ingestThreadActivitySnapshots.put(task.getThreadId(), new IngestThreadActivitySnapshot(task.getThreadId(), task.getIngestJob().getId(), ingestModuleDisplayName, task.getDataSource()));
@@ -212,20 +212,22 @@ public class IngestManager {
 
     /**
      * Called each time a module in a file ingest pipeline starts
+     *
      * @param task
-     * @param ingestModuleDisplayName 
+     * @param ingestModuleDisplayName
      */
     void setIngestTaskProgress(FileIngestTask task, String ingestModuleDisplayName) {
         IngestThreadActivitySnapshot prevSnap = ingestThreadActivitySnapshots.get(task.getThreadId());
         IngestThreadActivitySnapshot newSnap = new IngestThreadActivitySnapshot(task.getThreadId(), task.getIngestJob().getId(), ingestModuleDisplayName, task.getDataSource(), task.getFile());
         ingestThreadActivitySnapshots.put(task.getThreadId(), newSnap);
-        
+
         incrementModuleRunTime(prevSnap.getActivity(), newSnap.getStartTime().getTime() - prevSnap.getStartTime().getTime());
     }
 
     /**
      * Called each time a data source ingest task completes
-     * @param task 
+     *
+     * @param task
      */
     void setIngestTaskProgressCompleted(DataSourceIngestTask task) {
         ingestThreadActivitySnapshots.put(task.getThreadId(), new IngestThreadActivitySnapshot(task.getThreadId()));
@@ -233,7 +235,8 @@ public class IngestManager {
 
     /**
      * Called when a file ingest pipeline is complete for a given file
-     * @param task 
+     *
+     * @param task
      */
     void setIngestTaskProgressCompleted(FileIngestTask task) {
         IngestThreadActivitySnapshot prevSnap = ingestThreadActivitySnapshots.get(task.getThreadId());
@@ -242,19 +245,21 @@ public class IngestManager {
         synchronized (processedFilesSnapshotLock) {
             processedFilesSnapshot.incrementProcessedFilesCount();
         }
-        
+
         incrementModuleRunTime(prevSnap.getActivity(), newSnap.getStartTime().getTime() - prevSnap.getStartTime().getTime());
     }
-    
+
     /**
-     * Internal method to update the times associated with each module. 
+     * Internal method to update the times associated with each module.
+     *
      * @param moduleName
-     * @param duration 
+     * @param duration
      */
     private void incrementModuleRunTime(String moduleName, Long duration) {
-        if (moduleName.equals("IDLE"))
+        if (moduleName.equals("IDLE")) {
             return;
-        
+        }
+
         synchronized (ingestModuleRunTimes) {
             Long prevTimeL = ingestModuleRunTimes.get(moduleName);
             long prevTime = 0;
@@ -262,12 +267,13 @@ public class IngestManager {
                 prevTime = prevTimeL;
             }
             prevTime += duration;
-            ingestModuleRunTimes.put(moduleName, prevTime);    
+            ingestModuleRunTimes.put(moduleName, prevTime);
         }
     }
-    
+
     /**
      * Return the list of run times for each module
+     *
      * @return Map of module name to run time (in milliseconds)
      */
     Map<String, Long> getModuleRunTimes() {
@@ -279,13 +285,13 @@ public class IngestManager {
 
     /**
      * Get the stats on current state of each thread
-     * @return 
+     *
+     * @return
      */
     List<IngestThreadActivitySnapshot> getIngestThreadActivitySnapshots() {
         return new ArrayList<>(ingestThreadActivitySnapshots.values());
     }
 
-
     public void cancelAllIngestJobs() {
         // Stop creating new ingest jobs.
         for (Future<Void> handle : startIngestJobsTasks.values()) {
@@ -293,7 +299,7 @@ public class IngestManager {
         }
 
         // Cancel all the jobs already created.
-        IngestScheduler.getInstance().cancelAllIngestJobs();
+        IngestJob.cancelAllJobs();
     }
 
     /**
@@ -418,7 +424,7 @@ public class IngestManager {
      * @param ingestJobId The ingest job id.
      */
     void fireIngestJobStarted(long ingestJobId) {
-        fireIngestEventsThreadPool.submit(new FireIngestEventTask(ingestJobEventPublisher, IngestJobEvent.STARTED, ingestJobId, null));
+        fireIngestEventsThreadPool.submit(new FireIngestEventRunnable(ingestJobEventPublisher, IngestJobEvent.STARTED, ingestJobId, null));
     }
 
     /**
@@ -427,7 +433,7 @@ public class IngestManager {
      * @param ingestJobId The ingest job id.
      */
     void fireIngestJobCompleted(long ingestJobId) {
-        fireIngestEventsThreadPool.submit(new FireIngestEventTask(ingestJobEventPublisher, IngestJobEvent.COMPLETED, ingestJobId, null));
+        fireIngestEventsThreadPool.submit(new FireIngestEventRunnable(ingestJobEventPublisher, IngestJobEvent.COMPLETED, ingestJobId, null));
     }
 
     /**
@@ -436,7 +442,7 @@ public class IngestManager {
      * @param ingestJobId The ingest job id.
      */
     void fireIngestJobCancelled(long ingestJobId) {
-        fireIngestEventsThreadPool.submit(new FireIngestEventTask(ingestJobEventPublisher, IngestJobEvent.CANCELLED, ingestJobId, null));
+        fireIngestEventsThreadPool.submit(new FireIngestEventRunnable(ingestJobEventPublisher, IngestJobEvent.CANCELLED, ingestJobId, null));
     }
 
     /**
@@ -445,7 +451,7 @@ public class IngestManager {
      * @param file The file that is completed.
      */
     void fireFileIngestDone(AbstractFile file) {
-        fireIngestEventsThreadPool.submit(new FireIngestEventTask(ingestModuleEventPublisher, IngestModuleEvent.FILE_DONE, file.getId(), file));
+        fireIngestEventsThreadPool.submit(new FireIngestEventRunnable(ingestModuleEventPublisher, IngestModuleEvent.FILE_DONE, file.getId(), file));
     }
 
     /**
@@ -454,7 +460,7 @@ public class IngestManager {
      * @param moduleDataEvent A ModuleDataEvent with the details of the posting.
      */
     void fireIngestModuleDataEvent(ModuleDataEvent moduleDataEvent) {
-        fireIngestEventsThreadPool.submit(new FireIngestEventTask(ingestModuleEventPublisher, IngestModuleEvent.DATA_ADDED, moduleDataEvent, null));
+        fireIngestEventsThreadPool.submit(new FireIngestEventRunnable(ingestModuleEventPublisher, IngestModuleEvent.DATA_ADDED, moduleDataEvent, null));
     }
 
     /**
@@ -465,7 +471,7 @@ public class IngestManager {
      * content.
      */
     void fireIngestModuleContentEvent(ModuleContentEvent moduleContentEvent) {
-        fireIngestEventsThreadPool.submit(new FireIngestEventTask(ingestModuleEventPublisher, IngestModuleEvent.CONTENT_CHANGED, moduleContentEvent, null));
+        fireIngestEventsThreadPool.submit(new FireIngestEventRunnable(ingestModuleEventPublisher, IngestModuleEvent.CONTENT_CHANGED, moduleContentEvent, null));
     }
 
     /**
@@ -509,7 +515,7 @@ public class IngestManager {
     /**
      * Creates ingest jobs.
      */
-    private class StartIngestJobsTask implements Callable<Void> {
+    private final class StartIngestJobsCallable implements Callable<Void> {
 
         private final long threadId;
         private final List<Content> dataSources;
@@ -517,7 +523,7 @@ public class IngestManager {
         private final boolean processUnallocatedSpace;
         private ProgressHandle progress;
 
-        StartIngestJobsTask(long threadId, List<Content> dataSources, List<IngestModuleTemplate> moduleTemplates, boolean processUnallocatedSpace) {
+        StartIngestJobsCallable(long threadId, List<Content> dataSources, List<IngestModuleTemplate> moduleTemplates, boolean processUnallocatedSpace) {
             this.threadId = threadId;
             this.dataSources = dataSources;
             this.moduleTemplates = moduleTemplates;
@@ -555,7 +561,7 @@ public class IngestManager {
                     }
 
                     // Start an ingest job for the data source.
-                    List<IngestModuleError> errors = IngestScheduler.getInstance().startIngestJob(dataSource, moduleTemplates, processUnallocatedSpace);
+                    List<IngestModuleError> errors = IngestJob.startJob(dataSource, moduleTemplates, processUnallocatedSpace);
                     if (!errors.isEmpty()) {
                         // Report the errors to the user. They have already been logged.
                         StringBuilder moduleStartUpErrors = new StringBuilder();
@@ -587,9 +593,6 @@ public class IngestManager {
                         break;
                     }
                 }
-            } catch (InterruptedException ex) {
-                // Reset interrupted status.
-                Thread.currentThread().interrupt();
             } catch (Exception ex) {
                 logger.log(Level.SEVERE, "Failed to create ingest job", ex); //NON-NLS
             } finally {
@@ -603,12 +606,12 @@ public class IngestManager {
     /**
      * A consumer for an ingest task queue.
      */
-    private class ExecuteIngestTasksTask implements Runnable {
+    private final class ExecuteIngestTasksRunnable implements Runnable {
 
         private final long threadId;
         private final IngestTaskQueue tasks;
 
-        ExecuteIngestTasksTask(long threadId, IngestTaskQueue tasks) {
+        ExecuteIngestTasksRunnable(long threadId, IngestTaskQueue tasks) {
             this.threadId = threadId;
             this.tasks = tasks;
         }
@@ -632,7 +635,7 @@ public class IngestManager {
     /**
      * Fires ingest events to ingest manager property change listeners.
      */
-    private static class FireIngestEventTask implements Runnable {
+    private static final class FireIngestEventRunnable implements Runnable {
 
         private final PropertyChangeSupport publisher;
         private final IngestJobEvent jobEvent;
@@ -640,7 +643,7 @@ public class IngestManager {
         private final Object oldValue;
         private final Object newValue;
 
-        FireIngestEventTask(PropertyChangeSupport publisher, IngestJobEvent event, Object oldValue, Object newValue) {
+        FireIngestEventRunnable(PropertyChangeSupport publisher, IngestJobEvent event, Object oldValue, Object newValue) {
             this.publisher = publisher;
             this.jobEvent = event;
             this.moduleEvent = null;
@@ -648,7 +651,7 @@ public class IngestManager {
             this.newValue = newValue;
         }
 
-        FireIngestEventTask(PropertyChangeSupport publisher, IngestModuleEvent event, Object oldValue, Object newValue) {
+        FireIngestEventRunnable(PropertyChangeSupport publisher, IngestModuleEvent event, Object oldValue, Object newValue) {
             this.publisher = publisher;
             this.jobEvent = null;
             this.moduleEvent = event;
@@ -695,9 +698,9 @@ public class IngestManager {
             startTime = new Date();
             this.activity = activity;
             this.dataSourceName = dataSource.getName();
-            this.fileName = "";    
+            this.fileName = "";
         }
-        
+
         // file ingest thread
         IngestThreadActivitySnapshot(long threadId, long jobId, String activity, Content dataSource, AbstractFile file) {
             this.threadId = threadId;
@@ -711,7 +714,7 @@ public class IngestManager {
         long getJobId() {
             return jobId;
         }
-        
+
         long getThreadId() {
             return threadId;
         }

Core/src/org/sleuthkit/autopsy/ingest/IngestModuleFactoryLoader.java

@@ -38,6 +38,7 @@ import org.sleuthkit.autopsy.modules.fileextmismatch.FileExtMismatchDetectorModu
 import org.sleuthkit.autopsy.modules.filetypeid.FileTypeIdModuleFactory;
 import org.sleuthkit.autopsy.modules.hashdatabase.HashLookupModuleFactory;
 import org.sleuthkit.autopsy.modules.interestingitems.InterestingItemsIngestModuleFactory;
+import org.sleuthkit.autopsy.modules.photoreccarver.PhotoRecCarverIngestModuleFactory;
 import org.sleuthkit.autopsy.modules.sevenzip.ArchiveFileExtractorModuleFactory;
 import org.sleuthkit.autopsy.python.JythonModuleLoader;
 
@@ -64,6 +65,7 @@ final class IngestModuleFactoryLoader {
             add(E01VerifierModuleFactory.class.getCanonicalName());
             add(AndroidModuleFactory.class.getCanonicalName());
             add(InterestingItemsIngestModuleFactory.class.getCanonicalName());
+            add(PhotoRecCarverIngestModuleFactory.class.getCanonicalName());
         }
     };
 

Core/src/org/sleuthkit/autopsy/ingest/IngestPipelinesConfiguration.java

@@ -18,13 +18,13 @@
  */
 package org.sleuthkit.autopsy.ingest;
 
-import java.io.File;
 import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.logging.Level;
 import org.sleuthkit.autopsy.coreutils.Logger;
-import org.sleuthkit.autopsy.coreutils.ModuleSettings;
 import org.sleuthkit.autopsy.coreutils.PlatformUtil;
 import org.sleuthkit.autopsy.coreutils.XMLUtil;
 import org.w3c.dom.Document;
@@ -33,32 +33,31 @@ import org.w3c.dom.NodeList;
 
 /**
  * Provides data source and file ingest pipeline configurations as ordered lists
- * of ingest module class names. The order of the module class names indicates
- * the desired sequence of ingest module instances in an ingest modules
- * pipeline.
+ * of ingest module factory class names.
  */
 final class IngestPipelinesConfiguration {
 
     private static final Logger logger = Logger.getLogger(IngestPipelinesConfiguration.class.getName());
-    private static final String PIPELINE_CONFIG_FILE_VERSION_KEY = "PipelineConfigFileVersion"; //NON-NLS
-    private static final String PIPELINE_CONFIG_FILE_VERSION_NO_STRING = "1";
-    private static final int PIPELINE_CONFIG_FILE_VERSION_NO = 1;
-    private static final String PIPELINES_CONFIG_FILE = "pipeline_config.xml"; //NON-NLS
-    private static final String PIPELINES_CONFIG_FILE_XSD = "PipelineConfigSchema.xsd"; //NON-NLS
-    private static final String XML_PIPELINE_ELEM = "PIPELINE"; //NON-NLS
-    private static final String XML_PIPELINE_TYPE_ATTR = "type"; //NON-NLS
-    private static final String DATA_SOURCE_INGEST_PIPELINE_TYPE = "ImageAnalysis"; //NON-NLS
-    private static final String FILE_INGEST_PIPELINE_TYPE = "FileAnalysis"; //NON-NLS
-    private static final String XML_MODULE_ELEM = "MODULE"; //NON-NLS
-    private static final String XML_MODULE_CLASS_NAME_ATTR = "location"; //NON-NLS
+    private static final String PIPELINES_CONFIG_FILE = "PipelineConfig.xml"; //NON-NLS
+    private static final String PIPELINE_ELEM = "PIPELINE"; //NON-NLS
+    private static final int NUMBER_OF_PIPELINE_DEFINITIONS = 3;
+    private static final String PIPELINE_TYPE_ATTR = "type"; //NON-NLS
+    private static final String STAGE_ONE_DATA_SOURCE_INGEST_PIPELINE_ELEM = "ImageAnalysisStageOne"; //NON-NLS
+    private static final String STAGE_TWO_DATA_SOURCE_INGEST_PIPELINE_ELEM = "ImageAnalysisStageTwo"; //NON-NLS
+    private static final String FILE_INGEST_PIPELINE_ELEM = "FileAnalysis"; //NON-NLS
+    private static final String INGEST_MODULE_ELEM = "MODULE"; //NON-NLS
+
     private static IngestPipelinesConfiguration instance;
-    private final List<String> dataSourceIngestPipelineConfig = new ArrayList<>();
+
+    private final List<String> stageOneDataSourceIngestPipelineConfig = new ArrayList<>();
     private final List<String> fileIngestPipelineConfig = new ArrayList<>();
+    private final List<String> stageTwoDataSourceIngestPipelineConfig = new ArrayList<>();
 
-    private IngestPipelinesConfiguration() {
-        readPipelinesConfigurationFile();
-    }
-
+    /**
+     * Gets the ingest pipelines configuration singleton.
+     *
+     * @return The singleton.
+     */
     synchronized static IngestPipelinesConfiguration getInstance() {
         if (instance == null) {
             Logger.getLogger(IngestPipelinesConfiguration.class.getName()).log(Level.INFO, "Creating ingest module loader instance"); //NON-NLS
@@ -67,57 +66,89 @@ final class IngestPipelinesConfiguration {
         return instance;
     }
 
-    List<String> getDataSourceIngestPipelineConfig() {
-        return new ArrayList<>(dataSourceIngestPipelineConfig);
+    /**
+     * Constructs an object that provides data source and file ingest pipeline
+     * configurations as ordered lists of ingest module factory class names.
+     */
+    private IngestPipelinesConfiguration() {
+        this.readPipelinesConfigurationFile();
     }
 
+    /**
+     * Gets the ordered list of ingest module factory class names for the
+     * file ingest pipeline.
+     *
+     * @return An ordered list of ingest module factory class names.
+     */
+    List<String> getStageOneDataSourceIngestPipelineConfig() {
+        return new ArrayList<>(stageOneDataSourceIngestPipelineConfig);
+    }
+
+    /**
+     * Gets the ordered list of ingest module factory class names for the
+     * first stage data source ingest pipeline.
+     *
+     * @return An ordered list of ingest module factory class names.
+     */
     List<String> getFileIngestPipelineConfig() {
         return new ArrayList<>(fileIngestPipelineConfig);
     }
 
+    /**
+     * Gets the ordered list of ingest module factory class names for the
+     * second stage data source ingest pipeline.
+     *
+     * @return An ordered list of ingest module factory class names.
+     */
+    List<String> getStageTwoDataSourceIngestPipelineConfig() {
+        return new ArrayList<>(stageTwoDataSourceIngestPipelineConfig);
+    }
+
+    /**
+     * Attempts to read the ingest pipeline configuration data from an XML file.
+     */
     private void readPipelinesConfigurationFile() {
         try {
-            boolean overWrite;
-            if (!ModuleSettings.settingExists(this.getClass().getSimpleName(), PIPELINE_CONFIG_FILE_VERSION_KEY)) {
-                ModuleSettings.setConfigSetting(this.getClass().getSimpleName(), PIPELINE_CONFIG_FILE_VERSION_KEY, PIPELINE_CONFIG_FILE_VERSION_NO_STRING);
-                overWrite = true;
-            } else {
-                int versionNumber = Integer.parseInt(ModuleSettings.getConfigSetting(this.getClass().getSimpleName(), PIPELINE_CONFIG_FILE_VERSION_KEY));
-                overWrite = versionNumber < PIPELINE_CONFIG_FILE_VERSION_NO;
-                // TODO: Migrate user edits
-            }
-            PlatformUtil.extractResourceToUserConfigDir(IngestPipelinesConfiguration.class, PIPELINES_CONFIG_FILE, overWrite);
+            PlatformUtil.extractResourceToUserConfigDir(IngestPipelinesConfiguration.class, PIPELINES_CONFIG_FILE, false);
 
-            String configFilePath = PlatformUtil.getUserConfigDirectory() + File.separator + PIPELINES_CONFIG_FILE;
-            Document doc = XMLUtil.loadDoc(IngestPipelinesConfiguration.class, configFilePath);
+            Path configFilePath = Paths.get(PlatformUtil.getUserConfigDirectory(), PIPELINES_CONFIG_FILE);
+            Document doc = XMLUtil.loadDoc(IngestPipelinesConfiguration.class, configFilePath.toAbsolutePath().toString());
             if (doc == null) {
                 return;
             }
 
+            // Get the document root element.
             Element rootElement = doc.getDocumentElement();
-            if (rootElement == null) {
+            if (null == rootElement) {
                 logger.log(Level.SEVERE, "Invalid pipelines config file"); //NON-NLS
                 return;
             }
 
-            NodeList pipelineElements = rootElement.getElementsByTagName(XML_PIPELINE_ELEM);
+            // Get the pipeline elements and confirm that the correct number is
+            // present.
+            NodeList pipelineElements = rootElement.getElementsByTagName(IngestPipelinesConfiguration.PIPELINE_ELEM);
             int numPipelines = pipelineElements.getLength();
-            if (numPipelines < 1 || numPipelines > 2) {
+            if (numPipelines != IngestPipelinesConfiguration.NUMBER_OF_PIPELINE_DEFINITIONS) {
                 logger.log(Level.SEVERE, "Invalid pipelines config file"); //NON-NLS
                 return;
             }
 
+            // Parse the pipeline elements to populate the pipeline 
+            // configuration lists.
             List<String> pipelineConfig = null;
             for (int pipelineNum = 0; pipelineNum < numPipelines; ++pipelineNum) {
                 Element pipelineElement = (Element) pipelineElements.item(pipelineNum);
-                String pipelineTypeAttr = pipelineElement.getAttribute(XML_PIPELINE_TYPE_ATTR);
-                if (pipelineTypeAttr != null) {
+                String pipelineTypeAttr = pipelineElement.getAttribute(PIPELINE_TYPE_ATTR);
+                if (null != pipelineTypeAttr) {
                     switch (pipelineTypeAttr) {
-                        case DATA_SOURCE_INGEST_PIPELINE_TYPE:
-                            pipelineConfig = dataSourceIngestPipelineConfig;
+                        case STAGE_ONE_DATA_SOURCE_INGEST_PIPELINE_ELEM:
+                            pipelineConfig = this.stageOneDataSourceIngestPipelineConfig;
                             break;
-                        case FILE_INGEST_PIPELINE_TYPE:
-                            pipelineConfig = fileIngestPipelineConfig;
+                        case FILE_INGEST_PIPELINE_ELEM:
+                            pipelineConfig = this.fileIngestPipelineConfig;
+                            break;
+                        case STAGE_TWO_DATA_SOURCE_INGEST_PIPELINE_ELEM:
+                            pipelineConfig = this.stageTwoDataSourceIngestPipelineConfig;
                             break;
                         default:
                             logger.log(Level.SEVERE, "Invalid pipelines config file"); //NON-NLS
@@ -128,16 +159,13 @@ final class IngestPipelinesConfiguration {
                 // Create an ordered list of class names. The sequence of class 
                 // names defines the sequence of modules in the pipeline.
                 if (pipelineConfig != null) {
-                    NodeList modulesElems = pipelineElement.getElementsByTagName(XML_MODULE_ELEM);
+                    NodeList modulesElems = pipelineElement.getElementsByTagName(INGEST_MODULE_ELEM);
                     int numModules = modulesElems.getLength();
-                    if (numModules == 0) {
-                        break;
-                    }
                     for (int moduleNum = 0; moduleNum < numModules; ++moduleNum) {
                         Element moduleElement = (Element) modulesElems.item(moduleNum);
-                        final String moduleClassName = moduleElement.getAttribute(XML_MODULE_CLASS_NAME_ATTR);
-                        if (moduleClassName != null) {
-                            pipelineConfig.add(moduleClassName);
+                        String className = moduleElement.getTextContent();
+                        if (null != className && !className.isEmpty()) {
+                            pipelineConfig.add(className);
                         }
                     }
                 }

Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java

@@ -29,7 +29,6 @@ import javax.swing.table.AbstractTableModel;
 import javax.swing.table.TableColumn;
 import org.apache.commons.lang3.time.DurationFormatUtils;
 import org.openide.util.NbBundle;
-import org.sleuthkit.autopsy.ingest.IngestScheduler.IngestJobSchedulerStats;
 
 public class IngestProgressSnapshotPanel extends javax.swing.JPanel {
 
@@ -161,20 +160,20 @@ public class IngestProgressSnapshotPanel extends javax.swing.JPanel {
 
         private final String[] columnNames = {"Job ID", 
             "Data Source", "Start", "Num Processed", "Files/Sec", "In Progress", "Files Queued", "Dir Queued", "Root Queued", "DS Queued"};
-        private List<IngestJobSchedulerStats> schedStats;
+        private List<IngestJob.IngestJobSnapshot> jobSnapshots;
 
         private IngestJobTableModel() {
             refresh();
         }
 
         private void refresh() {
-            schedStats = IngestScheduler.getInstance().getJobStats();
+            jobSnapshots = IngestJob.getJobSnapshots();
             fireTableDataChanged();
         }
 
         @Override
         public int getRowCount() {
-            return schedStats.size();
+            return jobSnapshots.size();
         }
 
         @Override
@@ -189,39 +188,39 @@ public class IngestProgressSnapshotPanel extends javax.swing.JPanel {
 
         @Override
         public Object getValueAt(int rowIndex, int columnIndex) {
-            IngestJobSchedulerStats schedStat = schedStats.get(rowIndex);
+            IngestJob.IngestJobSnapshot snapShot = jobSnapshots.get(rowIndex);
             Object cellValue;
             switch (columnIndex) {
                 case 0:
-                    cellValue = schedStat.getJobId();
+                    cellValue = snapShot.getJobId();
                     break;
                 case 1:
-                    cellValue = schedStat.getDataSource();
+                    cellValue = snapShot.getDataSource();
                     break;
                 case 2:
                     SimpleDateFormat dateFormat = new SimpleDateFormat("HH:mm:ss");
-                    cellValue = dateFormat.format(new Date(schedStat.getIngestJobStats().getStartTime()));
+                    cellValue = dateFormat.format(new Date(snapShot.getStartTime()));
                     break;
                 case 3:
-                    cellValue = schedStat.getIngestJobStats().getFilesProcessed();
+                    cellValue = snapShot.getFilesProcessed();
                     break;
                 case 4:
-                    cellValue = schedStat.getIngestJobStats().getSpeed();
+                    cellValue = snapShot.getSpeed();
                     break;
                 case 5:
-                    cellValue = schedStat.getRunningListSize();
+                    cellValue = snapShot.getRunningListSize();
                     break;
                 case 6:
-                    cellValue = schedStat.getFileQueueSize();
+                    cellValue = snapShot.getFileQueueSize();
                     break;
                 case 7:
-                    cellValue = schedStat.getDirQueueSize();
+                    cellValue = snapShot.getDirQueueSize();
                     break;
                 case 8:
-                    cellValue = schedStat.getRootQueueSize();
+                    cellValue = snapShot.getRootQueueSize();
                     break;
                 case 9:
-                    cellValue = schedStat.getDsQueueSize();
+                    cellValue = snapShot.getDsQueueSize();
                     break;
                 default:
                     cellValue = null;

Core/src/org/sleuthkit/autopsy/ingest/IngestScheduler.java

@@ -1,678 +0,0 @@
-/*
- * Autopsy Forensic Browser
- * 
- * Copyright 2012-2014 Basis Technology Corp.
- * Contact: carrier <at> sleuthkit <dot> org
- * 
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.sleuthkit.autopsy.ingest;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.Iterator;
-import java.util.List;
-import java.util.TreeSet;
-import java.util.concurrent.BlockingDeque;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.LinkedBlockingDeque;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.logging.Level;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-import org.sleuthkit.autopsy.coreutils.Logger;
-import org.sleuthkit.autopsy.ingest.IngestJob.IngestJobStats;
-import org.sleuthkit.datamodel.AbstractFile;
-import org.sleuthkit.datamodel.Content;
-import org.sleuthkit.datamodel.File;
-import org.sleuthkit.datamodel.FileSystem;
-import org.sleuthkit.datamodel.TskCoreException;
-import org.sleuthkit.datamodel.TskData;
-
-/**
- * Creates ingest jobs and their constituent ingest tasks, queuing the tasks in
- * priority order for execution by the ingest manager's ingest threads.
- */
-final class IngestScheduler {
-
-    private static final Logger logger = Logger.getLogger(IngestScheduler.class.getName());
-
-    private static final int FAT_NTFS_FLAGS = TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT12.getValue() | TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT16.getValue() | TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT32.getValue() | TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_NTFS.getValue();
-
-    private static IngestScheduler instance = null;
-
-    private final AtomicLong nextIngestJobId = new AtomicLong(0L);
-
-    private final ConcurrentHashMap<Long, IngestJob> ingestJobsById = new ConcurrentHashMap<>();
-
-    private volatile boolean enabled = false;
-
-    private final DataSourceIngestTaskQueue dataSourceTaskDispenser = new DataSourceIngestTaskQueue();
-
-    private final FileIngestTaskQueue fileTaskDispenser = new FileIngestTaskQueue();
-
-    // The following five collections lie at the heart of the scheduler.
-    // The pending tasks queues are used to schedule tasks for an ingest job. If 
-    // multiple jobs are scheduled, tasks from different jobs may become 
-    // interleaved in these queues. 
-    // FIFO queue for data source-level tasks. 
-    private final LinkedBlockingQueue<DataSourceIngestTask> pendingDataSourceTasks = new LinkedBlockingQueue<>(); // Guarded by this
-
-    // File tasks are "shuffled" 
-    // through root directory (priority queue), directory (LIFO), and file tasks 
-    // queues (LIFO). If a file task makes it into the pending file tasks queue,
-    // it is consumed by the ingest threads. 
-    private final TreeSet<FileIngestTask> pendingRootDirectoryTasks = new TreeSet<>(new RootDirectoryTaskComparator()); // Guarded by this
-
-    private final List<FileIngestTask> pendingDirectoryTasks = new ArrayList<>();  // Guarded by this
-
-    private final BlockingDeque<FileIngestTask> pendingFileTasks = new LinkedBlockingDeque<>(); // Not guarded
-
-    // The "tasks in progress" list has:
-    // - File and data source tasks that are running
-    // - File tasks that are in the pending file queue
-    // It is used to determine when a job is done.  It has both pending and running
-    // tasks because we do not lock the 'pendingFileTasks' and a task needs to be in
-    // at least one of the pending or inprogress lists at all times before it is completed. 
-    // files are added to this when the are added to pendingFilesTasks and removed when they complete
-    private final List<IngestTask> tasksInProgressAndPending = new ArrayList<>();  // Guarded by this 
-
-    synchronized static IngestScheduler getInstance() {
-        if (instance == null) {
-            instance = new IngestScheduler();
-        }
-        return instance;
-    }
-
-    private IngestScheduler() {
-    }
-
-    void setEnabled(boolean enabled) {
-        this.enabled = enabled;
-    }
-
-    /**
-     * Creates an ingest job for a data source.
-     *
-     * @param dataSource The data source to ingest.
-     * @param ingestModuleTemplates The ingest module templates to use to create
-     * the ingest pipelines for the job.
-     * @param processUnallocatedSpace Whether or not the job should include
-     * processing of unallocated space.
-     *
-     * @return A collection of ingest module start up errors, empty on success.
-     *
-     * @throws InterruptedException
-     */
-    List<IngestModuleError> startIngestJob(Content dataSource, List<IngestModuleTemplate> ingestModuleTemplates, boolean processUnallocatedSpace) throws InterruptedException {
-        List<IngestModuleError> errors = new ArrayList<>();
-        if (enabled) {
-            long jobId = nextIngestJobId.incrementAndGet();
-            IngestJob job = new IngestJob(jobId, dataSource, processUnallocatedSpace);
-            errors = job.start(ingestModuleTemplates);
-            if (errors.isEmpty() && (job.hasDataSourceIngestPipeline() || job.hasFileIngestPipeline())) {
-                ingestJobsById.put(jobId, job);
-                IngestManager.getInstance().fireIngestJobStarted(jobId);
-                scheduleIngestTasks(job);
-                logger.log(Level.INFO, "Ingest job {0} started", jobId);
-            }
-        }
-        return errors;
-    }
-
-    synchronized private void scheduleIngestTasks(IngestJob job) throws InterruptedException {
-        // This is synchronized to guard the task queues and make ingest 
-        // scheduling for a job an an atomic operation. Otherwise, the data 
-        // source task might be completed before the file tasks were scheduled, 
-        // resulting in a false positive for a job completion check.
-        if (job.hasDataSourceIngestPipeline()) {
-            scheduleDataSourceIngestTask(job);
-        }
-        if (job.hasFileIngestPipeline()) {
-            scheduleFileIngestTasks(job);
-        }
-    }
-
-    synchronized private void scheduleDataSourceIngestTask(IngestJob job) throws InterruptedException {
-        DataSourceIngestTask task = new DataSourceIngestTask(job);
-        tasksInProgressAndPending.add(task);
-        try {
-            // Should not block, queue is (theoretically) unbounded.
-            pendingDataSourceTasks.put(task);
-        } catch (InterruptedException ex) {
-            tasksInProgressAndPending.remove(task);
-            Logger.getLogger(IngestScheduler.class.getName()).log(Level.SEVERE, "Interruption of unexpected block on pending data source tasks queue", ex); //NON-NLS
-            throw ex;
-        }
-    }
-
-    synchronized private void scheduleFileIngestTasks(IngestJob job) throws InterruptedException {
-        List<AbstractFile> topLevelFiles = getTopLevelFiles(job.getDataSource());
-        for (AbstractFile firstLevelFile : topLevelFiles) {
-            FileIngestTask task = new FileIngestTask(job, firstLevelFile);
-            if (shouldEnqueueFileTask(task)) {
-                pendingRootDirectoryTasks.add(task);
-            }
-        }
-        updatePendingFileTasksQueues();
-    }
-
-    private static List<AbstractFile> getTopLevelFiles(Content dataSource) {
-        List<AbstractFile> topLevelFiles = new ArrayList<>();
-        Collection<AbstractFile> rootObjects = dataSource.accept(new GetRootDirectoryVisitor());
-        if (rootObjects.isEmpty() && dataSource instanceof AbstractFile) {
-            // The data source is itself a file to be processed.
-            topLevelFiles.add((AbstractFile) dataSource);
-        } else {
-            for (AbstractFile root : rootObjects) {
-                List<Content> children;
-                try {
-                    children = root.getChildren();
-                    if (children.isEmpty()) {
-                        // Add the root object itself, it could be an unallocated
-                        // space file, or a child of a volume or an image.
-                        topLevelFiles.add(root);
-                    } else {
-                        // The root object is a file system root directory, get
-                        // the files within it.
-                        for (Content child : children) {
-                            if (child instanceof AbstractFile) {
-                                topLevelFiles.add((AbstractFile) child);
-                            }
-                        }
-                    }
-                } catch (TskCoreException ex) {
-                    logger.log(Level.WARNING, "Could not get children of root to enqueue: " + root.getId() + ": " + root.getName(), ex); //NON-NLS
-                }
-            }
-        }
-        return topLevelFiles;
-    }
-
-    synchronized private void updatePendingFileTasksQueues() throws InterruptedException {
-        // This is synchronized to guard the pending file tasks queues and make 
-        // this an atomic operation.
-        if (enabled) {
-            while (true) {
-                // Loop until either the pending file tasks queue is NOT empty
-                // or the upstream queues that feed into it ARE empty.
-                if (pendingFileTasks.isEmpty() == false) {
-                    return;
-                }
-                if (pendingDirectoryTasks.isEmpty()) {
-                    if (pendingRootDirectoryTasks.isEmpty()) {
-                        return;
-                    }
-                    pendingDirectoryTasks.add(pendingRootDirectoryTasks.pollFirst());
-                }
-
-                // Try to add the most recently added from the pending directory tasks queue to 
-                // the pending file tasks queue. 
-                boolean tasksEnqueuedForDirectory = false;
-                FileIngestTask directoryTask = pendingDirectoryTasks.remove(pendingDirectoryTasks.size() - 1);
-                if (shouldEnqueueFileTask(directoryTask)) {
-                    addToPendingFileTasksQueue(directoryTask);
-                    tasksEnqueuedForDirectory = true;
-                }
-
-                // If the directory contains subdirectories or files, try to 
-                // enqueue tasks for them as well. 
-                final AbstractFile directory = directoryTask.getFile();
-                try {
-                    for (Content child : directory.getChildren()) {
-                        if (child instanceof AbstractFile) {
-                            AbstractFile file = (AbstractFile) child;
-                            FileIngestTask childTask = new FileIngestTask(directoryTask.getIngestJob(), file);
-                            if (file.hasChildren()) {
-                                // Found a subdirectory, put the task in the 
-                                // pending directory tasks queue.
-                                pendingDirectoryTasks.add(childTask);
-                                tasksEnqueuedForDirectory = true;
-                            } else if (shouldEnqueueFileTask(childTask)) {
-                                // Found a file, put the task directly into the
-                                // pending file tasks queue.
-                                addToPendingFileTasksQueue(childTask);
-                                tasksEnqueuedForDirectory = true;
-                            }
-                        }
-                    }
-                } catch (TskCoreException ex) {
-                    String errorMessage = String.format("An error occurred getting the children of %s", directory.getName()); //NON-NLS
-                    logger.log(Level.SEVERE, errorMessage, ex);
-                }
-
-                // In the case where the directory task is not pushed into the
-                // the pending file tasks queue and has no children, check to 
-                // see if the job is completed - the directory task might have 
-                // been the last task for the job.                
-                if (!tasksEnqueuedForDirectory) {
-                    IngestJob job = directoryTask.getIngestJob();
-                    if (ingestJobIsComplete(job)) {
-                        finishIngestJob(job);
-                    }
-                }
-            }
-        }
-    }
-
-    private static boolean shouldEnqueueFileTask(final FileIngestTask processTask) {
-        final AbstractFile aFile = processTask.getFile();
-        //if it's unalloc file, skip if so scheduled
-        if (processTask.getIngestJob().shouldProcessUnallocatedSpace() == false && aFile.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS)) {
-            return false;
-        }
-        String fileName = aFile.getName();
-        if (fileName.equals(".") || fileName.equals("..")) {
-            return false;
-        } else if (aFile instanceof org.sleuthkit.datamodel.File) {
-            final org.sleuthkit.datamodel.File f = (File) aFile;
-            //skip files in root dir, starting with $, containing : (not default attributes)
-            //with meta address < 32, i.e. some special large NTFS and FAT files
-            FileSystem fs = null;
-            try {
-                fs = f.getFileSystem();
-            } catch (TskCoreException ex) {
-                logger.log(Level.SEVERE, "Could not get FileSystem for " + f, ex); //NON-NLS
-            }
-            TskData.TSK_FS_TYPE_ENUM fsType = TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_UNSUPP;
-            if (fs != null) {
-                fsType = fs.getFsType();
-            }
-            if ((fsType.getValue() & FAT_NTFS_FLAGS) == 0) {
-                //not fat or ntfs, accept all files
-                return true;
-            }
-            boolean isInRootDir = false;
-            try {
-                isInRootDir = f.getParentDirectory().isRoot();
-            } catch (TskCoreException ex) {
-                logger.log(Level.WARNING, "Could not check if should enqueue the file: " + f.getName(), ex); //NON-NLS
-            }
-            if (isInRootDir && f.getMetaAddr() < 32) {
-                String name = f.getName();
-                if (name.length() > 0 && name.charAt(0) == '$' && name.contains(":")) {
-                    return false;
-                }
-            } else {
-                return true;
-            }
-        }
-        return true;
-    }
-
-    synchronized private void addToPendingFileTasksQueue(FileIngestTask task) throws IllegalStateException {
-        tasksInProgressAndPending.add(task);
-        try {
-            // Should not block, queue is (theoretically) unbounded.
-            /* add to top of list because we had one image that had a folder
-             * with
-             * lots of zip files. This queue had thousands of entries because
-             * it just kept on getting bigger and bigger. So focus on pushing
-             * out
-             * the ZIP file contents out of the queue to try to keep it small.
-             */
-            pendingFileTasks.addFirst(task);
-        } catch (IllegalStateException ex) {
-            tasksInProgressAndPending.remove(task);
-            Logger.getLogger(IngestScheduler.class.getName()).log(Level.SEVERE, "Interruption of unexpected block on pending file tasks queue", ex); //NON-NLS
-            throw ex;
-        }
-    }
-
-    void scheduleAdditionalFileIngestTask(IngestJob job, AbstractFile file) throws InterruptedException {
-        if (enabled) {
-            FileIngestTask task = new FileIngestTask(job, file);
-            if (shouldEnqueueFileTask(task)) {
-                // Send the file task directly to file tasks queue, no need to
-                // update the pending root directory or pending directory tasks 
-                // queues.
-                addToPendingFileTasksQueue(task);
-            }
-        }
-    }
-
-    IngestTaskQueue getDataSourceIngestTaskQueue() {
-        return dataSourceTaskDispenser;
-    }
-
-    IngestTaskQueue getFileIngestTaskQueue() {
-        return fileTaskDispenser;
-    }
-
-    void notifyTaskCompleted(IngestTask task) {
-        boolean jobIsCompleted;
-        IngestJob job = task.getIngestJob();
-        synchronized (this) {
-            tasksInProgressAndPending.remove(task);
-            jobIsCompleted = ingestJobIsComplete(job);
-        }
-        if (jobIsCompleted) {
-            // The lock does not need to be held for the job shut down.
-            finishIngestJob(job);
-        }
-    }
-
-    /**
-     * Queries whether or not ingest jobs are running.
-     *
-     * @return True or false.
-     */
-    boolean ingestJobsAreRunning() {
-        return !ingestJobsById.isEmpty();
-    }
-
-    /**
-     * Clears the pending ingest task queues for an ingest job. If job is
-     * complete (no pending or in progress tasks) the job is finished up.
-     * Otherwise, the last worker thread with an in progress task will finish /
-     * clean up the job.
-     *
-     * @param job The job to cancel.
-     */
-    synchronized void cancelPendingTasksForIngestJob(IngestJob job) {
-        long jobId = job.getId();
-        removeAllPendingTasksForJob(pendingRootDirectoryTasks, jobId);
-        removeAllPendingTasksForJob(pendingDirectoryTasks, jobId);
-        removeAllPendingTasksForJob(pendingFileTasks, jobId);
-        removeAllPendingTasksForJob(pendingDataSourceTasks, jobId);
-        if (ingestJobIsComplete(job)) {
-            finishIngestJob(job);
-        }
-    }
-
-    /**
-     * Return the number of tasks in the queue for the given job ID
-     *
-     * @param <T>
-     * @param queue
-     * @param jobId
-     *
-     * @return
-     */
-    <T> int countJobsInCollection(Collection<T> queue, long jobId) {
-        Iterator<T> iterator = queue.iterator();
-        int count = 0;
-        while (iterator.hasNext()) {
-            IngestTask task = (IngestTask) iterator.next();
-            if (task.getIngestJob().getId() == jobId) {
-                count++;
-            }
-        }
-        return count;
-    }
-
-    synchronized private void removeAllPendingTasksForJob(Collection<? extends IngestTask> taskQueue, long jobId) {
-        Iterator<? extends IngestTask> iterator = taskQueue.iterator();
-        while (iterator.hasNext()) {
-            IngestTask task = iterator.next();
-            if (task.getIngestJob().getId() == jobId) {
-                tasksInProgressAndPending.remove(task);
-                iterator.remove();
-            }
-        }
-    }
-
-    void cancelAllIngestJobs() {
-        synchronized (this) {
-            removeAllPendingTasks(pendingRootDirectoryTasks);
-            removeAllPendingTasks(pendingDirectoryTasks);
-            removeAllPendingTasks(pendingFileTasks);
-            removeAllPendingTasks(pendingDataSourceTasks);
-            for (IngestJob job : ingestJobsById.values()) {
-                job.cancel();
-                if (ingestJobIsComplete(job)) {
-                    finishIngestJob(job);
-                }
-            }
-        }
-    }
-
-    synchronized private <T> void removeAllPendingTasks(Collection<T> taskQueue) {
-        Iterator<T> iterator = taskQueue.iterator();
-        while (iterator.hasNext()) {
-            tasksInProgressAndPending.remove((IngestTask) iterator.next());
-            iterator.remove();
-        }
-    }
-
-    synchronized private boolean ingestJobIsComplete(IngestJob job) {
-        for (IngestTask task : tasksInProgressAndPending) {
-            if (task.getIngestJob().getId() == job.getId()) {
-                return false;
-            }
-        }
-        return true;
-    }
-
-    /**
-     * Called after all work is completed to free resources.
-     *
-     * @param job
-     */
-    private void finishIngestJob(IngestJob job) {
-        job.finish();
-        long jobId = job.getId();
-        ingestJobsById.remove(jobId);
-        if (!job.isCancelled()) {
-            logger.log(Level.INFO, "Ingest job {0} completed", jobId);
-            IngestManager.getInstance().fireIngestJobCompleted(job.getId());
-        } else {
-            logger.log(Level.INFO, "Ingest job {0} cancelled", jobId);
-            IngestManager.getInstance().fireIngestJobCancelled(job.getId());
-        }
-    }
-
-    private static class RootDirectoryTaskComparator implements Comparator<FileIngestTask> {
-
-        @Override
-        public int compare(FileIngestTask q1, FileIngestTask q2) {
-            AbstractFilePriority.Priority p1 = AbstractFilePriority.getPriority(q1.getFile());
-            AbstractFilePriority.Priority p2 = AbstractFilePriority.getPriority(q2.getFile());
-            if (p1 == p2) {
-                return (int) (q2.getFile().getId() - q1.getFile().getId());
-            } else {
-                return p2.ordinal() - p1.ordinal();
-            }
-        }
-
-        private static class AbstractFilePriority {
-
-            enum Priority {
-
-                LAST, LOW, MEDIUM, HIGH
-            }
-
-            static final List<Pattern> LAST_PRI_PATHS = new ArrayList<>();
-
-            static final List<Pattern> LOW_PRI_PATHS = new ArrayList<>();
-
-            static final List<Pattern> MEDIUM_PRI_PATHS = new ArrayList<>();
-
-            static final List<Pattern> HIGH_PRI_PATHS = new ArrayList<>();
-            /* prioritize root directory folders based on the assumption that we
-             * are
-             * looking for user content. Other types of investigations may want
-             * different
-             * priorities. */
-
-            static /* prioritize root directory
-             * folders based on the assumption that we are
-             * looking for user content. Other types of investigations may want
-             * different
-             * priorities. */ {
-                // these files have no structure, so they go last
-                //unalloc files are handled as virtual files in getPriority()
-                //LAST_PRI_PATHS.schedule(Pattern.compile("^\\$Unalloc", Pattern.CASE_INSENSITIVE));
-                //LAST_PRI_PATHS.schedule(Pattern.compile("^\\Unalloc", Pattern.CASE_INSENSITIVE));
-                LAST_PRI_PATHS.add(Pattern.compile("^pagefile", Pattern.CASE_INSENSITIVE));
-                LAST_PRI_PATHS.add(Pattern.compile("^hiberfil", Pattern.CASE_INSENSITIVE));
-                // orphan files are often corrupt and windows does not typically have
-                // user content, so put them towards the bottom
-                LOW_PRI_PATHS.add(Pattern.compile("^\\$OrphanFiles", Pattern.CASE_INSENSITIVE));
-                LOW_PRI_PATHS.add(Pattern.compile("^Windows", Pattern.CASE_INSENSITIVE));
-                // all other files go into the medium category too
-                MEDIUM_PRI_PATHS.add(Pattern.compile("^Program Files", Pattern.CASE_INSENSITIVE));
-                // user content is top priority
-                HIGH_PRI_PATHS.add(Pattern.compile("^Users", Pattern.CASE_INSENSITIVE));
-                HIGH_PRI_PATHS.add(Pattern.compile("^Documents and Settings", Pattern.CASE_INSENSITIVE));
-                HIGH_PRI_PATHS.add(Pattern.compile("^home", Pattern.CASE_INSENSITIVE));
-                HIGH_PRI_PATHS.add(Pattern.compile("^ProgramData", Pattern.CASE_INSENSITIVE));
-            }
-
-            /**
-             * Get the enabled priority for a given file.
-             *
-             * @param abstractFile
-             *
-             * @return
-             */
-            static AbstractFilePriority.Priority getPriority(final AbstractFile abstractFile) {
-                if (!abstractFile.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.FS)) {
-                    //quickly filter out unstructured content
-                    //non-fs virtual files and dirs, such as representing unalloc space
-                    return AbstractFilePriority.Priority.LAST;
-                }
-                //determine the fs files priority by name
-                final String path = abstractFile.getName();
-                if (path == null) {
-                    return AbstractFilePriority.Priority.MEDIUM;
-                }
-                for (Pattern p : HIGH_PRI_PATHS) {
-                    Matcher m = p.matcher(path);
-                    if (m.find()) {
-                        return AbstractFilePriority.Priority.HIGH;
-                    }
-                }
-                for (Pattern p : MEDIUM_PRI_PATHS) {
-                    Matcher m = p.matcher(path);
-                    if (m.find()) {
-                        return AbstractFilePriority.Priority.MEDIUM;
-                    }
-                }
-                for (Pattern p : LOW_PRI_PATHS) {
-                    Matcher m = p.matcher(path);
-                    if (m.find()) {
-                        return AbstractFilePriority.Priority.LOW;
-                    }
-                }
-                for (Pattern p : LAST_PRI_PATHS) {
-                    Matcher m = p.matcher(path);
-                    if (m.find()) {
-                        return AbstractFilePriority.Priority.LAST;
-                    }
-                }
-                //default is medium
-                return AbstractFilePriority.Priority.MEDIUM;
-            }
-        }
-    }
-
-    private final class DataSourceIngestTaskQueue implements IngestTaskQueue {
-
-        @Override
-        public IngestTask getNextTask() throws InterruptedException {
-            return pendingDataSourceTasks.take();
-        }
-    }
-
-    private final class FileIngestTaskQueue implements IngestTaskQueue {
-
-        @Override
-        public IngestTask getNextTask() throws InterruptedException {
-            FileIngestTask task = pendingFileTasks.takeFirst();
-            updatePendingFileTasksQueues();
-            return task;
-        }
-    }
-
-    /**
-     * Stores basic stats for a given job
-     */
-    class IngestJobSchedulerStats {
-
-        private final IngestJobStats ingestJobStats;
-
-        private final long jobId;
-
-        private final String dataSource;
-
-        private final long rootQueueSize;
-
-        private final long dirQueueSize;
-
-        private final long fileQueueSize;
-
-        private final long dsQueueSize;
-
-        private final long runningListSize;
-
-        IngestJobSchedulerStats(IngestJob job) {
-            ingestJobStats = job.getStats();
-            jobId = job.getId();
-            dataSource = job.getDataSource().getName();
-            rootQueueSize = countJobsInCollection(pendingRootDirectoryTasks, jobId);
-            dirQueueSize = countJobsInCollection(pendingDirectoryTasks, jobId);
-            fileQueueSize = countJobsInCollection(pendingFileTasks, jobId);
-            dsQueueSize = countJobsInCollection(pendingDataSourceTasks, jobId);
-            runningListSize = countJobsInCollection(tasksInProgressAndPending, jobId) - fileQueueSize - dsQueueSize;
-        }
-
-        protected long getJobId() {
-            return jobId;
-        }
-
-        protected String getDataSource() {
-            return dataSource;
-        }
-
-        protected long getRootQueueSize() {
-            return rootQueueSize;
-        }
-
-        protected long getDirQueueSize() {
-            return dirQueueSize;
-        }
-
-        protected long getFileQueueSize() {
-            return fileQueueSize;
-        }
-
-        protected long getDsQueueSize() {
-            return dsQueueSize;
-        }
-
-        protected long getRunningListSize() {
-            return runningListSize;
-        }
-
-        protected IngestJobStats getIngestJobStats() {
-            return ingestJobStats;
-        }
-    }
-
-    /**
-     * Get basic performance / stats on all running jobs
-     *
-     * @return
-     */
-    synchronized List<IngestJobSchedulerStats> getJobStats() {
-        List<IngestJobSchedulerStats> stats = new ArrayList<>();
-        for (IngestJob job : Collections.list(ingestJobsById.elements())) {
-            stats.add(new IngestJobSchedulerStats(job));
-        }
-        return stats;
-    }
-}

Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java

@@ -0,0 +1,717 @@
+/*
+ * Autopsy Forensic Browser
+ * 
+ * Copyright 2012-2014 Basis Technology Corp.
+ * Contact: carrier <at> sleuthkit <dot> org
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.sleuthkit.autopsy.ingest;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.List;
+import java.util.TreeSet;
+import java.util.concurrent.BlockingDeque;
+import java.util.concurrent.LinkedBlockingDeque;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.logging.Level;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import org.sleuthkit.autopsy.coreutils.Logger;
+import org.sleuthkit.datamodel.AbstractFile;
+import org.sleuthkit.datamodel.Content;
+import org.sleuthkit.datamodel.FileSystem;
+import org.sleuthkit.datamodel.TskCoreException;
+import org.sleuthkit.datamodel.TskData;
+
+/**
+ * Creates ingest tasks for ingest jobs, queuing the tasks in priority order for
+ * execution by the ingest manager's ingest threads.
+ */
+final class IngestTasksScheduler {
+
+    private static final Logger logger = Logger.getLogger(IngestTasksScheduler.class.getName());
+    private static final int FAT_NTFS_FLAGS = TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT12.getValue() | TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT16.getValue() | TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_FAT32.getValue() | TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_NTFS.getValue();
+    private static IngestTasksScheduler instance;
+
+    /**
+     * Scheduling of data source ingest tasks is accomplished by putting them in
+     * a FIFO queue to be consumed by the ingest threads, so the queue is
+     * wrapped in a "dispenser" that implements the IngestTaskQueue interface
+     * and is exposed via a getter method.
+     */
+    private final LinkedBlockingQueue<DataSourceIngestTask> pendingDataSourceTasks;
+    private final DataSourceIngestTaskQueue dataSourceTasksDispenser;
+
+    /**
+     * Scheduling of file ingest tasks is accomplished by "shuffling" them
+     * through a sequence of internal queues that allows for the interleaving of
+     * tasks from different ingest jobs based on priority. These scheduling
+     * queues are:
+     *
+     * 1. Root directory tasks (priority queue)
+     *
+     * 2. Directory tasks (FIFO queue)
+     *
+     * 3. Pending file tasks (LIFO queue).
+     *
+     * The pending file tasks queue is LIFO to handle large numbers of files
+     * extracted from archive files. At least one image has been processed that
+     * had a folder full of archive files. The queue grew to have thousands of
+     * entries, as each successive archive file was expanded, so now extracted
+     * files get added to the front of the queue so that in such a scenario they
+     * would be processed before the expansion of the next archive file.
+     *
+     * Tasks in the pending file tasks queue are ready to be consumed by the
+     * ingest threads, so the queue is wrapped in a "dispenser" that implements
+     * the IngestTaskQueue interface and is exposed via a getter method.
+     */
+    private final TreeSet<FileIngestTask> rootDirectoryTasks;
+    private final List<FileIngestTask> directoryTasks;
+    private final BlockingDeque<FileIngestTask> pendingFileTasks;
+    private final FileIngestTaskQueue fileTasksDispenser;
+
+    /**
+     * The ingest tasks scheduler allows ingest jobs to query it to see if there
+     * are any tasks in progress for the job. To make this possible, the ingest
+     * tasks scheduler needs to keep track not only of the tasks in its queues,
+     * but also of the tasks that have been handed out for processing by the
+     * ingest threads. Therefore all ingest tasks are added to this list when
+     * they are created and are not removed when an ingest thread takes an
+     * ingest task. Instead, the ingest thread calls back into the scheduler
+     * when the task is completed, at which time the task will be removed from
+     * this list.
+     */
+    private final List<IngestTask> tasksInProgress;
+
+    /**
+     * Gets the ingest tasks scheduler singleton.
+     */
+    synchronized static IngestTasksScheduler getInstance() {
+        if (IngestTasksScheduler.instance == null) {
+            IngestTasksScheduler.instance = new IngestTasksScheduler();
+        }
+        return IngestTasksScheduler.instance;
+    }
+
+    /**
+     * Constructs an ingest tasks scheduler.
+     */
+    private IngestTasksScheduler() {
+        this.pendingDataSourceTasks = new LinkedBlockingQueue<>();
+        this.dataSourceTasksDispenser = new DataSourceIngestTaskQueue();
+        this.rootDirectoryTasks = new TreeSet<>(new RootDirectoryTaskComparator());
+        this.directoryTasks = new ArrayList<>();
+        this.pendingFileTasks = new LinkedBlockingDeque<>();
+        this.fileTasksDispenser = new FileIngestTaskQueue();
+        this.tasksInProgress = new ArrayList<>();
+    }
+
+    /**
+     * Gets this ingest task scheduler's implementation of the IngestTaskQueue
+     * interface for data source ingest tasks.
+     *
+     * @return The data source ingest tasks queue.
+     */
+    IngestTaskQueue getDataSourceIngestTaskQueue() {
+        return this.dataSourceTasksDispenser;
+    }
+
+    /**
+     * Gets this ingest task scheduler's implementation of the IngestTaskQueue
+     * interface for file ingest tasks.
+     *
+     * @return The file ingest tasks queue.
+     */
+    IngestTaskQueue getFileIngestTaskQueue() {
+        return this.fileTasksDispenser;
+    }
+
+    /**
+     * Schedules a data source ingest task and file ingest tasks for an ingest
+     * job.
+     *
+     * @param job The job for which the tasks are to be scheduled.
+     * @throws InterruptedException if the calling thread is blocked due to a
+     * full tasks queue and is interrupted.
+     */
+    synchronized void scheduleIngestTasks(IngestJob job) {
+        // Scheduling of both a data source ingest task and file ingest tasks 
+        // for a job must be an atomic operation. Otherwise, the data source 
+        // task might be completed before the file tasks are scheduled, 
+        // resulting in a potential false positive when another thread checks 
+        // whether or not all the tasks for the job are completed.
+        this.scheduleDataSourceIngestTask(job);
+        this.scheduleFileIngestTasks(job);
+    }
+
+    /**
+     * Schedules a data source ingest task for an ingest job.
+     *
+     * @param job The job for which the tasks are to be scheduled.
+     */
+    synchronized void scheduleDataSourceIngestTask(IngestJob job) {
+        DataSourceIngestTask task = new DataSourceIngestTask(job);
+        this.tasksInProgress.add(task);
+        try {
+            this.pendingDataSourceTasks.put(task);
+        } catch (InterruptedException ex) {
+            /**
+             * The current thread was interrupted while blocked on a full queue.
+             * Discard the task and reset the interrupted flag.
+             */
+            this.tasksInProgress.remove(task);
+            Thread.currentThread().interrupt();
+        }
+    }
+
+    /**
+     * Schedules file ingest tasks for an ingest job.
+     *
+     * @param job The job for which the tasks are to be scheduled.
+     */
+    synchronized void scheduleFileIngestTasks(IngestJob job) {
+        // Get the top level files for the data source associated with this job
+        // and add them to the root directories priority queue.  
+        List<AbstractFile> topLevelFiles = getTopLevelFiles(job.getDataSource());
+        for (AbstractFile firstLevelFile : topLevelFiles) {
+            FileIngestTask task = new FileIngestTask(job, firstLevelFile);
+            if (IngestTasksScheduler.shouldEnqueueFileTask(task)) {
+                this.tasksInProgress.add(task);
+                this.rootDirectoryTasks.add(task);
+            }
+        }
+        shuffleFileTaskQueues();
+    }
+
+    /**
+     * Schedules a file ingest task for an ingest job.
+     *
+     * @param job The job for which the tasks are to be scheduled.
+     * @param file The file to be associated with the task.
+     */
+    synchronized void scheduleFileIngestTask(IngestJob job, AbstractFile file) {
+        FileIngestTask task = new FileIngestTask(job, file);
+        if (IngestTasksScheduler.shouldEnqueueFileTask(task)) {
+            this.tasksInProgress.add(task);
+            addToPendingFileTasksQueue(task);
+        }
+    }
+
+    /**
+     * Allows an ingest thread to notify this ingest task scheduler that a task
+     * has been completed.
+     *
+     * @param task The completed task.
+     */
+    synchronized void notifyTaskCompleted(IngestTask task) {
+        tasksInProgress.remove(task);
+    }
+
+    /**
+     * Queries the task scheduler to determine whether or not all current ingest
+     * tasks for an ingest job are completed.
+     *
+     * @param job The job for which the query is to be performed.
+     * @return True or false.
+     */
+    synchronized boolean tasksForJobAreCompleted(IngestJob job) {
+        for (IngestTask task : tasksInProgress) {
+            if (task.getIngestJob().getId() == job.getId()) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    /**
+     * Clears the task scheduling queues for an ingest job, but does nothing
+     * about tasks that have already been taken by ingest threads. Those tasks
+     * will be flushed out when the ingest threads call back with their task
+     * completed notifications.
+     *
+     * @param job The job for which the tasks are to to canceled.
+     */
+    synchronized void cancelPendingTasksForIngestJob(IngestJob job) {
+        long jobId = job.getId();
+        this.removeTasksForJob(this.rootDirectoryTasks, jobId);
+        this.removeTasksForJob(this.directoryTasks, jobId);
+        this.removeTasksForJob(this.pendingFileTasks, jobId);
+        this.removeTasksForJob(this.pendingDataSourceTasks, jobId);
+    }
+
+    /**
+     * Gets the top level files such as file system root directories, layout
+     * files and virtual directories for a data source. Used to create file
+     * tasks to put into the root directories queue.
+     *
+     * @param dataSource The data source.
+     * @return A list of top level files.
+     */
+    private static List<AbstractFile> getTopLevelFiles(Content dataSource) {
+        List<AbstractFile> topLevelFiles = new ArrayList<>();
+        Collection<AbstractFile> rootObjects = dataSource.accept(new GetRootDirectoryVisitor());
+        if (rootObjects.isEmpty() && dataSource instanceof AbstractFile) {
+            // The data source is itself a file to be processed.
+            topLevelFiles.add((AbstractFile) dataSource);
+        } else {
+            for (AbstractFile root : rootObjects) {
+                List<Content> children;
+                try {
+                    children = root.getChildren();
+                    if (children.isEmpty()) {
+                        // Add the root object itself, it could be an unallocated
+                        // space file, or a child of a volume or an image.
+                        topLevelFiles.add(root);
+                    } else {
+                        // The root object is a file system root directory, get
+                        // the files within it.
+                        for (Content child : children) {
+                            if (child instanceof AbstractFile) {
+                                topLevelFiles.add((AbstractFile) child);
+                            }
+                        }
+                    }
+                } catch (TskCoreException ex) {
+                    logger.log(Level.WARNING, "Could not get children of root to enqueue: " + root.getId() + ": " + root.getName(), ex); //NON-NLS
+                }
+            }
+        }
+        return topLevelFiles;
+    }
+
+    /**
+     * "Shuffles" the file task queues to ensure that there is at least one task
+     * in the pending file ingest tasks queue, as long as there are still file
+     * ingest tasks to be performed.
+     */
+    synchronized private void shuffleFileTaskQueues() {
+        // This is synchronized because it is called both by synchronized 
+        // methods of this ingest scheduler and an unsynchronized method of its
+        // file tasks "dispenser".
+        while (true) {
+            // Loop until either the pending file tasks queue is NOT empty
+            // or the upstream queues that feed into it ARE empty.
+            if (!this.pendingFileTasks.isEmpty()) {
+                // There are file tasks ready to be consumed, exit.
+                return;
+            }
+            if (this.directoryTasks.isEmpty()) {
+                if (this.rootDirectoryTasks.isEmpty()) {
+                    // There are no root directory tasks to move into the
+                    // directory queue, exit.
+                    return;
+                } else {
+                    // Move the next root directory task into the 
+                    // directories queue. Note that the task was already 
+                    // added to the tasks in progress list when the task was
+                    // created in scheduleFileIngestTasks().
+                    this.directoryTasks.add(this.rootDirectoryTasks.pollFirst());
+                }
+            }
+
+            // Try to add the most recently added directory from the 
+            // directory tasks queue to the pending file tasks queue. 
+            FileIngestTask directoryTask = this.directoryTasks.remove(this.directoryTasks.size() - 1);
+            this.tasksInProgress.remove(directoryTask);
+            if (shouldEnqueueFileTask(directoryTask)) {
+                addToPendingFileTasksQueue(directoryTask);
+            } else {
+                this.tasksInProgress.remove(directoryTask);
+            }
+
+            // If the directory contains subdirectories or files, try to 
+            // enqueue tasks for them as well. 
+            final AbstractFile directory = directoryTask.getFile();
+            try {
+                for (Content child : directory.getChildren()) {
+                    if (child instanceof AbstractFile) {
+                        AbstractFile file = (AbstractFile) child;
+                        FileIngestTask childTask = new FileIngestTask(directoryTask.getIngestJob(), file);
+                        if (file.hasChildren()) {
+                            // Found a subdirectory, put the task in the 
+                            // pending directory tasks queue. Note the 
+                            // addition of the task to the tasks in progress
+                            // list. This is necessary because this is the
+                            // first appearance of this task in the queues.
+                            this.tasksInProgress.add(childTask);
+                            this.directoryTasks.add(childTask);
+                        } else if (shouldEnqueueFileTask(childTask)) {
+                            // Found a file, put the task directly into the
+                            // pending file tasks queue. The new task will 
+                            // be put into the tasks in progress list by the
+                            // addToPendingFileTasksQueue() helper.
+                            this.tasksInProgress.add(childTask);
+                            addToPendingFileTasksQueue(childTask);
+                        }
+                    }
+                }
+            } catch (TskCoreException ex) {
+                String errorMessage = String.format("An error occurred getting the children of %s", directory.getName()); //NON-NLS
+                logger.log(Level.SEVERE, errorMessage, ex);
+            }
+        }
+    }
+
+    /**
+     * Examines the file associated with a file ingest task to determine whether
+     * or not the file should be processed and therefore whether or not the task
+     * should be enqueued.
+     *
+     * @param task The task to be scrutinized.
+     * @return True or false.
+     */
+    private static boolean shouldEnqueueFileTask(final FileIngestTask task) {
+        final AbstractFile file = task.getFile();
+
+        // Skip the task if the file is an unallocated space file and the
+        // process unallocated space flag is not set for this job.
+        if (!task.getIngestJob().shouldProcessUnallocatedSpace()
+                && file.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS)) {
+            return false;
+        }
+
+        // Skip the task if the file is actually the pseudo-file for the parent
+        // or current directory.
+        String fileName = file.getName();
+        if (fileName.equals(".") || fileName.equals("..")) {
+            return false;
+        }
+
+        // Skip the task if the file is one of a select group of special, large
+        // NTFS or FAT file system files.
+        if (file instanceof org.sleuthkit.datamodel.File) {
+            final org.sleuthkit.datamodel.File f = (org.sleuthkit.datamodel.File) file;
+
+            // Get the type of the file system, if any, that owns the file.
+            TskData.TSK_FS_TYPE_ENUM fsType = TskData.TSK_FS_TYPE_ENUM.TSK_FS_TYPE_UNSUPP;
+            try {
+                FileSystem fs = f.getFileSystem();
+                if (fs != null) {
+                    fsType = fs.getFsType();
+                }
+            } catch (TskCoreException ex) {
+                logger.log(Level.SEVERE, "Error querying file system for " + f, ex); //NON-NLS
+            }
+
+            // If the file system is not NTFS or FAT, don't skip the file.
+            if ((fsType.getValue() & FAT_NTFS_FLAGS) == 0) {
+                return true;
+            }
+
+            // Find out whether the file is in a root directory. 
+            boolean isInRootDir = false;
+            try {
+                AbstractFile parent = f.getParentDirectory();
+                isInRootDir = parent.isRoot();
+            } catch (TskCoreException ex) {
+                logger.log(Level.WARNING, "Error querying parent directory for" + f.getName(), ex); //NON-NLS
+            }
+
+            // If the file is in the root directory of an NTFS or FAT file 
+            // system, check its meta-address and check its name for the '$'
+            // character and a ':' character (not a default attribute).
+            if (isInRootDir && f.getMetaAddr() < 32) {
+                String name = f.getName();
+                if (name.length() > 0 && name.charAt(0) == '$' && name.contains(":")) {
+                    return false;
+                }
+            }
+        }
+
+        return true;
+    }
+
+    /**
+     * Adds a file ingest task to the blocking pending tasks queue.
+     *
+     * @param task The task to add.
+     */
+    synchronized private void addToPendingFileTasksQueue(FileIngestTask task) {
+        try {
+            this.pendingFileTasks.putFirst(task);
+        } catch (InterruptedException ex) {
+            /**
+             * The current thread was interrupted while blocked on a full queue.
+             * Discard the task and reset the interrupted flag.
+             */
+            this.tasksInProgress.remove(task);
+            Thread.currentThread().interrupt();
+        }
+    }
+
+    /**
+     * Removes all of the ingest tasks associated with an ingest job from a
+     * tasks queue. The task is removed from the the tasks in progress list as
+     * well.
+     *
+     * @param taskQueue The queue from which to remove the tasks.
+     * @param jobId The id of the job for which the tasks are to be removed.
+     */
+    private void removeTasksForJob(Collection<? extends IngestTask> taskQueue, long jobId) {
+        Iterator<? extends IngestTask> iterator = taskQueue.iterator();
+        while (iterator.hasNext()) {
+            IngestTask task = iterator.next();
+            if (task.getIngestJob().getId() == jobId) {
+                this.tasksInProgress.remove(task);
+                iterator.remove();
+            }
+        }
+    }
+
+    /**
+     * Counts the number of ingest tasks in a task queue for a given job.
+     *
+     * @param queue The queue for which to count tasks.
+     * @param jobId The id of the job for which the tasks are to be counted.
+     * @return The count.
+     */
+    private static int countTasksForJob(Collection<? extends IngestTask> queue, long jobId) {
+        Iterator<? extends IngestTask> iterator = queue.iterator();
+        int count = 0;
+        while (iterator.hasNext()) {
+            IngestTask task = (IngestTask) iterator.next();
+            if (task.getIngestJob().getId() == jobId) {
+                count++;
+            }
+        }
+        return count;
+    }
+
+    /**
+     * Returns a snapshot of the states of the tasks in progress for an ingest
+     * job.
+     *
+     * @param jobId The identifier assigned to the job.
+     * @return
+     */
+    synchronized IngestJobTasksSnapshot getTasksSnapshotForJob(long jobId) {
+        return new IngestJobTasksSnapshot(jobId);
+    }
+
+    /**
+     * Prioritizes tasks for the root directories file ingest tasks queue (file
+     * system root directories, layout files and virtual directories).
+     */
+    private static class RootDirectoryTaskComparator implements Comparator<FileIngestTask> {
+
+        @Override
+        public int compare(FileIngestTask q1, FileIngestTask q2) {
+            AbstractFilePriority.Priority p1 = AbstractFilePriority.getPriority(q1.getFile());
+            AbstractFilePriority.Priority p2 = AbstractFilePriority.getPriority(q2.getFile());
+            if (p1 == p2) {
+                return (int) (q2.getFile().getId() - q1.getFile().getId());
+            } else {
+                return p2.ordinal() - p1.ordinal();
+            }
+        }
+
+        private static class AbstractFilePriority {
+
+            enum Priority {
+
+                LAST, LOW, MEDIUM, HIGH
+            }
+
+            static final List<Pattern> LAST_PRI_PATHS = new ArrayList<>();
+
+            static final List<Pattern> LOW_PRI_PATHS = new ArrayList<>();
+
+            static final List<Pattern> MEDIUM_PRI_PATHS = new ArrayList<>();
+
+            static final List<Pattern> HIGH_PRI_PATHS = new ArrayList<>();
+            /* prioritize root directory folders based on the assumption that we
+             * are
+             * looking for user content. Other types of investigations may want
+             * different
+             * priorities. */
+
+            static /* prioritize root directory
+             * folders based on the assumption that we are
+             * looking for user content. Other types of investigations may want
+             * different
+             * priorities. */ {
+                // these files have no structure, so they go last
+                //unalloc files are handled as virtual files in getPriority()
+                //LAST_PRI_PATHS.schedule(Pattern.compile("^\\$Unalloc", Pattern.CASE_INSENSITIVE));
+                //LAST_PRI_PATHS.schedule(Pattern.compile("^\\Unalloc", Pattern.CASE_INSENSITIVE));
+                LAST_PRI_PATHS.add(Pattern.compile("^pagefile", Pattern.CASE_INSENSITIVE));
+                LAST_PRI_PATHS.add(Pattern.compile("^hiberfil", Pattern.CASE_INSENSITIVE));
+                // orphan files are often corrupt and windows does not typically have
+                // user content, so put them towards the bottom
+                LOW_PRI_PATHS.add(Pattern.compile("^\\$OrphanFiles", Pattern.CASE_INSENSITIVE));
+                LOW_PRI_PATHS.add(Pattern.compile("^Windows", Pattern.CASE_INSENSITIVE));
+                // all other files go into the medium category too
+                MEDIUM_PRI_PATHS.add(Pattern.compile("^Program Files", Pattern.CASE_INSENSITIVE));
+                // user content is top priority
+                HIGH_PRI_PATHS.add(Pattern.compile("^Users", Pattern.CASE_INSENSITIVE));
+                HIGH_PRI_PATHS.add(Pattern.compile("^Documents and Settings", Pattern.CASE_INSENSITIVE));
+                HIGH_PRI_PATHS.add(Pattern.compile("^home", Pattern.CASE_INSENSITIVE));
+                HIGH_PRI_PATHS.add(Pattern.compile("^ProgramData", Pattern.CASE_INSENSITIVE));
+            }
+
+            /**
+             * Get the enabled priority for a given file.
+             *
+             * @param abstractFile
+             *
+             * @return
+             */
+            static AbstractFilePriority.Priority getPriority(final AbstractFile abstractFile) {
+                if (!abstractFile.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.FS)) {
+                    //quickly filter out unstructured content
+                    //non-fs virtual files and dirs, such as representing unalloc space
+                    return AbstractFilePriority.Priority.LAST;
+                }
+                //determine the fs files priority by name
+                final String path = abstractFile.getName();
+                if (path == null) {
+                    return AbstractFilePriority.Priority.MEDIUM;
+                }
+                for (Pattern p : HIGH_PRI_PATHS) {
+                    Matcher m = p.matcher(path);
+                    if (m.find()) {
+                        return AbstractFilePriority.Priority.HIGH;
+                    }
+                }
+                for (Pattern p : MEDIUM_PRI_PATHS) {
+                    Matcher m = p.matcher(path);
+                    if (m.find()) {
+                        return AbstractFilePriority.Priority.MEDIUM;
+                    }
+                }
+                for (Pattern p : LOW_PRI_PATHS) {
+                    Matcher m = p.matcher(path);
+                    if (m.find()) {
+                        return AbstractFilePriority.Priority.LOW;
+                    }
+                }
+                for (Pattern p : LAST_PRI_PATHS) {
+                    Matcher m = p.matcher(path);
+                    if (m.find()) {
+                        return AbstractFilePriority.Priority.LAST;
+                    }
+                }
+                //default is medium
+                return AbstractFilePriority.Priority.MEDIUM;
+            }
+        }
+    }
+
+    /**
+     * Wraps access to pending data source ingest tasks in the interface
+     * required by the ingest threads.
+     */
+    private final class DataSourceIngestTaskQueue implements IngestTaskQueue {
+
+        /**
+         * @inheritDoc
+         */
+        @Override
+        public IngestTask getNextTask() throws InterruptedException {
+            return IngestTasksScheduler.this.pendingDataSourceTasks.take();
+        }
+    }
+
+    /**
+     * Wraps access to pending file ingest tasks in the interface required by
+     * the ingest threads.
+     */
+    private final class FileIngestTaskQueue implements IngestTaskQueue {
+
+        /**
+         * @inheritDoc
+         */
+        @Override
+        public IngestTask getNextTask() throws InterruptedException {
+            FileIngestTask task = IngestTasksScheduler.this.pendingFileTasks.takeFirst();
+            shuffleFileTaskQueues();
+            return task;
+        }
+
+    }
+
+    /**
+     * A snapshot of ingest tasks data for an ingest job.
+     */
+    class IngestJobTasksSnapshot {
+
+        private final long jobId;
+        private final long rootQueueSize;
+        private final long dirQueueSize;
+        private final long fileQueueSize;
+        private final long dsQueueSize;
+        private final long runningListSize;
+
+        /**
+         * Constructs a snapshot of ingest tasks data for an ingest job.
+         *
+         * @param jobId The identifier associated with the job.
+         */
+        IngestJobTasksSnapshot(long jobId) {
+            this.jobId = jobId;
+            this.rootQueueSize = countTasksForJob(IngestTasksScheduler.this.rootDirectoryTasks, jobId);
+            this.dirQueueSize = countTasksForJob(IngestTasksScheduler.this.directoryTasks, jobId);
+            this.fileQueueSize = countTasksForJob(IngestTasksScheduler.this.pendingFileTasks, jobId);
+            this.dsQueueSize = countTasksForJob(IngestTasksScheduler.this.pendingDataSourceTasks, jobId);
+            this.runningListSize = countTasksForJob(IngestTasksScheduler.this.tasksInProgress, jobId) - fileQueueSize - dsQueueSize;
+        }
+
+        /**
+         * Gets the identifier associated with the ingest job for which this
+         * snapshot was created.
+         *
+         * @return The ingest job identifier.
+         */
+        long getJobId() {
+            return jobId;
+        }
+
+        /**
+         * Gets the number of file ingest tasks associated with the job that are
+         * in the root directories queue.
+         *
+         * @return The tasks count.
+         */
+        long getRootQueueSize() {
+            return rootQueueSize;
+        }
+
+        /**
+         * Gets the number of file ingest tasks associated with the job that are
+         * in the root directories queue.
+         *
+         * @return The tasks count.
+         */
+        long getDirectoryTasksQueueSize() {
+            return dirQueueSize;
+        }
+
+        long getFileQueueSize() {
+            return fileQueueSize;
+        }
+
+        long getDsQueueSize() {
+            return dsQueueSize;
+        }
+
+        long getRunningListSize() {
+            return runningListSize;
+        }
+
+    }
+
+}

Core/src/org/sleuthkit/autopsy/ingest/PipelineConfig.xml

@@ -0,0 +1,26 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Default initial pipeline_config.xml
+Contains only the core ingest modules that ship with Autopsy -->
+<PIPELINE_CONFIG>
+    <PIPELINE type="ImageAnalysisStageOne">
+        <MODULE>org.sleuthkit.autopsy.recentactivity.RecentActivityExtracterModuleFactory</MODULE>
+        <MODULE>org.sleuthkit.autopsy.modules.android.AndroidModuleFactory</MODULE>
+    </PIPELINE>
+
+    <PIPELINE type="FileAnalysis">
+        <MODULE>org.sleuthkit.autopsy.modules.hashdatabase.HashLookupModuleFactory</MODULE>
+        <MODULE>org.sleuthkit.autopsy.modules.filetypeid.FileTypeIdModuleFactory</MODULE>
+        <MODULE>org.sleuthkit.autopsy.modules.sevenzip.ArchiveFileExtractorModuleFactory</MODULE>
+        <MODULE>org.sleuthkit.autopsy.modules.exif.ExifParserModuleFactory</MODULE>
+        <MODULE>org.sleuthkit.autopsy.keywordsearch.KeywordSearchModuleFactory</MODULE>
+        <MODULE>org.sleuthkit.autopsy.thunderbirdparser.EmailParserModuleFactory</MODULE>
+        <MODULE>org.sleuthkit.autopsy.modules.fileextmismatch.FileExtMismatchDetectorModuleFactory</MODULE>
+        <MODULE>org.sleuthkit.autopsy.modules.interestingitems.InterestingItemsIngestModuleFactory</MODULE>
+		<MODULE>org.sleuthkit.autopsy.modules.photoreccarver.PhotoRecCarverIngestModuleFactory</MODULE>
+    </PIPELINE>
+    
+    <PIPELINE type="ImageAnalysisStageTwo">
+        <MODULE>org.sleuthkit.autopsy.modules.e01verify.E01VerifierModuleFactory</MODULE>
+    </PIPELINE>
+    
+</PIPELINE_CONFIG>

Core/src/org/sleuthkit/autopsy/ingest/pipeline_config.xml

@@ -1,18 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<!-- Default initial pipeline_config.xml
-Contains only the core ingest modules that ship with Autopsy -->
-<PIPELINE_CONFIG>
-    <PIPELINE type="FileAnalysis">
-      <MODULE order="1" type="plugin" location="org.sleuthkit.autopsy.hashdatabase.HashDbIngestModule" arguments="" />
-      <MODULE order="2" type="plugin" location="org.sleuthkit.autopsy.modules.filetypeid.FileTypeIdIngestModule" arguments=""/>
-      <MODULE order="3" type="plugin" location="org.sleuthkit.autopsy.modules.sevenzip.SevenZipIngestModule" arguments="" />
-      <MODULE order="4" type="plugin" location="org.sleuthkit.autopsy.modules.exifparser.ExifParserFileIngestModule"/>
-      <MODULE order="5" type="plugin" location="org.sleuthkit.autopsy.keywordsearch.KeywordSearchIngestModule"/>
-      <MODULE order="6" type="plugin" location="org.sleuthkit.autopsy.thunderbirdparser.ThunderbirdMboxFileIngestModule" arguments=""/>
-      <MODULE order="7" type="plugin" location="org.sleuthkit.autopsy.modules.fileextmismatch.FileExtMismatchIngestModule" arguments=""/>
-    </PIPELINE>
-
-    <PIPELINE type="ImageAnalysis">
-      <MODULE order="1" type="plugin" location="org.sleuthkit.autopsy.recentactivity.RAImageIngestModule" arguments=""/>      
-    </PIPELINE>
-</PIPELINE_CONFIG>

RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java

@@ -393,6 +393,7 @@ class ExtractRegistry extends Extract {
             result = result.replaceAll("\\r", ""); //NON-NLS
             result = result.replaceAll("'", "'"); //NON-NLS
             result = result.replaceAll("&", "&"); //NON-NLS
+            result = result.replace('\0', ' '); // NON-NLS
             String enddoc = "</document>"; //NON-NLS
             String stringdoc = startdoc + result + enddoc;
             DocumentBuilder builder = DocumentBuilderFactory.newInstance().newDocumentBuilder();

docs/doxygen-user/Doxyfile

@@ -901,7 +901,7 @@ HTML_HEADER            =
 # each generated HTML page. If it is left blank doxygen will generate a
 # standard footer.
 
-HTML_FOOTER            =
+HTML_FOOTER            = footer.html
 
 # The HTML_STYLESHEET tag can be used to specify a user-defined cascading
 # style sheet that is used by each HTML page. It can be used to

docs/doxygen-user/footer.html

@@ -0,0 +1,7 @@
+<hr/>
+<p><i>Copyright &#169; 2012-2014 Basis Technology <br/> 
+This work is licensed under a
+<a rel="license" href="http://creativecommons.org/licenses/by-sa/3.0/us/">Creative Commons Attribution-Share Alike 3.0 United States License</a>.
+</i></p>
+</body>
+</html>

docs/doxygen-user/quick_start.dox

@@ -1,200 +1,115 @@
 /*! \page quick_start_page Quick Start Guide
 
+\section s1 Adding a Data Source (image, local disk, logical files)
 
-        <h1>Adding a Data Source (image, local disk, logical files)</h1>
-        <p>
-            Data sources are added to a <strong>case</strong>. A case can have a single data source or it can have multiple data source if they are related.
-            Currently, a single report is generated for an entire case, so if you need to report on individual data sources, then you should use one data source per case.
-        </p>
+Data sources are added to a <strong>case</strong>. A case can have a single data source or it can have multiple data source if they are related. Currently, a single report is generated for an entire case, so if you need to report on individual data sources, then you should use one data source per case.
 
-        <h2>Creating a Case</h2>
-        <p>
-            To create a case, use either the &quot;Create New Case&quot; option on the Welcome screen or from the &quot;File&quot; menu.
-            This will start the <strong>New Case Wizard</strong>. You will need to supply it with the name of the case and a directory to store the case results into.
-            You can optionally provide case numbers and other details.
-        </p>
+\subsection s2 Creating a Case
+To create a case, use either the "Create New Case" option on the Welcome screen or from the "File" menu. This will start the <strong>New Case Wizard</strong>. You will need to supply it with the name of the case and a directory to store the case results into. You can optionally provide case numbers and other details.
+
+\subsection s3 Adding a Data Source
+The next step is to add input data source to the case. The <strong>Add Data Source Wizard</strong> will start automatically after the case is created or you can manually start it from the &quot;File&quot; menu or toolbar. You will need to choose the type of input data source to add (image, local disk or logical files and folders).  Next, supply it with the location of the source to add. 
+        
+
+- For a disk image, browse to the first file in the set (Autopsy will find the rest of the files). Autopsy currently supports E01 and raw (dd) files.
+- For local disk, select one of the detected disks.   Autopsy will add the current view of the disk to the case (i.e. snapshot of the meta-data).  However, the individual file content (not meta-data) does get updated with the changes made to the disk. Note, you may need run Autopsy as an Administrator to detect all disks.
+- For logical files (a single file or folder of files), use the "Add" button to add one or more files or folders on your system to the case.  Folders will be recursively added to the case.
+
+There are a couple of options in the wizard that will allow you to make the ingest process faster. These typically deal with deleted files. It will take longer if unallocated space is analyzed and the entire drive is searched for deleted files. In some scenarios, these recovery steps must be performed and in other scenarios these steps are not needed and instead fast results on the allocated files are needed. Use these options to control how long the analysis will take.
+        
+Autopsy will start to analyze these data sources and add them to the case and internal database.  While it is doing that, it will prompt you to configure the Ingest Modules.
 
 
-        <h2>Adding a Data Source</h2>
-        <p>
-            The next step is to add input data source to the case.
-            The <strong>Add Data Source Wizard</strong> will start automatically after the case is created or you can manually start it from the &quot;File&quot; menu or toolbar.
-            You will need to choose the type of input data source to add (image, local disk or logical files and folders).
-            Next, supply it with the location of the source to add. 
-        </p>
-        <ul>
-            <li>For a disk image, browse to the first file in the set (Autopsy will find the rest of the files). Autopsy currently supports E01 and raw (dd) files.  
-            </li>
-            <li>
-            For local disk, select one of the detected disks.  
-            Autopsy will add the current view of the disk to the case (i.e. snapshot of the meta-data). 
-            However, the individual file content (not meta-data) does get updated with the changes made to the disk.
-            Note, you may need run Autopsy as an Administrator to detect all disks.
-            </li>
-            <li>For logical files (a single file or folder of files), use the "Add" button to add one or more files or folders on your system to the case.  Folders will be recursively added to the case.</li>
-        </ul>
-            
-     
-        <p>
-            There are a couple of options in the wizard that will allow you to make the ingest process faster.
-            These typically deal with deleted files.
-            It will take longer if unallocated space is analyzed and the entire drive is searched for deleted files.
-            In some scenarios, these recovery steps must be performed and in other scenarios these steps are not needed and instead fast results on the allocated files are needed.
-            Use these options to control how long the analysis will take.
-        </p>
+\subsection s4 Ingest Modules
 
-        <p>
-          Autopsy will start to analyze these data sources and add them to the case and internal database.  While it is doing that, it will prompt you to configure the Ingest Modules. </p>
+You will next be prompted to configure the Ingest Modules. Ingest modules will run in the background and perform specific tasks. The Ingest Modules analyze files in a prioritized order so that files in a user's directory are analyzed before files in other folders. Ingest modules can be developed by third-parties and here are some of the standard ingest modules that come with Autopsy:
+
+- <strong>Recent Activity</strong> extracts user activity as saved by web browsers and the OS.  Also runs regripper on the registry hive. 
+- <strong>Hash Lookup</strong> uses hash databases to ignore known files from the NIST NSRL and flag known bad files. Use the "Advanced" button to add and configure the hash databases to use during this process. You will get updates on known bad file hits as the ingest occurs.  You can later add hash databases via the Tools -&gt; Options menu in the main UI.  You can download an index of the NIST NSRL from http://sourceforge.net/projects/autopsy/files/NSRL/
+- <strong>Keyword Search</strong> uses keyword lists to identify files with specific words in them. You can select the keyword lists to search for automatically and you can create new lists using the "Advanced" button. Note that with keyword search, you can always conduct searches after ingest has finished. The keyword lists that you select during ingest will be searched for at periodic intervals and you will get the results in real-time. You do not need to wait for all files to be indexed.
+- <strong>Archive Extractor</strong> opens ZIP, RAR, and other archive formats and sends the files from those archive files back through the pipelines for analysis.
+- <strong>Exif Image Parser</strong> extracts EXIF information from JPEG files and posts the results into the tree in the main UI.
+- <strong>Thunderbird Parser</strong> Identifies Thunderbird MBOX files and extracts the e-mails from them.
 
 
-        <h2>Ingest Modules</h2>
-        <p>
-            You will next be prompted to configure the Ingest Modules.
-            Ingest modules will run in the background and perform specific tasks.
-            The Ingest Modules analyze files in a prioritized order so that files in a user's directory are analyzed before files in other folders.
-            Ingest modules can be developed by third-parties and here are some of the standard ingest modules that come with Autopsy:
-        </p>
-        <ul>
-            <li><strong>Recent Activity</strong>
-                extracts user activity as saved by web browsers and the OS.  Also runs regripper on the registry hive. 
-            </li>
-            <li><strong>Hash Lookup</strong>
-                uses hash databases to ignore known files from the NIST NSRL and flag known bad files.
-                Use the "Advanced" button to add and configure the hash databases to use during this process.
-                You will get updates on known bad file hits as the ingest occurs.  You can later add hash databases
-                via the Tools -&gt; Options menu in the main UI.  You can download an index of the NIST NSRL from 
-                <a href="http://sourceforge.net/projects/autopsy/files/NSRL/">here</a>. 
-            </li>
-            <li><strong>Keyword Search</strong>
-                uses keyword lists to identify files with specific words in them.
-                You can select the keyword lists to search for automatically and you can create new lists using the "Advanced" button.
-                Note that with keyword search, you can always conduct searches after ingest has finished.
-                The keyword lists that you select during ingest will be searched for at periodic intervals and you will get the results in real-time.
-                You do not need to wait for all files to be indexed.
-            </li>
-            <li><strong>Archive Extractor</strong> opens ZIP, RAR, and other archive formats and sends the files from those archive files back
-            through the pipelines for analysis.</li>
-            <li><strong>Exif Image Parser</strong> extracts EXIF information from JPEG files and posts the results into the tree in the main UI.</li>
-            <li><strong>Thunderbird Parser</strong> Identifies Thunderbird MBOX files and extracts the e-mails from them.</li>
-        </ul>
-        <p>
-            When you select a module, you will have the option to change its settings.  
-            For example, you can configure which keyword search lists to use during ingest and which hash databases to use.
-            Refer to the help system inside of Autopsy for details on configuring each module.
-        </p>
-        <p>
-            While ingest modules are running in the background, you will see a progress bar in the lower right.
-            You can use the GUI to review incoming results and perform other tasks while ingest at that time.
-        </p>
+When you select a module, you will have the option to change its settings.  For example, you can configure which keyword search lists to use during ingest and which hash databases to use. Refer to the help system inside of Autopsy for details on configuring each module.
+
+While ingest modules are running in the background, you will see a progress bar in the lower right. You can use the GUI to review incoming results and perform other tasks while ingest at that time.
+
+\section s1a Analysis Basics
+
+\image html screenshot.png
+
+You will start all of your analysis techniques from the tree on the left.
+
+- The Data Sources root node shows all data in the case.
+ - The individual image nodes show the file system structure of the disk images or local disks in the case.
+ - The LogicalFileSet nodes show the logical files in the case.
+- The Views node shows the same data from a file type or timeline perspective.
+- The Results node shows the output from the ingest modules.
+
+When you select a node from the tree on the left, a list of files will be shown in the upper right.  You can use the Thumbnail view in the upper right to view the pictures. When you select a file from the upper right, its contents will be shown in the lower right. You can use the tabs in the lower right to view the text of the file, an image, or the hex data.
+
+If you are viewing files from the Views and Results nodes, you can right-click on a file to go to its file system location. This feature is useful to see what else the user stored in the same folder as the file that you are currently looking at. You can also right click on a file to extract it to the local system.
+
+If you want to search for single keywords, then you can use the search box in the upper right of the program. The results will be shown in a table in the upper right.
+
+You can tag (or bookmark) arbitrary files so that you can more quickly find them later or so that you can include them specifically in a report.
+
+\subsection s2a Ingest Inbox
+
+As you are going through the results in the tree, the ingest modules are running in the background.
+The results are shown in the tree as soon as the ingest modules find them and report them.
+
+The Ingest Inbox receives messages from the ingest modules as they find results.
+You can open the inbox to see what has been recently found.
+It keeps track of what messages you have read.
+
+The intended use of this inbox is that you can focus on some data for a while and then check back on the inbox at a time that is convenient for them.
+You can then see what else was found while you were focused on the previous task.
+You may learn that a known bad file was found or that a file was found with a relevant keyword and then decide to focus on that for a while.
+
+When you select a message, you can then jump to the Results tree where more details can be found or jump to the file's location in the filesystem.
+
+\subsection s2b Timeline (beta)
+There is a basic timeline view that you can access via the Tools -&gt; Make Timeline feature. This will take a few minutes to create the timeline for analysis.  Its features are still in development.
 
 
-        <h1>Analysis Basics</h1>
+\section s5 Example Use Cases
+In this section, we will provide examples of how to do common analysis tasks.
 
+\subsection s5a Web Artifacts
 
-        <img src="screenshot.png" alt="Autopsy Screenshot" />
-        <p>You will start all of your analysis techniques from the tree on the left.</p>
-        <ul>
-            <li>The Data Sources root node shows all data in the case.</li>
-            <ul>
-              <li>The individual image nodes show the file system structure of the disk images or local disks in the case.</li>
-              <li>The LogicalFileSet nodes show the logical files in the case.</li>
-            </ul>
-            <li>The Views node shows the same data from a file type or timeline perspective.</li>
-            <li>The Results node shows the output from the ingest modules.</li>
-        </ul>
+If you want to view the user's recent web activity, make sure that the Recent Activity ingest module was enabled.
+You can then go to the &quot;Results &quot; node in the tree on the left and then into the &quot;Extracted Data&quot; node.
+There, you can find bookmarks, cookies, downloads, and history.
 
-        <p>
-            When you select a node from the tree on the left, a list of files will be shown in the upper right. 
-            You can use the Thumbnail view in the upper right to view the pictures.
-            When you select a file from the upper right, its contents will be shown in the lower right.
-            You can use the tabs in the lower right to view the text of the file, an image, or the hex data.
-        </p>
+\subsection s5b Known Bad Hash Files
 
-        <p>
-            If you are viewing files from the Views and Results nodes, you can right-click on a file to go to its file system location.
-            This feature is useful to see what else the user stored in the same folder as the file that you are currently looking at.
-            You can also right click on a file to extract it to the local system.
-        </p>
-        <p>
-            If you want to search for single keywords, then you can use the search box in the upper right of the program.
-            The results will be shown in a table in the upper right.
-        </p>
+If you want to see if the data source had known bad files, make sure that the Hash Lookup ingest module was enabled.
+You can then view the &quot;Hashset Hits&quot; section in the &quot;Results&quot; area of the tree on the left.
+Note that hash lookup can take a long time, so this section will be updated as long as the ingest process is occurring.
+Use the Ingest Inbox to keep track of what known bad files were recently found.
 
-        <p> You can tag (or bookmark) arbitrary files so that you can more quickly find them later or so that you can include them specifically in a report.</p>
+When you find a known bad file in this interface, you may want to right click on the file to also view the file's original location.
+You may find additional files that are relevant and stored in the same folder as this file.
 
-        <h2>Ingest Inbox</h2>
-        <p>
-            As you are going through the results in the tree, the ingest modules are running in the background.
-            The results are shown in the tree as soon as the ingest modules find them and report them.
-        </p>
-        <p>
-            The Ingest Inbox receives messages from the ingest modules as they find results.
-            You can open the inbox to see what has been recently found.
-            It keeps track of what messages you have read.
-        </p>
-        <p>
-            The intended use of this inbox is that you can focus on some data for a while and then check back on the inbox at a time that is convenient for them.
-            You can then see what else was found while you were focused on the previous task.
-            You may learn that a known bad file was found or that a file was found with a relevant keyword and then decide to focus on that for a while.
-        </p>
-        <p> When you select a message, you can then jump to the Results tree where more details can be found or jump to the file's location in the filesystem.</p>
+\subsection s5c Media: Images and Videos
 
-        <h2>Timeline (Beta)</h2>
-        <p>There is a basic timeline view that you can access via the Tools -&gt; Make Timeline feature. This will take a few minutes to create the timeline for analysis.  Its features are still in development.</p>
+If you want to see all images and video on the disk image, then go to the &quot;Views&quot; section in the tree on the left and then &quot;File Types&quot;.
+Select either &quot;Images&quot; or &quot;Videos&quot;.
+You can use the thumbnail option in the upper right to view thumbnails of all images.
 
+<strong>Note</strong>: We are working on making this more efficient when there are lots of images and we are working on the feature to display video thumbnails.
 
-        <h1>Example Use Cases</h1>
-        <p>In this section, we will provide examples of how to do common analysis tasks.</p>
-
-        <h2>Web Artifacts</h2>
-        <p>
-            If you want to view the user's recent web activity, make sure that the Recent Activity ingest module was enabled.
-            You can then go to the &quot;Results &quot; node in the tree on the left and then into the &quot;Extracted Data&quot; node.
-            There, you can find bookmarks, cookies, downloads, and history.
-        </p>
-
-        <h2>Known Bad Hash Files</h2>
-        <p>
-            If you want to see if the data source had known bad files, make sure that the Hash Lookup ingest module was enabled.
-            You can then view the &quot;Hashset Hits&quot; section in the &quot;Results&quot; area of the tree on the left.
-            Note that hash lookup can take a long time, so this section will be updated as long as the ingest process is occurring.
-            Use the Ingest Inbox to keep track of what known bad files were recently found.
-        </p>
-        <p>
-            When you find a known bad file in this interface, you may want to right click on the file to also view the file's original location.
-            You may find additional files that are relevant and stored in the same folder as this file.
-        </p>
-
-        <h2>Media: Images and Videos</h2>
-        <p>
-            If you want to see all images and video on the disk image, then go to the &quot;Views&quot; section in the tree on the left and then &quot;File Types&quot;.
-            Select either &quot;Images&quot; or &quot;Videos&quot;.
-            You can use the thumbnail option in the upper right to view thumbnails of all images.
-        </p>
-        <ul class="note">
-            <li><strong>Note</strong>:
-                We are working on making this more efficient when there are lots of images and we are working on the feature to display video thumbnails.
-            </li>
-        </ul>
-        <p>You can select an image or video from the upper right and view the video or image in the lower right. Video will be played with sound.</p>
-
-
-        <h1>Reporting</h1>
-        <p>
-            A final report can be generated that will include all analysis results.
-            Use the &quot;Generate Report&quot; button to create this.
-            It will create an HTML or XLS report in the Reports folder of the case folder.
-            If you forgot the location of your case folder, you can determine it using the &quot;Case Properties&quot; option in the &quot;File&quot; menu.
-            There is also an option to export report files to a separate folder outside of the case folder. 
-        </p>
-
-        <hr>
-        <p><i>Copyright &#169; 2012-2013 Basis Technology.</i></p>
-        <p><i>
-            This work is licensed under a
-            <a rel="license" href="http://creativecommons.org/licenses/by-sa/3.0/us/">Creative Commons Attribution-Share Alike 3.0 United States License</a>.
-        </i></p>
+You can select an image or video from the upper right and view the video or image in the lower right. Video will be played with sound.
 
+\section s6 Reporting
 
+A final report can be generated that will include all analysis results.
+Use the &quot;Generate Report&quot; button to create this.
+It will create an HTML or XLS report in the Reports folder of the case folder.
+If you forgot the location of your case folder, you can determine it using the &quot;Case Properties&quot; option in the &quot;File&quot; menu.
+There is also an option to export report files to a separate folder outside of the case folder. 
 
 */

docs/doxygen/Doxyfile

@@ -918,7 +918,7 @@ HTML_HEADER            =
 # each generated HTML page. If it is left blank doxygen will generate a
 # standard footer.
 
-HTML_FOOTER            =
+HTML_FOOTER            = footer.html
 
 # The HTML_STYLESHEET tag can be used to specify a user-defined cascading
 # style sheet that is used by each HTML page. It can be used to

docs/doxygen/modReport.dox

@@ -66,6 +66,10 @@ Autopsy will also display the panel returned by getConfigurationPanel() in the g
 
 Typically a general report module should interact with both the Blackboard API in the org.sleuthkit.datamodel.SleuthkitCase class, in addition to an API (possibly external/thirdparty) to convert Blackboard Artifacts to the desired reporting format.
 
+\subsection report_create_module_showing Showing Results
+
+You should call Case.addReport() with the path to your report so that it is shown in the Autopsy tree.  You can specify a specific file or folder and the user can then view it later.
+
 \subsection report_create_module_layer Installing your Report Module
 
 Report modules developed using Java must be registered in a layer.xml file. This file allows Autopsy to find the report module.

docs/doxygen/platformConcepts.dox

@@ -55,6 +55,7 @@ The followig are basic services that are available to any module:
 - Pop-up Windows: 
 If you have a background task that needs the provide the user with feedback, you can use the org.sleuthkit.autopsy.coreutils.MessageNotifyUtil.Notify.show() method to make a message in the lower right hand area.
 - Module Settings: If you want to persist settings between invocations of Autopsy, you can use org.sleuthkit.autopsy.coreutils.ModuleSettings.
+- Content Utilities: The org.sleuthkit.autopsy.datamodel.ContentUtils class has utility methods to write files from Autopsy to local disk. Specifically the org.sleuthkit.autopsy.datamodel.ContentUtils.writeToFile() method. 
 - Platform Utilities: The org.sleuthkit.autopsy.coreutils.PlatformUtil class allows you to save resources into the user folder and determine paths for the user folders.  Specifically:
  - PlatformUtil.extractResourceToUserConfigDir()
  - PlatformUtil.isWindowsOS()