Use streaming ingest for disk images in auto ingest

Core/src/org/sleuthkit/autopsy/casemodule/DefaultIngestStream.java

@@ -19,6 +19,7 @@
 package org.sleuthkit.autopsy.casemodule;
 
 import java.util.List;
+import org.sleuthkit.autopsy.ingest.IngestJob;
 import org.sleuthkit.autopsy.ingest.IngestStream;
 import org.sleuthkit.autopsy.ingest.IngestStreamClosedException;
 
@@ -36,6 +37,11 @@ class DefaultIngestStream implements IngestStream {
         // Do nothing
     }
     
+    @Override
+    public IngestJob getIngestJob() {
+        return null;
+    }
+
     @Override
     public synchronized boolean isClosed() {
         return isClosed;

Core/src/org/sleuthkit/autopsy/casemodule/ImageDSProcessor.java

@@ -465,6 +465,42 @@ public class ImageDSProcessor implements DataSourceProcessor, AutoIngestDataSour
         doAddImageProcess(deviceId, dataSourcePath.toString(), sectorSize, timeZone, ignoreFatOrphanFiles, null, null, null, progressMonitor, callBack);
     }
     
+    @Override
+    public IngestStream processWithIngestStream(String deviceId, Path dataSourcePath, IngestJobSettings settings, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callBack) {
+        this.deviceId = deviceId;
+        this.imagePath = dataSourcePath.toString();
+        this.sectorSize = 0;
+        this.timeZone = Calendar.getInstance().getTimeZone().getID();
+        this.ignoreFatOrphanFiles = false;
+        setDataSourceOptionsCalled = true;
+    
+        // Set up the data source before creating the ingest stream
+        try {
+            image = SleuthkitJNI.addImageToDatabase(Case.getCurrentCase().getSleuthkitCase(),
+                new String[]{imagePath}, sectorSize, timeZone, md5, sha1, sha256, deviceId);
+        } catch (TskCoreException ex) {
+            logger.log(Level.SEVERE, "Error adding data source with path " + imagePath + " to database", ex);
+            final List<String> errors = new ArrayList<>();
+            errors.add(ex.getMessage());
+            callBack.done(DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS, errors, new ArrayList<>());
+            return null;
+        }
+
+        // Now initialize the ingest stream
+        try {
+            ingestStream = IngestManager.getInstance().openIngestStream(image, settings);
+        } catch (TskCoreException ex) {
+            logger.log(Level.SEVERE, "Error starting ingest modules", ex);
+            final List<String> errors = new ArrayList<>();
+            errors.add(ex.getMessage());
+            callBack.done(DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS, errors, new ArrayList<>());
+            return null;
+        }        
+        
+        doAddImageProcess(deviceId, dataSourcePath.toString(), sectorSize, timeZone, ignoreFatOrphanFiles, null, null, null, progressMonitor, callBack);
+        return ingestStream;
+    }
+
     /**
      * Sets the configuration of the data source processor without using the
      * selection and configuration panel.

Core/src/org/sleuthkit/autopsy/datasourceprocessors/AutoIngestDataSourceProcessor.java

@@ -22,6 +22,8 @@ import java.nio.file.Path;
 import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessor;
 import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorCallback;
 import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorProgressMonitor;
+import org.sleuthkit.autopsy.ingest.IngestJobSettings;
+import org.sleuthkit.autopsy.ingest.IngestStream;
 
 /**
  * Interface implemented by DataSourceProcessors in order to be supported by
@@ -66,6 +68,10 @@ public interface AutoIngestDataSourceProcessor extends DataSourceProcessor {
      */
     void process(String deviceId, Path dataSourcePath, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callBack);
 
+    default IngestStream processWithIngestStream(String deviceId, Path dataSourcePath, IngestJobSettings settings, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callBack) {
+        throw new UnsupportedOperationException("Streaming ingest not supported for this data source processor");
+    }
+   
     /**
      * A custom exception for the use of AutomatedIngestDataSourceProcessor.
      */

Core/src/org/sleuthkit/autopsy/ingest/IngestJobInputStream.java

@@ -59,6 +59,11 @@ class IngestJobInputStream implements IngestStream {
         ingestJob.addStreamingIngestFiles(fileObjectIds);
     }
     
+    @Override
+    public IngestJob getIngestJob() {
+        return ingestJob;
+    }
+
     @Override
     public synchronized void close() {
         closed = true;

Core/src/org/sleuthkit/autopsy/ingest/IngestStream.java

@@ -35,6 +35,8 @@ public interface IngestStream {
      */
     void addFiles(List<Long> fileObjectIds) throws IngestStreamClosedException;
     
+    IngestJob getIngestJob();
+
     /**
      * Closes the ingest stream. Should be called after all files from data
      * source have been sent to the stream.

Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java

@@ -101,6 +101,7 @@ import org.sleuthkit.autopsy.ingest.IngestJobSettings;
 import org.sleuthkit.autopsy.ingest.IngestJobStartResult;
 import org.sleuthkit.autopsy.ingest.IngestManager;
 import org.sleuthkit.autopsy.ingest.IngestModuleError;
+import org.sleuthkit.autopsy.ingest.IngestStream;
 import org.sleuthkit.autopsy.keywordsearch.KeywordSearchModuleException;
 import org.sleuthkit.autopsy.keywordsearch.Server;
 import org.sleuthkit.datamodel.Content;
@@ -165,6 +166,7 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
     private AutoIngestJob currentJob;
     @GuardedBy("jobsLock")
     private List<AutoIngestJob> completedJobs;
+    private IngestStream currentIngestStream = null;
     private CoordinationService coordinationService;
     private JobProcessingTask jobProcessingTask;
     private Future<?> jobProcessingTaskFuture;
@@ -2443,6 +2445,7 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
                 return;
             }
 
+            currentIngestStream = null;
             runDataSourceProcessor(caseForJob, dataSource);
             if (dataSource.getContent().isEmpty()) {
                 currentJob.setProcessingStage(AutoIngestJob.Stage.COMPLETED, Date.from(Instant.now()));
@@ -2558,7 +2561,13 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
                         caseForJob.notifyAddingDataSource(taskId);
                         jobLogger.logDataSourceProcessorSelected(selectedProcessor.getDataSourceType());
                         sysLogger.log(Level.INFO, "Identified data source type for {0} as {1}", new Object[]{manifestPath, selectedProcessor.getDataSourceType()});
+                        if (selectedProcessor.supportsIngestStream()) {
+                            IngestJobSettings ingestJobSettings = new IngestJobSettings(AutoIngestUserPreferences.getAutoModeIngestModuleContextString());
+                            // TODO check for settings errors
+                            currentIngestStream = selectedProcessor.processWithIngestStream(dataSource.getDeviceId(), dataSource.getPath(), ingestJobSettings, progressMonitor, callBack);
+                        } else {
                             selectedProcessor.process(dataSource.getDeviceId(), dataSource.getPath(), progressMonitor, callBack);
+                        }
                         ingestLock.wait();
 
                         // at this point we got the content object(s) from the current DSP.
@@ -2674,11 +2683,19 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
             IngestManager.getInstance().addIngestJobEventListener(INGEST_JOB_EVENTS_OF_INTEREST, ingestJobEventListener);
             try {
                 synchronized (ingestLock) {
+                    // TODO don't do all this
                     IngestJobSettings ingestJobSettings = new IngestJobSettings(AutoIngestUserPreferences.getAutoModeIngestModuleContextString());
                     List<String> settingsWarnings = ingestJobSettings.getWarnings();
                     if (settingsWarnings.isEmpty()) {
-                        IngestJobStartResult ingestJobStartResult = IngestManager.getInstance().beginIngestJob(dataSource.getContent(), ingestJobSettings);
+                        
-                        IngestJob ingestJob = ingestJobStartResult.getJob();
+                        IngestJobStartResult ingestJobStartResult = null;
+                        IngestJob ingestJob;
+                        if (currentIngestStream == null) {
+                            ingestJobStartResult = IngestManager.getInstance().beginIngestJob(dataSource.getContent(), ingestJobSettings);
+                            ingestJob = ingestJobStartResult.getJob();
+                        } else {
+                            ingestJob = currentIngestStream.getIngestJob();
+                        }
                         if (null != ingestJob) {
                             currentJob.setIngestJob(ingestJob);
                             /*
@@ -2714,7 +2731,7 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
                                     }
                                 }
                             }
-                        } else if (!ingestJobStartResult.getModuleErrors().isEmpty()) {
+                        } else if (ingestJobStartResult != null && !ingestJobStartResult.getModuleErrors().isEmpty()) {
                             for (IngestModuleError error : ingestJobStartResult.getModuleErrors()) {
                                 sysLogger.log(Level.SEVERE, String.format("%s ingest module startup error for %s", error.getModuleDisplayName(), manifestPath), error.getThrowable());
                             }
@@ -2722,7 +2739,7 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
                             setErrorsOccurredFlagForCase(caseDirectoryPath);
                             jobLogger.logIngestModuleStartupErrors();
                             throw new AnalysisStartupException(String.format("Error(s) during ingest module startup for %s", manifestPath));
-                        } else {
+                        } else if (ingestJobStartResult != null) {
                             sysLogger.log(Level.SEVERE, String.format("Ingest manager ingest job start error for %s", manifestPath), ingestJobStartResult.getStartupException());
                             currentJob.setErrorsOccurred(true);
                             setErrorsOccurredFlagForCase(caseDirectoryPath);