diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/DefaultIngestStream.java b/Core/src/org/sleuthkit/autopsy/casemodule/DefaultIngestStream.java index 08bc0fa427..dd83f18694 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/DefaultIngestStream.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/DefaultIngestStream.java @@ -19,6 +19,7 @@ package org.sleuthkit.autopsy.casemodule; import java.util.List; +import org.sleuthkit.autopsy.ingest.IngestJob; import org.sleuthkit.autopsy.ingest.IngestStream; import org.sleuthkit.autopsy.ingest.IngestStreamClosedException; @@ -35,6 +36,11 @@ class DefaultIngestStream implements IngestStream { public void addFiles(List fileObjectIds) throws IngestStreamClosedException { // Do nothing } + + @Override + public IngestJob getIngestJob() { + return null; + } @Override public synchronized boolean isClosed() { diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/ImageDSProcessor.java b/Core/src/org/sleuthkit/autopsy/casemodule/ImageDSProcessor.java index 9dd9a39fd4..207b83ed8a 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/ImageDSProcessor.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/ImageDSProcessor.java @@ -464,6 +464,42 @@ public class ImageDSProcessor implements DataSourceProcessor, AutoIngestDataSour doAddImageProcess(deviceId, dataSourcePath.toString(), sectorSize, timeZone, ignoreFatOrphanFiles, null, null, null, progressMonitor, callBack); } + + @Override + public IngestStream processWithIngestStream(String deviceId, Path dataSourcePath, IngestJobSettings settings, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callBack) { + this.deviceId = deviceId; + this.imagePath = dataSourcePath.toString(); + this.sectorSize = 0; + this.timeZone = Calendar.getInstance().getTimeZone().getID(); + this.ignoreFatOrphanFiles = false; + setDataSourceOptionsCalled = true; + + // Set up the data source before creating the ingest stream + try { + image = SleuthkitJNI.addImageToDatabase(Case.getCurrentCase().getSleuthkitCase(), + new String[]{imagePath}, sectorSize, timeZone, md5, sha1, sha256, deviceId); + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, "Error adding data source with path " + imagePath + " to database", ex); + final List errors = new ArrayList<>(); + errors.add(ex.getMessage()); + callBack.done(DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS, errors, new ArrayList<>()); + return null; + } + + // Now initialize the ingest stream + try { + ingestStream = IngestManager.getInstance().openIngestStream(image, settings); + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, "Error starting ingest modules", ex); + final List errors = new ArrayList<>(); + errors.add(ex.getMessage()); + callBack.done(DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS, errors, new ArrayList<>()); + return null; + } + + doAddImageProcess(deviceId, dataSourcePath.toString(), sectorSize, timeZone, ignoreFatOrphanFiles, null, null, null, progressMonitor, callBack); + return ingestStream; + } /** * Sets the configuration of the data source processor without using the diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/AutoIngestDataSourceProcessor.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/AutoIngestDataSourceProcessor.java index 20f3bb59bd..95956bb5b0 100644 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/AutoIngestDataSourceProcessor.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/AutoIngestDataSourceProcessor.java @@ -22,6 +22,8 @@ import java.nio.file.Path; import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessor; import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorCallback; import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorProgressMonitor; +import org.sleuthkit.autopsy.ingest.IngestJobSettings; +import org.sleuthkit.autopsy.ingest.IngestStream; /** * Interface implemented by DataSourceProcessors in order to be supported by @@ -66,6 +68,10 @@ public interface AutoIngestDataSourceProcessor extends DataSourceProcessor { */ void process(String deviceId, Path dataSourcePath, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callBack); + default IngestStream processWithIngestStream(String deviceId, Path dataSourcePath, IngestJobSettings settings, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callBack) { + throw new UnsupportedOperationException("Streaming ingest not supported for this data source processor"); + } + /** * A custom exception for the use of AutomatedIngestDataSourceProcessor. */ diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobInputStream.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobInputStream.java index 4e8e9c4019..a2687d5c1d 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobInputStream.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobInputStream.java @@ -58,6 +58,11 @@ class IngestJobInputStream implements IngestStream { } ingestJob.addStreamingIngestFiles(fileObjectIds); } + + @Override + public IngestJob getIngestJob() { + return ingestJob; + } @Override public synchronized void close() { diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestStream.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestStream.java index 62a42af208..4776056c98 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestStream.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestStream.java @@ -34,6 +34,8 @@ public interface IngestStream { * @throws IngestStreamClosedException */ void addFiles(List fileObjectIds) throws IngestStreamClosedException; + + IngestJob getIngestJob(); /** * Closes the ingest stream. Should be called after all files from data diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java index 48b86025af..da86ff8263 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java @@ -101,6 +101,7 @@ import org.sleuthkit.autopsy.ingest.IngestJobSettings; import org.sleuthkit.autopsy.ingest.IngestJobStartResult; import org.sleuthkit.autopsy.ingest.IngestManager; import org.sleuthkit.autopsy.ingest.IngestModuleError; +import org.sleuthkit.autopsy.ingest.IngestStream; import org.sleuthkit.autopsy.keywordsearch.KeywordSearchModuleException; import org.sleuthkit.autopsy.keywordsearch.Server; import org.sleuthkit.datamodel.Content; @@ -165,6 +166,7 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen private AutoIngestJob currentJob; @GuardedBy("jobsLock") private List completedJobs; + private IngestStream currentIngestStream = null; private CoordinationService coordinationService; private JobProcessingTask jobProcessingTask; private Future jobProcessingTaskFuture; @@ -2443,6 +2445,7 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen return; } + currentIngestStream = null; runDataSourceProcessor(caseForJob, dataSource); if (dataSource.getContent().isEmpty()) { currentJob.setProcessingStage(AutoIngestJob.Stage.COMPLETED, Date.from(Instant.now())); @@ -2558,7 +2561,13 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen caseForJob.notifyAddingDataSource(taskId); jobLogger.logDataSourceProcessorSelected(selectedProcessor.getDataSourceType()); sysLogger.log(Level.INFO, "Identified data source type for {0} as {1}", new Object[]{manifestPath, selectedProcessor.getDataSourceType()}); - selectedProcessor.process(dataSource.getDeviceId(), dataSource.getPath(), progressMonitor, callBack); + if (selectedProcessor.supportsIngestStream()) { + IngestJobSettings ingestJobSettings = new IngestJobSettings(AutoIngestUserPreferences.getAutoModeIngestModuleContextString()); + // TODO check for settings errors + currentIngestStream = selectedProcessor.processWithIngestStream(dataSource.getDeviceId(), dataSource.getPath(), ingestJobSettings, progressMonitor, callBack); + } else { + selectedProcessor.process(dataSource.getDeviceId(), dataSource.getPath(), progressMonitor, callBack); + } ingestLock.wait(); // at this point we got the content object(s) from the current DSP. @@ -2674,11 +2683,19 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen IngestManager.getInstance().addIngestJobEventListener(INGEST_JOB_EVENTS_OF_INTEREST, ingestJobEventListener); try { synchronized (ingestLock) { + // TODO don't do all this IngestJobSettings ingestJobSettings = new IngestJobSettings(AutoIngestUserPreferences.getAutoModeIngestModuleContextString()); List settingsWarnings = ingestJobSettings.getWarnings(); if (settingsWarnings.isEmpty()) { - IngestJobStartResult ingestJobStartResult = IngestManager.getInstance().beginIngestJob(dataSource.getContent(), ingestJobSettings); - IngestJob ingestJob = ingestJobStartResult.getJob(); + + IngestJobStartResult ingestJobStartResult = null; + IngestJob ingestJob; + if (currentIngestStream == null) { + ingestJobStartResult = IngestManager.getInstance().beginIngestJob(dataSource.getContent(), ingestJobSettings); + ingestJob = ingestJobStartResult.getJob(); + } else { + ingestJob = currentIngestStream.getIngestJob(); + } if (null != ingestJob) { currentJob.setIngestJob(ingestJob); /* @@ -2714,7 +2731,7 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen } } } - } else if (!ingestJobStartResult.getModuleErrors().isEmpty()) { + } else if (ingestJobStartResult != null && !ingestJobStartResult.getModuleErrors().isEmpty()) { for (IngestModuleError error : ingestJobStartResult.getModuleErrors()) { sysLogger.log(Level.SEVERE, String.format("%s ingest module startup error for %s", error.getModuleDisplayName(), manifestPath), error.getThrowable()); } @@ -2722,7 +2739,7 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen setErrorsOccurredFlagForCase(caseDirectoryPath); jobLogger.logIngestModuleStartupErrors(); throw new AnalysisStartupException(String.format("Error(s) during ingest module startup for %s", manifestPath)); - } else { + } else if (ingestJobStartResult != null) { sysLogger.log(Level.SEVERE, String.format("Ingest manager ingest job start error for %s", manifestPath), ingestJobStartResult.getStartupException()); currentJob.setErrorsOccurred(true); setErrorsOccurredFlagForCase(caseDirectoryPath);