mstoeck3/autopsy-flatpak
1
0
Fork 0
mirror of https://github.com/overcuriousity/autopsy-flatpak.git synced 2025-07-19 11:07:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

Use streaming ingest for disk images in auto ingest

Browse Source
This commit is contained in:
apriestman 2020-07-16 13:24:25 -04:00
parent b7839b3aa9
commit bd480827b5
6 changed files with 77 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
Core/src/org/sleuthkit/autopsy/casemodule/DefaultIngestStream.java
View File

@ -19,6 +19,7 @@
package org.sleuthkit.autopsy.casemodule;
import java.util.List;
import org.sleuthkit.autopsy.ingest.IngestJob;
import org.sleuthkit.autopsy.ingest.IngestStream;
import org.sleuthkit.autopsy.ingest.IngestStreamClosedException;
@ -36,6 +37,11 @@ class DefaultIngestStream implements IngestStream {
// Do nothing
}
@Override
public IngestJob getIngestJob() {
return null;
}
@Override
public synchronized boolean isClosed() {
return isClosed;

36
Core/src/org/sleuthkit/autopsy/casemodule/ImageDSProcessor.java
View File

@ -465,6 +465,42 @@ public class ImageDSProcessor implements DataSourceProcessor, AutoIngestDataSour
doAddImageProcess(deviceId, dataSourcePath.toString(), sectorSize, timeZone, ignoreFatOrphanFiles, null, null, null, progressMonitor, callBack);
}
@Override
public IngestStream processWithIngestStream(String deviceId, Path dataSourcePath, IngestJobSettings settings, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callBack) {
this.deviceId = deviceId;
this.imagePath = dataSourcePath.toString();
this.sectorSize = 0;
this.timeZone = Calendar.getInstance().getTimeZone().getID();
this.ignoreFatOrphanFiles = false;
setDataSourceOptionsCalled = true;
// Set up the data source before creating the ingest stream
try {
image = SleuthkitJNI.addImageToDatabase(Case.getCurrentCase().getSleuthkitCase(),
new String[]{imagePath}, sectorSize, timeZone, md5, sha1, sha256, deviceId);
} catch (TskCoreException ex) {
logger.log(Level.SEVERE, "Error adding data source with path " + imagePath + " to database", ex);
final List<String> errors = new ArrayList<>();
errors.add(ex.getMessage());
callBack.done(DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS, errors, new ArrayList<>());
return null;
}
// Now initialize the ingest stream
try {
ingestStream = IngestManager.getInstance().openIngestStream(image, settings);
} catch (TskCoreException ex) {
logger.log(Level.SEVERE, "Error starting ingest modules", ex);
final List<String> errors = new ArrayList<>();
errors.add(ex.getMessage());
callBack.done(DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS, errors, new ArrayList<>());
return null;
}
doAddImageProcess(deviceId, dataSourcePath.toString(), sectorSize, timeZone, ignoreFatOrphanFiles, null, null, null, progressMonitor, callBack);
return ingestStream;
}
/**
* Sets the configuration of the data source processor without using the
* selection and configuration panel.

6
Core/src/org/sleuthkit/autopsy/datasourceprocessors/AutoIngestDataSourceProcessor.java
View File

@ -22,6 +22,8 @@ import java.nio.file.Path;
import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessor;
import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorCallback;
import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorProgressMonitor;
import org.sleuthkit.autopsy.ingest.IngestJobSettings;
import org.sleuthkit.autopsy.ingest.IngestStream;
/**
* Interface implemented by DataSourceProcessors in order to be supported by
@ -66,6 +68,10 @@ public interface AutoIngestDataSourceProcessor extends DataSourceProcessor {
*/
void process(String deviceId, Path dataSourcePath, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callBack);
default IngestStream processWithIngestStream(String deviceId, Path dataSourcePath, IngestJobSettings settings, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callBack) {
throw new UnsupportedOperationException("Streaming ingest not supported for this data source processor");
}
/**
* A custom exception for the use of AutomatedIngestDataSourceProcessor.
*/

5
Core/src/org/sleuthkit/autopsy/ingest/IngestJobInputStream.java
View File

@ -59,6 +59,11 @@ class IngestJobInputStream implements IngestStream {
ingestJob.addStreamingIngestFiles(fileObjectIds);
}
@Override
public IngestJob getIngestJob() {
return ingestJob;
}
@Override
public synchronized void close() {
closed = true;

2
Core/src/org/sleuthkit/autopsy/ingest/IngestStream.java
View File

@ -35,6 +35,8 @@ public interface IngestStream {
*/
void addFiles(List<Long> fileObjectIds) throws IngestStreamClosedException;
IngestJob getIngestJob();
/**
* Closes the ingest stream. Should be called after all files from data
* source have been sent to the stream.

27
Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java
View File

@ -101,6 +101,7 @@ import org.sleuthkit.autopsy.ingest.IngestJobSettings;
import org.sleuthkit.autopsy.ingest.IngestJobStartResult;
import org.sleuthkit.autopsy.ingest.IngestManager;
import org.sleuthkit.autopsy.ingest.IngestModuleError;
import org.sleuthkit.autopsy.ingest.IngestStream;
import org.sleuthkit.autopsy.keywordsearch.KeywordSearchModuleException;
import org.sleuthkit.autopsy.keywordsearch.Server;
import org.sleuthkit.datamodel.Content;
@ -165,6 +166,7 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
private AutoIngestJob currentJob;
@GuardedBy("jobsLock")
private List<AutoIngestJob> completedJobs;
private IngestStream currentIngestStream = null;
private CoordinationService coordinationService;
private JobProcessingTask jobProcessingTask;
private Future<?> jobProcessingTaskFuture;
@ -2443,6 +2445,7 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
return;
}
currentIngestStream = null;
runDataSourceProcessor(caseForJob, dataSource);
if (dataSource.getContent().isEmpty()) {
currentJob.setProcessingStage(AutoIngestJob.Stage.COMPLETED, Date.from(Instant.now()));
@ -2558,7 +2561,13 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
caseForJob.notifyAddingDataSource(taskId);
jobLogger.logDataSourceProcessorSelected(selectedProcessor.getDataSourceType());
sysLogger.log(Level.INFO, "Identified data source type for {0} as {1}", new Object[]{manifestPath, selectedProcessor.getDataSourceType()});
selectedProcessor.process(dataSource.getDeviceId(), dataSource.getPath(), progressMonitor, callBack);
if (selectedProcessor.supportsIngestStream()) {
IngestJobSettings ingestJobSettings = new IngestJobSettings(AutoIngestUserPreferences.getAutoModeIngestModuleContextString());
// TODO check for settings errors
currentIngestStream = selectedProcessor.processWithIngestStream(dataSource.getDeviceId(), dataSource.getPath(), ingestJobSettings, progressMonitor, callBack);
} else {
selectedProcessor.process(dataSource.getDeviceId(), dataSource.getPath(), progressMonitor, callBack);
}
ingestLock.wait();
// at this point we got the content object(s) from the current DSP.
@ -2674,11 +2683,19 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
IngestManager.getInstance().addIngestJobEventListener(INGEST_JOB_EVENTS_OF_INTEREST, ingestJobEventListener);
try {
synchronized (ingestLock) {
// TODO don't do all this
IngestJobSettings ingestJobSettings = new IngestJobSettings(AutoIngestUserPreferences.getAutoModeIngestModuleContextString());
List<String> settingsWarnings = ingestJobSettings.getWarnings();
if (settingsWarnings.isEmpty()) {
IngestJobStartResult ingestJobStartResult = IngestManager.getInstance().beginIngestJob(dataSource.getContent(), ingestJobSettings);
IngestJob ingestJob = ingestJobStartResult.getJob();
IngestJobStartResult ingestJobStartResult = null;
IngestJob ingestJob;
if (currentIngestStream == null) {
ingestJobStartResult = IngestManager.getInstance().beginIngestJob(dataSource.getContent(), ingestJobSettings);
ingestJob = ingestJobStartResult.getJob();
} else {
ingestJob = currentIngestStream.getIngestJob();
}
if (null != ingestJob) {
currentJob.setIngestJob(ingestJob);
/*
@ -2714,7 +2731,7 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
}
}
}
} else if (!ingestJobStartResult.getModuleErrors().isEmpty()) {
} else if (ingestJobStartResult != null && !ingestJobStartResult.getModuleErrors().isEmpty()) {
for (IngestModuleError error : ingestJobStartResult.getModuleErrors()) {
sysLogger.log(Level.SEVERE, String.format("%s ingest module startup error for %s", error.getModuleDisplayName(), manifestPath), error.getThrowable());
}
@ -2722,7 +2739,7 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
setErrorsOccurredFlagForCase(caseDirectoryPath);
jobLogger.logIngestModuleStartupErrors();
throw new AnalysisStartupException(String.format("Error(s) during ingest module startup for %s", manifestPath));
} else {
} else if (ingestJobStartResult != null) {
sysLogger.log(Level.SEVERE, String.format("Ingest manager ingest job start error for %s", manifestPath), ingestJobStartResult.getStartupException());
currentJob.setErrorsOccurred(true);
setErrorsOccurredFlagForCase(caseDirectoryPath);