mstoeck3/autopsy-flatpak
1
0
Fork 0
mirror of https://github.com/overcuriousity/autopsy-flatpak.git synced 2025-07-15 09:17:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

Complete most of the multi-stage pipeline feature

Browse Source
This commit is contained in:
Richard Cordovano 2014-11-02 23:25:47 -05:00
parent 90a103455b
commit b6ca38fef4
6 changed files with 186 additions and 93 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestPipeline.java
View File

@ -43,7 +43,7 @@ final class DataSourceIngestPipeline {
DataSourceIngestModuleDecorator module = new DataSourceIngestModuleDecorator(template.createDataSourceIngestModule(), template.getModuleName());
modules.add(module);
}
}
}
}
boolean isEmpty() {

102
Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java
View File

@ -20,7 +20,9 @@ package org.sleuthkit.autopsy.ingest;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.atomic.AtomicLong;
@ -114,7 +116,7 @@ final class IngestJob {
long jobId = nextIngestJobId.incrementAndGet();
IngestJob job = new IngestJob(jobId, dataSource, processUnallocatedSpace);
errors = job.start(ingestModuleTemplates);
if (errors.isEmpty() && (job.hasDataSourceIngestPipeline() || job.hasFileIngestPipeline())) {
if (errors.isEmpty() && (job.hasDataSourceIngestPipeline() || job.hasFileIngestPipeline())) { // RJCTODO: What about 2nd stage only?
ingestJobsById.put(jobId, job);
IngestManager.getInstance().fireIngestJobStarted(jobId);
IngestJob.ingestScheduler.scheduleIngestTasks(job);
@ -135,7 +137,8 @@ final class IngestJob {
/**
* RJCTODO
* @return
*
* @return
*/
static List<IngestJobSnapshot> getJobSnapshots() {
List<IngestJobSnapshot> snapShots = new ArrayList<>();
@ -144,7 +147,7 @@ final class IngestJob {
}
return snapShots;
}
/**
* RJCTODO
*/
@ -555,20 +558,38 @@ final class IngestJob {
* @throws InterruptedException
*/
private void createIngestPipelines(List<IngestModuleTemplate> ingestModuleTemplates) throws InterruptedException {
// RJCTODO: Use config file
// Sort the ingest module templates as required for the pipelines.
List<IngestModuleTemplate> firstStageDataSourceModuleTemplates = new ArrayList<>();
List<IngestModuleTemplate> secondStageDataSourceModuleTemplates = new ArrayList<>();
List<IngestModuleTemplate> fileIngestModuleTemplates = new ArrayList<>();
// RJCTODO: Improve variable names!
// Make mappings of ingest module factory class names to templates.
Map<String, IngestModuleTemplate> dataSourceModuleTemplates = new HashMap<>();
Map<String, IngestModuleTemplate> fileModuleTemplates = new HashMap<>();
for (IngestModuleTemplate template : ingestModuleTemplates) {
if (template.isDataSourceIngestModuleTemplate()) {
firstStageDataSourceModuleTemplates.add(template);
} else {
firstStageDataSourceModuleTemplates.add(template);
dataSourceModuleTemplates.put(template.getModuleFactory().getClass().getCanonicalName(), template);
}
if (template.isFileIngestModuleTemplate()) {
fileModuleTemplates.put(template.getModuleFactory().getClass().getCanonicalName(), template);
}
}
// Contruct the pipelines.
// Use the mappings and the ingest pipelines configuration to create
// ordered lists of ingest module templates for each ingest pipeline.
IngestPipelinesConfiguration pipelineConfigs = IngestPipelinesConfiguration.getInstance();
List<IngestModuleTemplate> firstStageDataSourceModuleTemplates = this.getConfiguredIngestModuleTemplates(dataSourceModuleTemplates, pipelineConfigs.getStageOneDataSourceIngestPipelineConfig());
List<IngestModuleTemplate> fileIngestModuleTemplates = this.getConfiguredIngestModuleTemplates(fileModuleTemplates, pipelineConfigs.getFileIngestPipelineConfig());
List<IngestModuleTemplate> secondStageDataSourceModuleTemplates = this.getConfiguredIngestModuleTemplates(dataSourceModuleTemplates, pipelineConfigs.getStageTwoDataSourceIngestPipelineConfig());
// Add any module templates that were not specified in the pipeline
// configurations to an appropriate pipeline - either the first stage
// data source ingest pipeline or the file ingest pipeline.
for (IngestModuleTemplate template : dataSourceModuleTemplates.values()) {
firstStageDataSourceModuleTemplates.add(template);
}
for (IngestModuleTemplate template : fileModuleTemplates.values()) {
fileIngestModuleTemplates.add(template);
}
// Contruct the data source ingest pipelines.
this.firstStageDataSourceIngestPipeline = new DataSourceIngestPipeline(this, firstStageDataSourceModuleTemplates);
this.secondStageDataSourceIngestPipeline = new DataSourceIngestPipeline(this, secondStageDataSourceModuleTemplates);
this.dataSourceIngestPipeline = firstStageDataSourceIngestPipeline;
@ -580,6 +601,28 @@ final class IngestJob {
}
}
/**
* Use an ordered list of ingest module factory class names to create an
* ordered subset of a collection ingest module templates. The ingest module
* templates are removed from the input collection as they are added to the
* output collection.
*
* @param ingestModuleTemplates A mapping of ingest module factory class
* names to ingest module templates.
* @param pipelineConfig An ordered list of ingest module factory class
* names representing an ingest pipeline.
* @return
*/
List<IngestModuleTemplate> getConfiguredIngestModuleTemplates(Map<String, IngestModuleTemplate> ingestModuleTemplates, List<String> pipelineConfig) {
List<IngestModuleTemplate> templates = new ArrayList<>();
for (String moduleClassName : pipelineConfig) {
if (ingestModuleTemplates.containsKey(moduleClassName)) {
templates.add(ingestModuleTemplates.remove(moduleClassName));
}
}
return templates;
}
/**
* Starts up each of the file and data source ingest modules to collect
* possible errors.
@ -748,7 +791,7 @@ final class IngestJob {
this.dataSourceIngestProgress = null;
}
}
IngestJob.ingestJobsById.remove(this.id);
if (!this.isCancelled()) {
logger.log(Level.INFO, "Ingest job {0} completed", this.id);
@ -784,7 +827,7 @@ final class IngestJob {
class IngestJobSnapshot {
private final long jobId;
private final String dataSource;
private final String dataSource;
private final long startTime;
private final long processedFiles;
private final long estimatedFilesToProcess;
@ -809,7 +852,8 @@ final class IngestJob {
/**
* RJCTODO
* @return
*
* @return
*/
long getJobId() {
return this.jobId;
@ -817,12 +861,13 @@ final class IngestJob {
/**
* RJCTODO
* @return
*
* @return
*/
String getDataSource() {
return dataSource;
}
}
/**
* Gets files per second throughput since job started.
*
@ -870,10 +915,11 @@ final class IngestJob {
long getFilesEstimated() {
return estimatedFilesToProcess;
}
/**
* RJCTODO
* @return
*
* @return
*/
long getRootQueueSize() {
return this.tasksSnapshot.getRootQueueSize();
@ -881,7 +927,8 @@ final class IngestJob {
/**
* RJCTODO
* @return
*
* @return
*/
long getDirQueueSize() {
return this.tasksSnapshot.getDirQueueSize();
@ -889,7 +936,8 @@ final class IngestJob {
/**
* RJCTODO
* @return
*
* @return
*/
long getFileQueueSize() {
return this.tasksSnapshot.getFileQueueSize();
@ -897,7 +945,8 @@ final class IngestJob {
/**
* RJCTODO
* @return
*
* @return
*/
long getDsQueueSize() {
return this.tasksSnapshot.getDsQueueSize();
@ -905,12 +954,13 @@ final class IngestJob {
/**
* RJCTODO
* @return
*
* @return
*/
long getRunningListSize() {
return this.tasksSnapshot.getRunningListSize();
}
}
}
}

2
Core/src/org/sleuthkit/autopsy/ingest/IngestModuleFactoryLoader.java
View File

@ -51,6 +51,8 @@ final class IngestModuleFactoryLoader {
private static final String SAMPLE_EXECUTABLE_MODULE_FACTORY_CLASS_NAME = SampleExecutableIngestModuleFactory.class.getCanonicalName();
private static final ArrayList<String> coreModuleOrdering = new ArrayList<String>() {
{
// RJCTODO: Find out wherer ot put the photorec carver
// The ordering of the core ingest module factories implemented
// using Java is hard-coded.
add("org.sleuthkit.autopsy.recentactivity.RecentActivityExtracterModuleFactory"); //NON-NLS

130
Core/src/org/sleuthkit/autopsy/ingest/IngestPipelinesConfiguration.java
View File

@ -18,13 +18,13 @@
*/
package org.sleuthkit.autopsy.ingest;
import java.io.File;
import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.logging.Level;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.coreutils.ModuleSettings;
import org.sleuthkit.autopsy.coreutils.PlatformUtil;
import org.sleuthkit.autopsy.coreutils.XMLUtil;
import org.w3c.dom.Document;
@ -33,32 +33,36 @@ import org.w3c.dom.NodeList;
/**
* Provides data source and file ingest pipeline configurations as ordered lists
* of ingest module class names. The order of the module class names indicates
* the desired sequence of ingest module instances in an ingest modules
* pipeline.
* of ingest module factory class names.
*/
final class IngestPipelinesConfiguration {
private static final Logger logger = Logger.getLogger(IngestPipelinesConfiguration.class.getName());
private static final String PIPELINE_CONFIG_FILE_VERSION_KEY = "PipelineConfigFileVersion"; //NON-NLS
private static final String PIPELINE_CONFIG_FILE_VERSION_NO_STRING = "1";
private static final int PIPELINE_CONFIG_FILE_VERSION_NO = 1;
private static final String PIPELINES_CONFIG_FILE = "pipeline_config.xml"; //NON-NLS
private static final String PIPELINES_CONFIG_FILE_XSD = "PipelineConfigSchema.xsd"; //NON-NLS
private static final String XML_PIPELINE_ELEM = "PIPELINE"; //NON-NLS
private static final String XML_PIPELINE_TYPE_ATTR = "type"; //NON-NLS
private static final String DATA_SOURCE_INGEST_PIPELINE_TYPE = "ImageAnalysis"; //NON-NLS
private static final String PIPELINES_CONFIG_FILE = "PipelineConfig.xml"; //NON-NLS
private static final String PIPELINE_ELEM = "PIPELINE"; //NON-NLS
private static final int NUMBER_OF_PIPELINE_DEFINITIONS = 3;
private static final String PIPELINE_TYPE_ATTR = "type"; //NON-NLS
private static final String STAGE_ONE_DATA_SOURCE_INGEST_PIPELINE_ELEM = "ImageAnalysisStageOne"; //NON-NLS
private static final String STAGE_TWO_DATA_SOURCE_INGEST_PIPELINE_ELEM = "ImageAnalysisStageTwo"; //NON-NLS
private static final String FILE_INGEST_PIPELINE_TYPE = "FileAnalysis"; //NON-NLS
private static final String XML_MODULE_ELEM = "MODULE"; //NON-NLS
private static final String INGEST_MODULE_ELEM = "MODULE"; //NON-NLS
private static final String XML_MODULE_CLASS_NAME_ATTR = "location"; //NON-NLS
private static IngestPipelinesConfiguration instance;
private final List<String> dataSourceIngestPipelineConfig = new ArrayList<>();
private final List<String> stageOneDataSourceIngestPipelineConfig = new ArrayList<>();
private final List<String> fileIngestPipelineConfig = new ArrayList<>();
private final List<String> stageTwoDataSourceIngestPipelineConfig = new ArrayList<>();
private IngestPipelinesConfiguration() {
readPipelinesConfigurationFile();
}
// RJCTODO: Bring this code back into use, use it in IngestJob to sort things
// into the now three pipelines. Other NBMs built on top of Autopsy that
// have custom pipeline config files can do a PlatformUtil.extractResourceToUserConfigDir()
// before this is called.
/**
* Gets the ingest pipelines configuration singleton.
*
* @return The singleton.
*/
synchronized static IngestPipelinesConfiguration getInstance() {
if (instance == null) {
Logger.getLogger(IngestPipelinesConfiguration.class.getName()).log(Level.INFO, "Creating ingest module loader instance"); //NON-NLS
@ -67,57 +71,90 @@ final class IngestPipelinesConfiguration {
return instance;
}
List<String> getDataSourceIngestPipelineConfig() {
return new ArrayList<>(dataSourceIngestPipelineConfig);
/**
* Constructs an object that provides data source and file ingest pipeline
* configurations as ordered lists of ingest module factory class names.
*/
private IngestPipelinesConfiguration() {
this.readPipelinesConfigurationFile();
}
/**
* Gets the ordered list of ingest module factory class names for the
* file ingest pipeline.
*
* @return An ordered list of ingest module factory class names.
*/
List<String> getStageOneDataSourceIngestPipelineConfig() {
return new ArrayList<>(stageOneDataSourceIngestPipelineConfig);
}
/**
* Gets the ordered list of ingest module factory class names for the
* first stage data source ingest pipeline.
*
* @return An ordered list of ingest module factory class names.
*/
List<String> getFileIngestPipelineConfig() {
return new ArrayList<>(fileIngestPipelineConfig);
}
/**
* Gets the ordered list of ingest module factory class names for the
* second stage data source ingest pipeline.
*
* @return An ordered list of ingest module factory class names.
*/
List<String> getStageTwoDataSourceIngestPipelineConfig() {
return new ArrayList<>(stageTwoDataSourceIngestPipelineConfig);
}
/**
* Attempts to read the ingest pipeline configuration data from an XML file.
*/
private void readPipelinesConfigurationFile() {
try {
boolean overWrite;
if (!ModuleSettings.settingExists(this.getClass().getSimpleName(), PIPELINE_CONFIG_FILE_VERSION_KEY)) {
ModuleSettings.setConfigSetting(this.getClass().getSimpleName(), PIPELINE_CONFIG_FILE_VERSION_KEY, PIPELINE_CONFIG_FILE_VERSION_NO_STRING);
overWrite = true;
} else {
int versionNumber = Integer.parseInt(ModuleSettings.getConfigSetting(this.getClass().getSimpleName(), PIPELINE_CONFIG_FILE_VERSION_KEY));
overWrite = versionNumber < PIPELINE_CONFIG_FILE_VERSION_NO;
// TODO: Migrate user edits
}
PlatformUtil.extractResourceToUserConfigDir(IngestPipelinesConfiguration.class, PIPELINES_CONFIG_FILE, overWrite);
PlatformUtil.extractResourceToUserConfigDir(IngestPipelinesConfiguration.class, PIPELINES_CONFIG_FILE, false);
String configFilePath = PlatformUtil.getUserConfigDirectory() + File.separator + PIPELINES_CONFIG_FILE;
Document doc = XMLUtil.loadDoc(IngestPipelinesConfiguration.class, configFilePath);
Path configFilePath = Paths.get(PlatformUtil.getUserConfigDirectory(), PIPELINES_CONFIG_FILE);
Document doc = XMLUtil.loadDoc(IngestPipelinesConfiguration.class, configFilePath.toAbsolutePath().toString());
if (doc == null) {
return;
}
// Get the document root element.
Element rootElement = doc.getDocumentElement();
if (rootElement == null) {
if (null == rootElement) {
logger.log(Level.SEVERE, "Invalid pipelines config file"); //NON-NLS
return;
}
NodeList pipelineElements = rootElement.getElementsByTagName(XML_PIPELINE_ELEM);
// Get the pipeline elements and confirm that the correct number is
// present.
NodeList pipelineElements = rootElement.getElementsByTagName(IngestPipelinesConfiguration.PIPELINE_ELEM);
int numPipelines = pipelineElements.getLength();
if (numPipelines < 1 || numPipelines > 2) {
if (numPipelines != IngestPipelinesConfiguration.NUMBER_OF_PIPELINE_DEFINITIONS) {
logger.log(Level.SEVERE, "Invalid pipelines config file"); //NON-NLS
return;
}
// Parse the pipeline elements to populate the pipeline
// configuration lists.
// RJCTODO: SHould check that each element is unique. Or could try the XSD bit.
List<String> pipelineConfig = null;
for (int pipelineNum = 0; pipelineNum < numPipelines; ++pipelineNum) {
Element pipelineElement = (Element) pipelineElements.item(pipelineNum);
String pipelineTypeAttr = pipelineElement.getAttribute(XML_PIPELINE_TYPE_ATTR);
if (pipelineTypeAttr != null) {
String pipelineTypeAttr = pipelineElement.getAttribute(PIPELINE_TYPE_ATTR);
if (null != pipelineTypeAttr) {
switch (pipelineTypeAttr) {
case DATA_SOURCE_INGEST_PIPELINE_TYPE:
pipelineConfig = dataSourceIngestPipelineConfig;
case STAGE_ONE_DATA_SOURCE_INGEST_PIPELINE_ELEM:
pipelineConfig = this.stageOneDataSourceIngestPipelineConfig;
break;
case FILE_INGEST_PIPELINE_TYPE:
pipelineConfig = fileIngestPipelineConfig;
pipelineConfig = this.fileIngestPipelineConfig;
break;
case STAGE_TWO_DATA_SOURCE_INGEST_PIPELINE_ELEM:
pipelineConfig = this.stageTwoDataSourceIngestPipelineConfig;
break;
default:
logger.log(Level.SEVERE, "Invalid pipelines config file"); //NON-NLS
@ -128,16 +165,13 @@ final class IngestPipelinesConfiguration {
// Create an ordered list of class names. The sequence of class
// names defines the sequence of modules in the pipeline.
if (pipelineConfig != null) {
NodeList modulesElems = pipelineElement.getElementsByTagName(XML_MODULE_ELEM);
NodeList modulesElems = pipelineElement.getElementsByTagName(INGEST_MODULE_ELEM);
int numModules = modulesElems.getLength();
if (numModules == 0) {
break;
}
for (int moduleNum = 0; moduleNum < numModules; ++moduleNum) {
Element moduleElement = (Element) modulesElems.item(moduleNum);
final String moduleClassName = moduleElement.getAttribute(XML_MODULE_CLASS_NAME_ATTR);
if (moduleClassName != null) {
pipelineConfig.add(moduleClassName);
String className = moduleElement.getTextContent();
if (null != className && !className.isEmpty()) {
pipelineConfig.add(className);
}
}
}

25
Core/src/org/sleuthkit/autopsy/ingest/PipelineConfig.xml Normal file
View File

@ -0,0 +1,25 @@
<?xml version="1.0" encoding="utf-8"?>
<!-- Default initial pipeline_config.xml
Contains only the core ingest modules that ship with Autopsy -->
<PIPELINE_CONFIG>
<PIPELINE type="ImageAnalysisStageOne">
<MODULE>org.sleuthkit.autopsy.recentactivity.RecentActivityExtracterModuleFactory</MODULE>
<MODULE>org.sleuthkit.autopsy.modules.android.AndroidModuleFactory</MODULE>
</PIPELINE>
<PIPELINE type="FileAnalysis">
<MODULE>org.sleuthkit.autopsy.modules.hashdatabase.HashLookupModuleFactory</MODULE>
<MODULE>org.sleuthkit.autopsy.modules.filetypeid.FileTypeIdModuleFactory</MODULE>
<MODULE>org.sleuthkit.autopsy.modules.sevenzip.ArchiveFileExtractorModuleFactory</MODULE>
<MODULE>org.sleuthkit.autopsy.modules.exif.ExifParserModuleFactory</MODULE>
<MODULE>org.sleuthkit.autopsy.keywordsearch.KeywordSearchModuleFactory</MODULE>
<MODULE>org.sleuthkit.autopsy.thunderbirdparser.EmailParserModuleFactory</MODULE>
<MODULE>org.sleuthkit.autopsy.modules.fileextmismatch.FileExtMismatchDetectorModuleFactory</MODULE>
<MODULE>org.sleuthkit.autopsy.modules.interestingitems.InterestingItemsIngestModuleFactory</MODULE>
</PIPELINE>
<PIPELINE type="ImageAnalysisStageTwo">
<MODULE>org.sleuthkit.autopsy.modules.e01verify.E01VerifierModuleFactory</MODULE>
</PIPELINE>
</PIPELINE_CONFIG>

18
Core/src/org/sleuthkit/autopsy/ingest/pipeline_config.xml
View File

@ -1,18 +0,0 @@
<?xml version="1.0" encoding="utf-8"?>
<!-- Default initial pipeline_config.xml
Contains only the core ingest modules that ship with Autopsy -->
<PIPELINE_CONFIG>
<PIPELINE type="FileAnalysis">
<MODULE order="1" type="plugin" location="org.sleuthkit.autopsy.hashdatabase.HashDbIngestModule" arguments="" />
<MODULE order="2" type="plugin" location="org.sleuthkit.autopsy.modules.filetypeid.FileTypeIdIngestModule" arguments=""/>
<MODULE order="3" type="plugin" location="org.sleuthkit.autopsy.modules.sevenzip.SevenZipIngestModule" arguments="" />
<MODULE order="4" type="plugin" location="org.sleuthkit.autopsy.modules.exifparser.ExifParserFileIngestModule"/>
<MODULE order="5" type="plugin" location="org.sleuthkit.autopsy.keywordsearch.KeywordSearchIngestModule"/>
<MODULE order="6" type="plugin" location="org.sleuthkit.autopsy.thunderbirdparser.ThunderbirdMboxFileIngestModule" arguments=""/>
<MODULE order="7" type="plugin" location="org.sleuthkit.autopsy.modules.fileextmismatch.FileExtMismatchIngestModule" arguments=""/>
</PIPELINE>
<PIPELINE type="ImageAnalysis">
<MODULE order="1" type="plugin" location="org.sleuthkit.autopsy.recentactivity.RAImageIngestModule" arguments=""/>
</PIPELINE>
</PIPELINE_CONFIG>