mirror of
https://github.com/overcuriousity/autopsy-flatpak.git
synced 2025-07-15 09:17:42 +00:00
Complete most of the multi-stage pipeline feature
This commit is contained in:
parent
90a103455b
commit
b6ca38fef4
@ -43,7 +43,7 @@ final class DataSourceIngestPipeline {
|
|||||||
DataSourceIngestModuleDecorator module = new DataSourceIngestModuleDecorator(template.createDataSourceIngestModule(), template.getModuleName());
|
DataSourceIngestModuleDecorator module = new DataSourceIngestModuleDecorator(template.createDataSourceIngestModule(), template.getModuleName());
|
||||||
modules.add(module);
|
modules.add(module);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
boolean isEmpty() {
|
boolean isEmpty() {
|
||||||
|
@ -20,7 +20,9 @@ package org.sleuthkit.autopsy.ingest;
|
|||||||
|
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.Date;
|
import java.util.Date;
|
||||||
|
import java.util.HashMap;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
import java.util.concurrent.ConcurrentHashMap;
|
import java.util.concurrent.ConcurrentHashMap;
|
||||||
import java.util.concurrent.LinkedBlockingQueue;
|
import java.util.concurrent.LinkedBlockingQueue;
|
||||||
import java.util.concurrent.atomic.AtomicLong;
|
import java.util.concurrent.atomic.AtomicLong;
|
||||||
@ -114,7 +116,7 @@ final class IngestJob {
|
|||||||
long jobId = nextIngestJobId.incrementAndGet();
|
long jobId = nextIngestJobId.incrementAndGet();
|
||||||
IngestJob job = new IngestJob(jobId, dataSource, processUnallocatedSpace);
|
IngestJob job = new IngestJob(jobId, dataSource, processUnallocatedSpace);
|
||||||
errors = job.start(ingestModuleTemplates);
|
errors = job.start(ingestModuleTemplates);
|
||||||
if (errors.isEmpty() && (job.hasDataSourceIngestPipeline() || job.hasFileIngestPipeline())) {
|
if (errors.isEmpty() && (job.hasDataSourceIngestPipeline() || job.hasFileIngestPipeline())) { // RJCTODO: What about 2nd stage only?
|
||||||
ingestJobsById.put(jobId, job);
|
ingestJobsById.put(jobId, job);
|
||||||
IngestManager.getInstance().fireIngestJobStarted(jobId);
|
IngestManager.getInstance().fireIngestJobStarted(jobId);
|
||||||
IngestJob.ingestScheduler.scheduleIngestTasks(job);
|
IngestJob.ingestScheduler.scheduleIngestTasks(job);
|
||||||
@ -135,7 +137,8 @@ final class IngestJob {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* RJCTODO
|
* RJCTODO
|
||||||
* @return
|
*
|
||||||
|
* @return
|
||||||
*/
|
*/
|
||||||
static List<IngestJobSnapshot> getJobSnapshots() {
|
static List<IngestJobSnapshot> getJobSnapshots() {
|
||||||
List<IngestJobSnapshot> snapShots = new ArrayList<>();
|
List<IngestJobSnapshot> snapShots = new ArrayList<>();
|
||||||
@ -144,7 +147,7 @@ final class IngestJob {
|
|||||||
}
|
}
|
||||||
return snapShots;
|
return snapShots;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* RJCTODO
|
* RJCTODO
|
||||||
*/
|
*/
|
||||||
@ -555,20 +558,38 @@ final class IngestJob {
|
|||||||
* @throws InterruptedException
|
* @throws InterruptedException
|
||||||
*/
|
*/
|
||||||
private void createIngestPipelines(List<IngestModuleTemplate> ingestModuleTemplates) throws InterruptedException {
|
private void createIngestPipelines(List<IngestModuleTemplate> ingestModuleTemplates) throws InterruptedException {
|
||||||
// RJCTODO: Use config file
|
// RJCTODO: Improve variable names!
|
||||||
// Sort the ingest module templates as required for the pipelines.
|
|
||||||
List<IngestModuleTemplate> firstStageDataSourceModuleTemplates = new ArrayList<>();
|
// Make mappings of ingest module factory class names to templates.
|
||||||
List<IngestModuleTemplate> secondStageDataSourceModuleTemplates = new ArrayList<>();
|
Map<String, IngestModuleTemplate> dataSourceModuleTemplates = new HashMap<>();
|
||||||
List<IngestModuleTemplate> fileIngestModuleTemplates = new ArrayList<>();
|
Map<String, IngestModuleTemplate> fileModuleTemplates = new HashMap<>();
|
||||||
for (IngestModuleTemplate template : ingestModuleTemplates) {
|
for (IngestModuleTemplate template : ingestModuleTemplates) {
|
||||||
if (template.isDataSourceIngestModuleTemplate()) {
|
if (template.isDataSourceIngestModuleTemplate()) {
|
||||||
firstStageDataSourceModuleTemplates.add(template);
|
dataSourceModuleTemplates.put(template.getModuleFactory().getClass().getCanonicalName(), template);
|
||||||
} else {
|
}
|
||||||
firstStageDataSourceModuleTemplates.add(template);
|
if (template.isFileIngestModuleTemplate()) {
|
||||||
|
fileModuleTemplates.put(template.getModuleFactory().getClass().getCanonicalName(), template);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Contruct the pipelines.
|
// Use the mappings and the ingest pipelines configuration to create
|
||||||
|
// ordered lists of ingest module templates for each ingest pipeline.
|
||||||
|
IngestPipelinesConfiguration pipelineConfigs = IngestPipelinesConfiguration.getInstance();
|
||||||
|
List<IngestModuleTemplate> firstStageDataSourceModuleTemplates = this.getConfiguredIngestModuleTemplates(dataSourceModuleTemplates, pipelineConfigs.getStageOneDataSourceIngestPipelineConfig());
|
||||||
|
List<IngestModuleTemplate> fileIngestModuleTemplates = this.getConfiguredIngestModuleTemplates(fileModuleTemplates, pipelineConfigs.getFileIngestPipelineConfig());
|
||||||
|
List<IngestModuleTemplate> secondStageDataSourceModuleTemplates = this.getConfiguredIngestModuleTemplates(dataSourceModuleTemplates, pipelineConfigs.getStageTwoDataSourceIngestPipelineConfig());
|
||||||
|
|
||||||
|
// Add any module templates that were not specified in the pipeline
|
||||||
|
// configurations to an appropriate pipeline - either the first stage
|
||||||
|
// data source ingest pipeline or the file ingest pipeline.
|
||||||
|
for (IngestModuleTemplate template : dataSourceModuleTemplates.values()) {
|
||||||
|
firstStageDataSourceModuleTemplates.add(template);
|
||||||
|
}
|
||||||
|
for (IngestModuleTemplate template : fileModuleTemplates.values()) {
|
||||||
|
fileIngestModuleTemplates.add(template);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Contruct the data source ingest pipelines.
|
||||||
this.firstStageDataSourceIngestPipeline = new DataSourceIngestPipeline(this, firstStageDataSourceModuleTemplates);
|
this.firstStageDataSourceIngestPipeline = new DataSourceIngestPipeline(this, firstStageDataSourceModuleTemplates);
|
||||||
this.secondStageDataSourceIngestPipeline = new DataSourceIngestPipeline(this, secondStageDataSourceModuleTemplates);
|
this.secondStageDataSourceIngestPipeline = new DataSourceIngestPipeline(this, secondStageDataSourceModuleTemplates);
|
||||||
this.dataSourceIngestPipeline = firstStageDataSourceIngestPipeline;
|
this.dataSourceIngestPipeline = firstStageDataSourceIngestPipeline;
|
||||||
@ -580,6 +601,28 @@ final class IngestJob {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Use an ordered list of ingest module factory class names to create an
|
||||||
|
* ordered subset of a collection ingest module templates. The ingest module
|
||||||
|
* templates are removed from the input collection as they are added to the
|
||||||
|
* output collection.
|
||||||
|
*
|
||||||
|
* @param ingestModuleTemplates A mapping of ingest module factory class
|
||||||
|
* names to ingest module templates.
|
||||||
|
* @param pipelineConfig An ordered list of ingest module factory class
|
||||||
|
* names representing an ingest pipeline.
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
List<IngestModuleTemplate> getConfiguredIngestModuleTemplates(Map<String, IngestModuleTemplate> ingestModuleTemplates, List<String> pipelineConfig) {
|
||||||
|
List<IngestModuleTemplate> templates = new ArrayList<>();
|
||||||
|
for (String moduleClassName : pipelineConfig) {
|
||||||
|
if (ingestModuleTemplates.containsKey(moduleClassName)) {
|
||||||
|
templates.add(ingestModuleTemplates.remove(moduleClassName));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return templates;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Starts up each of the file and data source ingest modules to collect
|
* Starts up each of the file and data source ingest modules to collect
|
||||||
* possible errors.
|
* possible errors.
|
||||||
@ -748,7 +791,7 @@ final class IngestJob {
|
|||||||
this.dataSourceIngestProgress = null;
|
this.dataSourceIngestProgress = null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
IngestJob.ingestJobsById.remove(this.id);
|
IngestJob.ingestJobsById.remove(this.id);
|
||||||
if (!this.isCancelled()) {
|
if (!this.isCancelled()) {
|
||||||
logger.log(Level.INFO, "Ingest job {0} completed", this.id);
|
logger.log(Level.INFO, "Ingest job {0} completed", this.id);
|
||||||
@ -784,7 +827,7 @@ final class IngestJob {
|
|||||||
class IngestJobSnapshot {
|
class IngestJobSnapshot {
|
||||||
|
|
||||||
private final long jobId;
|
private final long jobId;
|
||||||
private final String dataSource;
|
private final String dataSource;
|
||||||
private final long startTime;
|
private final long startTime;
|
||||||
private final long processedFiles;
|
private final long processedFiles;
|
||||||
private final long estimatedFilesToProcess;
|
private final long estimatedFilesToProcess;
|
||||||
@ -809,7 +852,8 @@ final class IngestJob {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* RJCTODO
|
* RJCTODO
|
||||||
* @return
|
*
|
||||||
|
* @return
|
||||||
*/
|
*/
|
||||||
long getJobId() {
|
long getJobId() {
|
||||||
return this.jobId;
|
return this.jobId;
|
||||||
@ -817,12 +861,13 @@ final class IngestJob {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* RJCTODO
|
* RJCTODO
|
||||||
* @return
|
*
|
||||||
|
* @return
|
||||||
*/
|
*/
|
||||||
String getDataSource() {
|
String getDataSource() {
|
||||||
return dataSource;
|
return dataSource;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Gets files per second throughput since job started.
|
* Gets files per second throughput since job started.
|
||||||
*
|
*
|
||||||
@ -870,10 +915,11 @@ final class IngestJob {
|
|||||||
long getFilesEstimated() {
|
long getFilesEstimated() {
|
||||||
return estimatedFilesToProcess;
|
return estimatedFilesToProcess;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* RJCTODO
|
* RJCTODO
|
||||||
* @return
|
*
|
||||||
|
* @return
|
||||||
*/
|
*/
|
||||||
long getRootQueueSize() {
|
long getRootQueueSize() {
|
||||||
return this.tasksSnapshot.getRootQueueSize();
|
return this.tasksSnapshot.getRootQueueSize();
|
||||||
@ -881,7 +927,8 @@ final class IngestJob {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* RJCTODO
|
* RJCTODO
|
||||||
* @return
|
*
|
||||||
|
* @return
|
||||||
*/
|
*/
|
||||||
long getDirQueueSize() {
|
long getDirQueueSize() {
|
||||||
return this.tasksSnapshot.getDirQueueSize();
|
return this.tasksSnapshot.getDirQueueSize();
|
||||||
@ -889,7 +936,8 @@ final class IngestJob {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* RJCTODO
|
* RJCTODO
|
||||||
* @return
|
*
|
||||||
|
* @return
|
||||||
*/
|
*/
|
||||||
long getFileQueueSize() {
|
long getFileQueueSize() {
|
||||||
return this.tasksSnapshot.getFileQueueSize();
|
return this.tasksSnapshot.getFileQueueSize();
|
||||||
@ -897,7 +945,8 @@ final class IngestJob {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* RJCTODO
|
* RJCTODO
|
||||||
* @return
|
*
|
||||||
|
* @return
|
||||||
*/
|
*/
|
||||||
long getDsQueueSize() {
|
long getDsQueueSize() {
|
||||||
return this.tasksSnapshot.getDsQueueSize();
|
return this.tasksSnapshot.getDsQueueSize();
|
||||||
@ -905,12 +954,13 @@ final class IngestJob {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* RJCTODO
|
* RJCTODO
|
||||||
* @return
|
*
|
||||||
|
* @return
|
||||||
*/
|
*/
|
||||||
long getRunningListSize() {
|
long getRunningListSize() {
|
||||||
return this.tasksSnapshot.getRunningListSize();
|
return this.tasksSnapshot.getRunningListSize();
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -51,6 +51,8 @@ final class IngestModuleFactoryLoader {
|
|||||||
private static final String SAMPLE_EXECUTABLE_MODULE_FACTORY_CLASS_NAME = SampleExecutableIngestModuleFactory.class.getCanonicalName();
|
private static final String SAMPLE_EXECUTABLE_MODULE_FACTORY_CLASS_NAME = SampleExecutableIngestModuleFactory.class.getCanonicalName();
|
||||||
private static final ArrayList<String> coreModuleOrdering = new ArrayList<String>() {
|
private static final ArrayList<String> coreModuleOrdering = new ArrayList<String>() {
|
||||||
{
|
{
|
||||||
|
// RJCTODO: Find out wherer ot put the photorec carver
|
||||||
|
|
||||||
// The ordering of the core ingest module factories implemented
|
// The ordering of the core ingest module factories implemented
|
||||||
// using Java is hard-coded.
|
// using Java is hard-coded.
|
||||||
add("org.sleuthkit.autopsy.recentactivity.RecentActivityExtracterModuleFactory"); //NON-NLS
|
add("org.sleuthkit.autopsy.recentactivity.RecentActivityExtracterModuleFactory"); //NON-NLS
|
||||||
|
@ -18,13 +18,13 @@
|
|||||||
*/
|
*/
|
||||||
package org.sleuthkit.autopsy.ingest;
|
package org.sleuthkit.autopsy.ingest;
|
||||||
|
|
||||||
import java.io.File;
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
import java.nio.file.Path;
|
||||||
|
import java.nio.file.Paths;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.logging.Level;
|
import java.util.logging.Level;
|
||||||
import org.sleuthkit.autopsy.coreutils.Logger;
|
import org.sleuthkit.autopsy.coreutils.Logger;
|
||||||
import org.sleuthkit.autopsy.coreutils.ModuleSettings;
|
|
||||||
import org.sleuthkit.autopsy.coreutils.PlatformUtil;
|
import org.sleuthkit.autopsy.coreutils.PlatformUtil;
|
||||||
import org.sleuthkit.autopsy.coreutils.XMLUtil;
|
import org.sleuthkit.autopsy.coreutils.XMLUtil;
|
||||||
import org.w3c.dom.Document;
|
import org.w3c.dom.Document;
|
||||||
@ -33,32 +33,36 @@ import org.w3c.dom.NodeList;
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Provides data source and file ingest pipeline configurations as ordered lists
|
* Provides data source and file ingest pipeline configurations as ordered lists
|
||||||
* of ingest module class names. The order of the module class names indicates
|
* of ingest module factory class names.
|
||||||
* the desired sequence of ingest module instances in an ingest modules
|
|
||||||
* pipeline.
|
|
||||||
*/
|
*/
|
||||||
final class IngestPipelinesConfiguration {
|
final class IngestPipelinesConfiguration {
|
||||||
|
|
||||||
private static final Logger logger = Logger.getLogger(IngestPipelinesConfiguration.class.getName());
|
private static final Logger logger = Logger.getLogger(IngestPipelinesConfiguration.class.getName());
|
||||||
private static final String PIPELINE_CONFIG_FILE_VERSION_KEY = "PipelineConfigFileVersion"; //NON-NLS
|
private static final String PIPELINES_CONFIG_FILE = "PipelineConfig.xml"; //NON-NLS
|
||||||
private static final String PIPELINE_CONFIG_FILE_VERSION_NO_STRING = "1";
|
private static final String PIPELINE_ELEM = "PIPELINE"; //NON-NLS
|
||||||
private static final int PIPELINE_CONFIG_FILE_VERSION_NO = 1;
|
private static final int NUMBER_OF_PIPELINE_DEFINITIONS = 3;
|
||||||
private static final String PIPELINES_CONFIG_FILE = "pipeline_config.xml"; //NON-NLS
|
private static final String PIPELINE_TYPE_ATTR = "type"; //NON-NLS
|
||||||
private static final String PIPELINES_CONFIG_FILE_XSD = "PipelineConfigSchema.xsd"; //NON-NLS
|
private static final String STAGE_ONE_DATA_SOURCE_INGEST_PIPELINE_ELEM = "ImageAnalysisStageOne"; //NON-NLS
|
||||||
private static final String XML_PIPELINE_ELEM = "PIPELINE"; //NON-NLS
|
private static final String STAGE_TWO_DATA_SOURCE_INGEST_PIPELINE_ELEM = "ImageAnalysisStageTwo"; //NON-NLS
|
||||||
private static final String XML_PIPELINE_TYPE_ATTR = "type"; //NON-NLS
|
|
||||||
private static final String DATA_SOURCE_INGEST_PIPELINE_TYPE = "ImageAnalysis"; //NON-NLS
|
|
||||||
private static final String FILE_INGEST_PIPELINE_TYPE = "FileAnalysis"; //NON-NLS
|
private static final String FILE_INGEST_PIPELINE_TYPE = "FileAnalysis"; //NON-NLS
|
||||||
private static final String XML_MODULE_ELEM = "MODULE"; //NON-NLS
|
private static final String INGEST_MODULE_ELEM = "MODULE"; //NON-NLS
|
||||||
private static final String XML_MODULE_CLASS_NAME_ATTR = "location"; //NON-NLS
|
private static final String XML_MODULE_CLASS_NAME_ATTR = "location"; //NON-NLS
|
||||||
|
|
||||||
private static IngestPipelinesConfiguration instance;
|
private static IngestPipelinesConfiguration instance;
|
||||||
private final List<String> dataSourceIngestPipelineConfig = new ArrayList<>();
|
|
||||||
|
private final List<String> stageOneDataSourceIngestPipelineConfig = new ArrayList<>();
|
||||||
private final List<String> fileIngestPipelineConfig = new ArrayList<>();
|
private final List<String> fileIngestPipelineConfig = new ArrayList<>();
|
||||||
|
private final List<String> stageTwoDataSourceIngestPipelineConfig = new ArrayList<>();
|
||||||
|
|
||||||
private IngestPipelinesConfiguration() {
|
// RJCTODO: Bring this code back into use, use it in IngestJob to sort things
|
||||||
readPipelinesConfigurationFile();
|
// into the now three pipelines. Other NBMs built on top of Autopsy that
|
||||||
}
|
// have custom pipeline config files can do a PlatformUtil.extractResourceToUserConfigDir()
|
||||||
|
// before this is called.
|
||||||
|
/**
|
||||||
|
* Gets the ingest pipelines configuration singleton.
|
||||||
|
*
|
||||||
|
* @return The singleton.
|
||||||
|
*/
|
||||||
synchronized static IngestPipelinesConfiguration getInstance() {
|
synchronized static IngestPipelinesConfiguration getInstance() {
|
||||||
if (instance == null) {
|
if (instance == null) {
|
||||||
Logger.getLogger(IngestPipelinesConfiguration.class.getName()).log(Level.INFO, "Creating ingest module loader instance"); //NON-NLS
|
Logger.getLogger(IngestPipelinesConfiguration.class.getName()).log(Level.INFO, "Creating ingest module loader instance"); //NON-NLS
|
||||||
@ -67,57 +71,90 @@ final class IngestPipelinesConfiguration {
|
|||||||
return instance;
|
return instance;
|
||||||
}
|
}
|
||||||
|
|
||||||
List<String> getDataSourceIngestPipelineConfig() {
|
/**
|
||||||
return new ArrayList<>(dataSourceIngestPipelineConfig);
|
* Constructs an object that provides data source and file ingest pipeline
|
||||||
|
* configurations as ordered lists of ingest module factory class names.
|
||||||
|
*/
|
||||||
|
private IngestPipelinesConfiguration() {
|
||||||
|
this.readPipelinesConfigurationFile();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Gets the ordered list of ingest module factory class names for the
|
||||||
|
* file ingest pipeline.
|
||||||
|
*
|
||||||
|
* @return An ordered list of ingest module factory class names.
|
||||||
|
*/
|
||||||
|
List<String> getStageOneDataSourceIngestPipelineConfig() {
|
||||||
|
return new ArrayList<>(stageOneDataSourceIngestPipelineConfig);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Gets the ordered list of ingest module factory class names for the
|
||||||
|
* first stage data source ingest pipeline.
|
||||||
|
*
|
||||||
|
* @return An ordered list of ingest module factory class names.
|
||||||
|
*/
|
||||||
List<String> getFileIngestPipelineConfig() {
|
List<String> getFileIngestPipelineConfig() {
|
||||||
return new ArrayList<>(fileIngestPipelineConfig);
|
return new ArrayList<>(fileIngestPipelineConfig);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Gets the ordered list of ingest module factory class names for the
|
||||||
|
* second stage data source ingest pipeline.
|
||||||
|
*
|
||||||
|
* @return An ordered list of ingest module factory class names.
|
||||||
|
*/
|
||||||
|
List<String> getStageTwoDataSourceIngestPipelineConfig() {
|
||||||
|
return new ArrayList<>(stageTwoDataSourceIngestPipelineConfig);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Attempts to read the ingest pipeline configuration data from an XML file.
|
||||||
|
*/
|
||||||
private void readPipelinesConfigurationFile() {
|
private void readPipelinesConfigurationFile() {
|
||||||
try {
|
try {
|
||||||
boolean overWrite;
|
PlatformUtil.extractResourceToUserConfigDir(IngestPipelinesConfiguration.class, PIPELINES_CONFIG_FILE, false);
|
||||||
if (!ModuleSettings.settingExists(this.getClass().getSimpleName(), PIPELINE_CONFIG_FILE_VERSION_KEY)) {
|
|
||||||
ModuleSettings.setConfigSetting(this.getClass().getSimpleName(), PIPELINE_CONFIG_FILE_VERSION_KEY, PIPELINE_CONFIG_FILE_VERSION_NO_STRING);
|
|
||||||
overWrite = true;
|
|
||||||
} else {
|
|
||||||
int versionNumber = Integer.parseInt(ModuleSettings.getConfigSetting(this.getClass().getSimpleName(), PIPELINE_CONFIG_FILE_VERSION_KEY));
|
|
||||||
overWrite = versionNumber < PIPELINE_CONFIG_FILE_VERSION_NO;
|
|
||||||
// TODO: Migrate user edits
|
|
||||||
}
|
|
||||||
PlatformUtil.extractResourceToUserConfigDir(IngestPipelinesConfiguration.class, PIPELINES_CONFIG_FILE, overWrite);
|
|
||||||
|
|
||||||
String configFilePath = PlatformUtil.getUserConfigDirectory() + File.separator + PIPELINES_CONFIG_FILE;
|
Path configFilePath = Paths.get(PlatformUtil.getUserConfigDirectory(), PIPELINES_CONFIG_FILE);
|
||||||
Document doc = XMLUtil.loadDoc(IngestPipelinesConfiguration.class, configFilePath);
|
Document doc = XMLUtil.loadDoc(IngestPipelinesConfiguration.class, configFilePath.toAbsolutePath().toString());
|
||||||
if (doc == null) {
|
if (doc == null) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Get the document root element.
|
||||||
Element rootElement = doc.getDocumentElement();
|
Element rootElement = doc.getDocumentElement();
|
||||||
if (rootElement == null) {
|
if (null == rootElement) {
|
||||||
logger.log(Level.SEVERE, "Invalid pipelines config file"); //NON-NLS
|
logger.log(Level.SEVERE, "Invalid pipelines config file"); //NON-NLS
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
NodeList pipelineElements = rootElement.getElementsByTagName(XML_PIPELINE_ELEM);
|
// Get the pipeline elements and confirm that the correct number is
|
||||||
|
// present.
|
||||||
|
NodeList pipelineElements = rootElement.getElementsByTagName(IngestPipelinesConfiguration.PIPELINE_ELEM);
|
||||||
int numPipelines = pipelineElements.getLength();
|
int numPipelines = pipelineElements.getLength();
|
||||||
if (numPipelines < 1 || numPipelines > 2) {
|
if (numPipelines != IngestPipelinesConfiguration.NUMBER_OF_PIPELINE_DEFINITIONS) {
|
||||||
logger.log(Level.SEVERE, "Invalid pipelines config file"); //NON-NLS
|
logger.log(Level.SEVERE, "Invalid pipelines config file"); //NON-NLS
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Parse the pipeline elements to populate the pipeline
|
||||||
|
// configuration lists.
|
||||||
|
// RJCTODO: SHould check that each element is unique. Or could try the XSD bit.
|
||||||
List<String> pipelineConfig = null;
|
List<String> pipelineConfig = null;
|
||||||
for (int pipelineNum = 0; pipelineNum < numPipelines; ++pipelineNum) {
|
for (int pipelineNum = 0; pipelineNum < numPipelines; ++pipelineNum) {
|
||||||
Element pipelineElement = (Element) pipelineElements.item(pipelineNum);
|
Element pipelineElement = (Element) pipelineElements.item(pipelineNum);
|
||||||
String pipelineTypeAttr = pipelineElement.getAttribute(XML_PIPELINE_TYPE_ATTR);
|
String pipelineTypeAttr = pipelineElement.getAttribute(PIPELINE_TYPE_ATTR);
|
||||||
if (pipelineTypeAttr != null) {
|
if (null != pipelineTypeAttr) {
|
||||||
switch (pipelineTypeAttr) {
|
switch (pipelineTypeAttr) {
|
||||||
case DATA_SOURCE_INGEST_PIPELINE_TYPE:
|
case STAGE_ONE_DATA_SOURCE_INGEST_PIPELINE_ELEM:
|
||||||
pipelineConfig = dataSourceIngestPipelineConfig;
|
pipelineConfig = this.stageOneDataSourceIngestPipelineConfig;
|
||||||
break;
|
break;
|
||||||
case FILE_INGEST_PIPELINE_TYPE:
|
case FILE_INGEST_PIPELINE_TYPE:
|
||||||
pipelineConfig = fileIngestPipelineConfig;
|
pipelineConfig = this.fileIngestPipelineConfig;
|
||||||
|
break;
|
||||||
|
case STAGE_TWO_DATA_SOURCE_INGEST_PIPELINE_ELEM:
|
||||||
|
pipelineConfig = this.stageTwoDataSourceIngestPipelineConfig;
|
||||||
break;
|
break;
|
||||||
default:
|
default:
|
||||||
logger.log(Level.SEVERE, "Invalid pipelines config file"); //NON-NLS
|
logger.log(Level.SEVERE, "Invalid pipelines config file"); //NON-NLS
|
||||||
@ -128,16 +165,13 @@ final class IngestPipelinesConfiguration {
|
|||||||
// Create an ordered list of class names. The sequence of class
|
// Create an ordered list of class names. The sequence of class
|
||||||
// names defines the sequence of modules in the pipeline.
|
// names defines the sequence of modules in the pipeline.
|
||||||
if (pipelineConfig != null) {
|
if (pipelineConfig != null) {
|
||||||
NodeList modulesElems = pipelineElement.getElementsByTagName(XML_MODULE_ELEM);
|
NodeList modulesElems = pipelineElement.getElementsByTagName(INGEST_MODULE_ELEM);
|
||||||
int numModules = modulesElems.getLength();
|
int numModules = modulesElems.getLength();
|
||||||
if (numModules == 0) {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
for (int moduleNum = 0; moduleNum < numModules; ++moduleNum) {
|
for (int moduleNum = 0; moduleNum < numModules; ++moduleNum) {
|
||||||
Element moduleElement = (Element) modulesElems.item(moduleNum);
|
Element moduleElement = (Element) modulesElems.item(moduleNum);
|
||||||
final String moduleClassName = moduleElement.getAttribute(XML_MODULE_CLASS_NAME_ATTR);
|
String className = moduleElement.getTextContent();
|
||||||
if (moduleClassName != null) {
|
if (null != className && !className.isEmpty()) {
|
||||||
pipelineConfig.add(moduleClassName);
|
pipelineConfig.add(className);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
25
Core/src/org/sleuthkit/autopsy/ingest/PipelineConfig.xml
Normal file
25
Core/src/org/sleuthkit/autopsy/ingest/PipelineConfig.xml
Normal file
@ -0,0 +1,25 @@
|
|||||||
|
<?xml version="1.0" encoding="utf-8"?>
|
||||||
|
<!-- Default initial pipeline_config.xml
|
||||||
|
Contains only the core ingest modules that ship with Autopsy -->
|
||||||
|
<PIPELINE_CONFIG>
|
||||||
|
<PIPELINE type="ImageAnalysisStageOne">
|
||||||
|
<MODULE>org.sleuthkit.autopsy.recentactivity.RecentActivityExtracterModuleFactory</MODULE>
|
||||||
|
<MODULE>org.sleuthkit.autopsy.modules.android.AndroidModuleFactory</MODULE>
|
||||||
|
</PIPELINE>
|
||||||
|
|
||||||
|
<PIPELINE type="FileAnalysis">
|
||||||
|
<MODULE>org.sleuthkit.autopsy.modules.hashdatabase.HashLookupModuleFactory</MODULE>
|
||||||
|
<MODULE>org.sleuthkit.autopsy.modules.filetypeid.FileTypeIdModuleFactory</MODULE>
|
||||||
|
<MODULE>org.sleuthkit.autopsy.modules.sevenzip.ArchiveFileExtractorModuleFactory</MODULE>
|
||||||
|
<MODULE>org.sleuthkit.autopsy.modules.exif.ExifParserModuleFactory</MODULE>
|
||||||
|
<MODULE>org.sleuthkit.autopsy.keywordsearch.KeywordSearchModuleFactory</MODULE>
|
||||||
|
<MODULE>org.sleuthkit.autopsy.thunderbirdparser.EmailParserModuleFactory</MODULE>
|
||||||
|
<MODULE>org.sleuthkit.autopsy.modules.fileextmismatch.FileExtMismatchDetectorModuleFactory</MODULE>
|
||||||
|
<MODULE>org.sleuthkit.autopsy.modules.interestingitems.InterestingItemsIngestModuleFactory</MODULE>
|
||||||
|
</PIPELINE>
|
||||||
|
|
||||||
|
<PIPELINE type="ImageAnalysisStageTwo">
|
||||||
|
<MODULE>org.sleuthkit.autopsy.modules.e01verify.E01VerifierModuleFactory</MODULE>
|
||||||
|
</PIPELINE>
|
||||||
|
|
||||||
|
</PIPELINE_CONFIG>
|
@ -1,18 +0,0 @@
|
|||||||
<?xml version="1.0" encoding="utf-8"?>
|
|
||||||
<!-- Default initial pipeline_config.xml
|
|
||||||
Contains only the core ingest modules that ship with Autopsy -->
|
|
||||||
<PIPELINE_CONFIG>
|
|
||||||
<PIPELINE type="FileAnalysis">
|
|
||||||
<MODULE order="1" type="plugin" location="org.sleuthkit.autopsy.hashdatabase.HashDbIngestModule" arguments="" />
|
|
||||||
<MODULE order="2" type="plugin" location="org.sleuthkit.autopsy.modules.filetypeid.FileTypeIdIngestModule" arguments=""/>
|
|
||||||
<MODULE order="3" type="plugin" location="org.sleuthkit.autopsy.modules.sevenzip.SevenZipIngestModule" arguments="" />
|
|
||||||
<MODULE order="4" type="plugin" location="org.sleuthkit.autopsy.modules.exifparser.ExifParserFileIngestModule"/>
|
|
||||||
<MODULE order="5" type="plugin" location="org.sleuthkit.autopsy.keywordsearch.KeywordSearchIngestModule"/>
|
|
||||||
<MODULE order="6" type="plugin" location="org.sleuthkit.autopsy.thunderbirdparser.ThunderbirdMboxFileIngestModule" arguments=""/>
|
|
||||||
<MODULE order="7" type="plugin" location="org.sleuthkit.autopsy.modules.fileextmismatch.FileExtMismatchIngestModule" arguments=""/>
|
|
||||||
</PIPELINE>
|
|
||||||
|
|
||||||
<PIPELINE type="ImageAnalysis">
|
|
||||||
<MODULE order="1" type="plugin" location="org.sleuthkit.autopsy.recentactivity.RAImageIngestModule" arguments=""/>
|
|
||||||
</PIPELINE>
|
|
||||||
</PIPELINE_CONFIG>
|
|
Loading…
x
Reference in New Issue
Block a user