diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java index 74fcf0301b..970f94c9fc 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java @@ -29,6 +29,9 @@ import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.atomic.AtomicLong; import java.util.logging.Level; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Stream; import javax.swing.JOptionPane; import org.netbeans.api.progress.ProgressHandle; import org.openide.util.Cancellable; @@ -57,10 +60,14 @@ import org.sleuthkit.autopsy.python.FactoryClassNameNormalizer; * it. */ public final class DataSourceIngestJob { + private static String AUTOPSY_MODULE_PREFIX = "org.sleuthkit.autopsy"; - + private static final Logger logger = Logger.getLogger(DataSourceIngestJob.class.getName()); + // to match something like: "org.python.proxies.GPX_Parser_Module$GPXParserFileIngestModuleFactory$14" + private static final Pattern JYTHON_REGEX = Pattern.compile("org\\.python\\.proxies\\.(.+?)\\$(.+?)(\\$[0-9]*)?$"); + /** * These fields define a data source ingest job: the parent ingest job, an * ID, the user's ingest job settings, and the data source to be analyzed. @@ -216,31 +223,75 @@ public final class DataSourceIngestJob { this.createTime = new Date().getTime(); this.createIngestPipelines(); } - - - + /** - * Adds ingest modules to a list with autopsy modules first and third party modules next. - * @param dest The destination for the modules to be added. - * @param src A map of fully qualified class name mapped to the IngestModuleTemplate. + * Adds ingest modules to a list with autopsy modules first and third party + * modules next. + * + * @param dest The destination for the modules to be added. + * @param src A map of fully qualified class name mapped to the + * IngestModuleTemplate. + * @param jythonSrc A map of fully qualified class name mapped to the + * IngestModuleTemplate for jython modules. */ - private static void addOrdered(final List dest, final Map src) { + private static void addOrdered(final List dest, + final Map src, final Map jythonSrc) { + final List autopsyModules = new ArrayList<>(); final List thirdPartyModules = new ArrayList<>(); - - src.entrySet().stream().forEach((templateEntry) -> { + + Stream.concat(src.entrySet().stream(), jythonSrc.entrySet().stream()).forEach((templateEntry) -> { if (templateEntry.getKey().startsWith(AUTOPSY_MODULE_PREFIX)) { autopsyModules.add(templateEntry.getValue()); - } - else { + } else { thirdPartyModules.add(templateEntry.getValue()); } }); - + dest.addAll(autopsyModules); dest.addAll(thirdPartyModules); } + /** + * Takes a classname like + * "org.python.proxies.GPX_Parser_Module$GPXParserFileIngestModuleFactory$14" + * and provides "GPX_Parser_Module.GPXParserFileIngestModuleFactory" or null + * if not in jython package. + * + * @param canonicalName The canonical name. + * + * @return The jython name or null if not in jython package. + */ + private static String getJythonName(String canonicalName) { + Matcher m = JYTHON_REGEX.matcher(canonicalName); + if (m.find()) { + return String.format("%s.%s", m.group(1), m.group(2)); + } else { + return null; + } + } + + /** + * Adds a template to the appropriate map. If the class is a jython class, + * then it is added to the jython map. Otherwise, it is added to the + * mapping. + * + * @param mapping Mapping for non-jython objects. + * @param jythonMapping Mapping for jython objects. + * @param template The template to add. + */ + private static void addModule(Map mapping, + Map jythonMapping, IngestModuleTemplate template) { + + String className = template.getModuleFactory().getClass().getCanonicalName(); + String jythonName = getJythonName(className); + if (jythonName != null) { + jythonMapping.put(jythonName, template); + } else { + mapping.put(className, template); + } + } + /** * Creates the file and data source ingest pipelines. */ @@ -252,12 +303,18 @@ public final class DataSourceIngestJob { */ Map dataSourceModuleTemplates = new LinkedHashMap<>(); Map fileModuleTemplates = new LinkedHashMap<>(); + + // mappings for jython modules. These mappings are only used to determine modules in the pipelineconfig.xml. + + Map jythonDataSourceModuleTemplates = new LinkedHashMap<>(); + Map jythonFileModuleTemplates = new LinkedHashMap<>(); + for (IngestModuleTemplate template : ingestModuleTemplates) { if (template.isDataSourceIngestModuleTemplate()) { - dataSourceModuleTemplates.put(template.getModuleFactory().getClass().getCanonicalName(), template); + addModule(dataSourceModuleTemplates, jythonDataSourceModuleTemplates, template); } if (template.isFileIngestModuleTemplate()) { - fileModuleTemplates.put(template.getModuleFactory().getClass().getCanonicalName(), template); + addModule(fileModuleTemplates, jythonFileModuleTemplates, template); } } @@ -266,18 +323,23 @@ public final class DataSourceIngestJob { * ordered lists of ingest module templates for each ingest pipeline. */ IngestPipelinesConfiguration pipelineConfigs = IngestPipelinesConfiguration.getInstance(); - List firstStageDataSourceModuleTemplates = DataSourceIngestJob.getConfiguredIngestModuleTemplates(dataSourceModuleTemplates, pipelineConfigs.getStageOneDataSourceIngestPipelineConfig()); - List fileIngestModuleTemplates = DataSourceIngestJob.getConfiguredIngestModuleTemplates(fileModuleTemplates, pipelineConfigs.getFileIngestPipelineConfig()); - List secondStageDataSourceModuleTemplates = DataSourceIngestJob.getConfiguredIngestModuleTemplates(dataSourceModuleTemplates, pipelineConfigs.getStageTwoDataSourceIngestPipelineConfig()); + List firstStageDataSourceModuleTemplates = DataSourceIngestJob.getConfiguredIngestModuleTemplates( + dataSourceModuleTemplates, jythonDataSourceModuleTemplates, pipelineConfigs.getStageOneDataSourceIngestPipelineConfig()); + + List fileIngestModuleTemplates = DataSourceIngestJob.getConfiguredIngestModuleTemplates( + fileModuleTemplates, jythonFileModuleTemplates, pipelineConfigs.getFileIngestPipelineConfig()); + + List secondStageDataSourceModuleTemplates = DataSourceIngestJob.getConfiguredIngestModuleTemplates( + dataSourceModuleTemplates, null, pipelineConfigs.getStageTwoDataSourceIngestPipelineConfig()); /** * Add any module templates that were not specified in the pipelines * configuration to an appropriate pipeline - either the first stage * data source ingest pipeline or the file ingest pipeline. */ - addOrdered(firstStageDataSourceModuleTemplates, dataSourceModuleTemplates); - addOrdered(fileIngestModuleTemplates, fileModuleTemplates); - + addOrdered(firstStageDataSourceModuleTemplates, dataSourceModuleTemplates, jythonDataSourceModuleTemplates); + addOrdered(fileIngestModuleTemplates, fileModuleTemplates, jythonFileModuleTemplates); + /** * Construct the data source ingest pipelines. */ @@ -325,19 +387,27 @@ public final class DataSourceIngestJob { * ingest pipeline. The ingest module templates are removed from the input * collection as they are added to the output collection. * - * @param ingestModuleTemplates A mapping of ingest module factory class - * names to ingest module templates. - * @param pipelineConfig An ordered list of ingest module factory - * class names representing an ingest pipeline. + * @param ingestModuleTemplates A mapping of ingest module factory + * class names to ingest module + * templates. + * @param jythonIngestModuleTemplates A mapping of jython processed class + * names to jython ingest module + * templates. + * @param pipelineConfig An ordered list of ingest module + * factory class names representing an + * ingest pipeline. * * @return An ordered list of ingest module templates, i.e., an * uninstantiated pipeline. */ - private static List getConfiguredIngestModuleTemplates(Map ingestModuleTemplates, List pipelineConfig) { + private static List getConfiguredIngestModuleTemplates( + Map ingestModuleTemplates, Map jythonIngestModuleTemplates, List pipelineConfig) { List templates = new ArrayList<>(); for (String moduleClassName : pipelineConfig) { if (ingestModuleTemplates.containsKey(moduleClassName)) { templates.add(ingestModuleTemplates.remove(moduleClassName)); + } else if (jythonIngestModuleTemplates.containsKey(moduleClassName)) { + templates.add(jythonIngestModuleTemplates.remove(moduleClassName)); } } return templates; diff --git a/Core/src/org/sleuthkit/autopsy/ingest/PipelineConfig.xml b/Core/src/org/sleuthkit/autopsy/ingest/PipelineConfig.xml index f80fbe0a4e..22576ac33e 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/PipelineConfig.xml +++ b/Core/src/org/sleuthkit/autopsy/ingest/PipelineConfig.xml @@ -17,9 +17,9 @@ Contains only the core ingest modules that ship with Autopsy --> org.sleuthkit.autopsy.modules.fileextmismatch.FileExtMismatchDetectorModuleFactory org.sleuthkit.autopsy.modules.interestingitems.InterestingItemsIngestModuleFactory org.sleuthkit.autopsy.modules.photoreccarver.PhotoRecCarverIngestModuleFactory - org.sleuthkit.autopsy.modules.GPX_Module.GPXParserFileIngestModuleFactory + GPX_Parser_Module.GPXParserFileIngestModuleFactory - + org.sleuthkit.autopsy.modules.dataSourceIntegrity.DataSourceIntegrityModuleFactory