fix for searching for jython classes

Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java

@@ -29,6 +29,9 @@ import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.logging.Level;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Stream;
 import javax.swing.JOptionPane;
 import org.netbeans.api.progress.ProgressHandle;
 import org.openide.util.Cancellable;
@@ -57,10 +60,14 @@ import org.sleuthkit.autopsy.python.FactoryClassNameNormalizer;
  * it.
  */
 public final class DataSourceIngestJob {
+
     private static String AUTOPSY_MODULE_PREFIX = "org.sleuthkit.autopsy";
 
     private static final Logger logger = Logger.getLogger(DataSourceIngestJob.class.getName());
 
+    // to match something like: "org.python.proxies.GPX_Parser_Module$GPXParserFileIngestModuleFactory$14"
+    private static final Pattern JYTHON_REGEX = Pattern.compile("org\\.python\\.proxies\\.(.+?)\\$(.+?)(\\$[0-9]*)?$");
+
     /**
      * These fields define a data source ingest job: the parent ingest job, an
      * ID, the user's ingest job settings, and the data source to be analyzed.
@@ -217,22 +224,26 @@ public final class DataSourceIngestJob {
         this.createIngestPipelines();
     }
 
-    
-    
     /**
-     * Adds ingest modules to a list with autopsy modules first and third party modules next.
+     * Adds ingest modules to a list with autopsy modules first and third party
-     * @param dest  The destination for the modules to be added.
+     * modules next.
-     * @param src   A map of fully qualified class name mapped to the IngestModuleTemplate.
+     *
+     * @param dest      The destination for the modules to be added.
+     * @param src       A map of fully qualified class name mapped to the
+     *                  IngestModuleTemplate.
+     * @param jythonSrc A map of fully qualified class name mapped to the
+     *                  IngestModuleTemplate for jython modules.
      */
-    private static void addOrdered(final List<IngestModuleTemplate> dest, final Map<String, IngestModuleTemplate> src) {
+    private static void addOrdered(final List<IngestModuleTemplate> dest,
+            final Map<String, IngestModuleTemplate> src, final Map<String, IngestModuleTemplate> jythonSrc) {
+
         final List<IngestModuleTemplate> autopsyModules = new ArrayList<>();
         final List<IngestModuleTemplate> thirdPartyModules = new ArrayList<>();
 
-        src.entrySet().stream().forEach((templateEntry) -> {
+        Stream.concat(src.entrySet().stream(), jythonSrc.entrySet().stream()).forEach((templateEntry) -> {
             if (templateEntry.getKey().startsWith(AUTOPSY_MODULE_PREFIX)) {
                 autopsyModules.add(templateEntry.getValue());
-            }
+            } else {
-            else {
                 thirdPartyModules.add(templateEntry.getValue());
             }
         });
@@ -241,6 +252,46 @@ public final class DataSourceIngestJob {
         dest.addAll(thirdPartyModules);
     }
 
+    /**
+     * Takes a classname like
+     * "org.python.proxies.GPX_Parser_Module$GPXParserFileIngestModuleFactory$14"
+     * and provides "GPX_Parser_Module.GPXParserFileIngestModuleFactory" or null
+     * if not in jython package.
+     *
+     * @param canonicalName The canonical name.
+     *
+     * @return The jython name or null if not in jython package.
+     */
+    private static String getJythonName(String canonicalName) {
+        Matcher m = JYTHON_REGEX.matcher(canonicalName);
+        if (m.find()) {
+            return String.format("%s.%s", m.group(1), m.group(2));
+        } else {
+            return null;
+        }
+    }
+
+    /**
+     * Adds a template to the appropriate map. If the class is a jython class,
+     * then it is added to the jython map. Otherwise, it is added to the
+     * mapping.
+     *
+     * @param mapping       Mapping for non-jython objects.
+     * @param jythonMapping Mapping for jython objects.
+     * @param template      The template to add.
+     */
+    private static void addModule(Map<String, IngestModuleTemplate> mapping,
+            Map<String, IngestModuleTemplate> jythonMapping, IngestModuleTemplate template) {
+
+        String className = template.getModuleFactory().getClass().getCanonicalName();
+        String jythonName = getJythonName(className);
+        if (jythonName != null) {
+            jythonMapping.put(jythonName, template);
+        } else {
+            mapping.put(className, template);
+        }
+    }
+
     /**
      * Creates the file and data source ingest pipelines.
      */
@@ -252,12 +303,18 @@ public final class DataSourceIngestJob {
          */
         Map<String, IngestModuleTemplate> dataSourceModuleTemplates = new LinkedHashMap<>();
         Map<String, IngestModuleTemplate> fileModuleTemplates = new LinkedHashMap<>();
+
+        // mappings for jython modules.  These mappings are only used to determine modules in the pipelineconfig.xml.
+
+        Map<String, IngestModuleTemplate> jythonDataSourceModuleTemplates = new LinkedHashMap<>();
+        Map<String, IngestModuleTemplate> jythonFileModuleTemplates = new LinkedHashMap<>();
+
         for (IngestModuleTemplate template : ingestModuleTemplates) {
             if (template.isDataSourceIngestModuleTemplate()) {
-                dataSourceModuleTemplates.put(template.getModuleFactory().getClass().getCanonicalName(), template);
+                addModule(dataSourceModuleTemplates, jythonDataSourceModuleTemplates, template);
             }
             if (template.isFileIngestModuleTemplate()) {
-                fileModuleTemplates.put(template.getModuleFactory().getClass().getCanonicalName(), template);
+                addModule(fileModuleTemplates, jythonFileModuleTemplates, template);
             }
         }
 
@@ -266,17 +323,22 @@ public final class DataSourceIngestJob {
          * ordered lists of ingest module templates for each ingest pipeline.
          */
         IngestPipelinesConfiguration pipelineConfigs = IngestPipelinesConfiguration.getInstance();
-        List<IngestModuleTemplate> firstStageDataSourceModuleTemplates = DataSourceIngestJob.getConfiguredIngestModuleTemplates(dataSourceModuleTemplates, pipelineConfigs.getStageOneDataSourceIngestPipelineConfig());
+        List<IngestModuleTemplate> firstStageDataSourceModuleTemplates = DataSourceIngestJob.getConfiguredIngestModuleTemplates(
-        List<IngestModuleTemplate> fileIngestModuleTemplates = DataSourceIngestJob.getConfiguredIngestModuleTemplates(fileModuleTemplates, pipelineConfigs.getFileIngestPipelineConfig());
+                dataSourceModuleTemplates, jythonDataSourceModuleTemplates, pipelineConfigs.getStageOneDataSourceIngestPipelineConfig());
-        List<IngestModuleTemplate> secondStageDataSourceModuleTemplates = DataSourceIngestJob.getConfiguredIngestModuleTemplates(dataSourceModuleTemplates, pipelineConfigs.getStageTwoDataSourceIngestPipelineConfig());
+
+        List<IngestModuleTemplate> fileIngestModuleTemplates = DataSourceIngestJob.getConfiguredIngestModuleTemplates(
+                fileModuleTemplates, jythonFileModuleTemplates, pipelineConfigs.getFileIngestPipelineConfig());
+
+        List<IngestModuleTemplate> secondStageDataSourceModuleTemplates = DataSourceIngestJob.getConfiguredIngestModuleTemplates(
+                dataSourceModuleTemplates, null, pipelineConfigs.getStageTwoDataSourceIngestPipelineConfig());
 
         /**
          * Add any module templates that were not specified in the pipelines
          * configuration to an appropriate pipeline - either the first stage
          * data source ingest pipeline or the file ingest pipeline.
          */
-        addOrdered(firstStageDataSourceModuleTemplates, dataSourceModuleTemplates);
+        addOrdered(firstStageDataSourceModuleTemplates, dataSourceModuleTemplates, jythonDataSourceModuleTemplates);
-        addOrdered(fileIngestModuleTemplates, fileModuleTemplates);
+        addOrdered(fileIngestModuleTemplates, fileModuleTemplates, jythonFileModuleTemplates);
 
         /**
          * Construct the data source ingest pipelines.
@@ -325,19 +387,27 @@ public final class DataSourceIngestJob {
      * ingest pipeline. The ingest module templates are removed from the input
      * collection as they are added to the output collection.
      *
-     * @param ingestModuleTemplates A mapping of ingest module factory class
+     * @param ingestModuleTemplates       A mapping of ingest module factory
-     *                              names to ingest module templates.
+     *                                    class names to ingest module
-     * @param pipelineConfig        An ordered list of ingest module factory
+     *                                    templates.
-     *                              class names representing an ingest pipeline.
+     * @param jythonIngestModuleTemplates A mapping of jython processed class
+     *                                    names to jython ingest module
+     *                                    templates.
+     * @param pipelineConfig              An ordered list of ingest module
+     *                                    factory class names representing an
+     *                                    ingest pipeline.
      *
      * @return An ordered list of ingest module templates, i.e., an
      *         uninstantiated pipeline.
      */
-    private static List<IngestModuleTemplate> getConfiguredIngestModuleTemplates(Map<String, IngestModuleTemplate> ingestModuleTemplates, List<String> pipelineConfig) {
+    private static List<IngestModuleTemplate> getConfiguredIngestModuleTemplates(
+            Map<String, IngestModuleTemplate> ingestModuleTemplates, Map<String, IngestModuleTemplate> jythonIngestModuleTemplates, List<String> pipelineConfig) {
         List<IngestModuleTemplate> templates = new ArrayList<>();
         for (String moduleClassName : pipelineConfig) {
             if (ingestModuleTemplates.containsKey(moduleClassName)) {
                 templates.add(ingestModuleTemplates.remove(moduleClassName));
+            } else if (jythonIngestModuleTemplates.containsKey(moduleClassName)) {
+                templates.add(jythonIngestModuleTemplates.remove(moduleClassName));
             }
         }
         return templates;

Core/src/org/sleuthkit/autopsy/ingest/PipelineConfig.xml

@@ -17,7 +17,7 @@ Contains only the core ingest modules that ship with Autopsy -->
         <MODULE>org.sleuthkit.autopsy.modules.fileextmismatch.FileExtMismatchDetectorModuleFactory</MODULE>
         <MODULE>org.sleuthkit.autopsy.modules.interestingitems.InterestingItemsIngestModuleFactory</MODULE>
 	    <MODULE>org.sleuthkit.autopsy.modules.photoreccarver.PhotoRecCarverIngestModuleFactory</MODULE>
-        <MODULE>org.sleuthkit.autopsy.modules.GPX_Module.GPXParserFileIngestModuleFactory</MODULE>
+        <MODULE>GPX_Parser_Module.GPXParserFileIngestModuleFactory</MODULE>
     </PIPELINE>
 
     <PIPELINE type="ImageAnalysisStageTwo">