From dcd23b96b75e7a16b73834a1e70499fe51ffc494 Mon Sep 17 00:00:00 2001 From: Oliver Spohngellert Date: Tue, 31 May 2016 08:28:21 -0400 Subject: [PATCH 1/9] Made updates to data source ingest job, waiting for jython stripping code. --- .../autopsy/ingest/DataSourceIngestJob.java | 41 ++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java index 12930abba9..0800267139 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java @@ -31,10 +31,19 @@ import java.util.logging.Level; import javax.swing.JOptionPane; import org.netbeans.api.progress.ProgressHandle; import org.openide.util.Cancellable; +import org.openide.util.Exceptions; import org.openide.util.NbBundle; +import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.autopsy.coreutils.NetworkUtils; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.Content; +import org.sleuthkit.datamodel.IngestJobInfo; +import org.sleuthkit.datamodel.IngestModuleInfo; +import org.sleuthkit.datamodel.IngestModuleType; +import org.sleuthkit.datamodel.SleuthkitCase; +import org.sleuthkit.datamodel.TskCoreException; +import org.sleuthkit.datamodel.TskDataException; /** * Encapsulates a data source and the ingest module pipelines used to process @@ -151,6 +160,8 @@ final class DataSourceIngestJob { private ProgressHandle fileIngestProgress; private String currentFileIngestModule = ""; private String currentFileIngestTask = ""; + private List ingestModules = new ArrayList<>(); + private IngestJobInfo ingestJob; /** * A data source ingest job uses this field to report its creation time. @@ -243,6 +254,23 @@ final class DataSourceIngestJob { */ Thread.currentThread().interrupt(); } + SleuthkitCase skCase = Case.getCurrentCase().getSleuthkitCase(); + try { + for (IngestModuleTemplate module : firstStageDataSourceModuleTemplates) { + IngestModuleType type = module.isDataSourceIngestModuleTemplate() ? IngestModuleType.DATA_SOURCE_LEVEL : IngestModuleType.FILE_LEVEL; + ingestModules.add(skCase.addIngestModule(module.getModuleName(), currentFileIngestTask, type, module.getModuleFactory().getModuleVersionNumber())); + } + for (IngestModuleTemplate module : fileIngestModuleTemplates) { + IngestModuleType type = module.isDataSourceIngestModuleTemplate() ? IngestModuleType.DATA_SOURCE_LEVEL : IngestModuleType.FILE_LEVEL; + ingestModules.add(skCase.addIngestModule(module.getModuleName(), currentFileIngestTask, type, module.getModuleFactory().getModuleVersionNumber())); + } + for (IngestModuleTemplate module : secondStageDataSourceModuleTemplates) { + IngestModuleType type = module.isDataSourceIngestModuleTemplate() ? IngestModuleType.DATA_SOURCE_LEVEL : IngestModuleType.FILE_LEVEL; + ingestModules.add(skCase.addIngestModule(module.getModuleName(), currentFileIngestTask, type, module.getModuleFactory().getModuleVersionNumber())); + } + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, "Failed to add ingest modules to database.", ex); + } } /** @@ -365,6 +393,11 @@ final class DataSourceIngestJob { logger.log(Level.INFO, "Starting second stage analysis for {0} (jobId={1}), no first stage configured", new Object[]{dataSource.getName(), this.id}); //NON-NLS this.startSecondStage(); } + try { + this.ingestJob = Case.getCurrentCase().getSleuthkitCase().addIngestJob(dataSource, NetworkUtils.getLocalHostName(), ingestModules, this.createTime); + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, "Failed to add ingest job to database.", ex); + } } return errors; } @@ -641,7 +674,13 @@ final class DataSourceIngestJob { } } } - + try { + this.ingestJob.setEndDate(new Date().toInstant().toEpochMilli()); + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, "Failed to set end date for ingest job in database.", ex); + } catch (TskDataException ex) { + Exceptions.printStackTrace(ex); + } this.parentJob.dataSourceJobFinished(this); } From e5fd92589fc21eb5734af75bf6312d64fbce06b4 Mon Sep 17 00:00:00 2001 From: Oliver Spohngellert Date: Tue, 31 May 2016 10:42:44 -0400 Subject: [PATCH 2/9] Made jobs end properly and created python stripping class. --- .../autopsy/ingest/DataSourceIngestJob.java | 26 ++++++------ .../ingest/FactoryClassNameNormalizer.java | 42 +++++++++++++++++++ .../autopsy/ingest/IngestJobSettings.java | 18 +++----- 3 files changed, 61 insertions(+), 25 deletions(-) create mode 100755 Core/src/org/sleuthkit/autopsy/ingest/FactoryClassNameNormalizer.java diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java index 0800267139..27ddb8bbb8 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java @@ -31,7 +31,6 @@ import java.util.logging.Level; import javax.swing.JOptionPane; import org.netbeans.api.progress.ProgressHandle; import org.openide.util.Cancellable; -import org.openide.util.Exceptions; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.coreutils.Logger; @@ -258,15 +257,18 @@ final class DataSourceIngestJob { try { for (IngestModuleTemplate module : firstStageDataSourceModuleTemplates) { IngestModuleType type = module.isDataSourceIngestModuleTemplate() ? IngestModuleType.DATA_SOURCE_LEVEL : IngestModuleType.FILE_LEVEL; - ingestModules.add(skCase.addIngestModule(module.getModuleName(), currentFileIngestTask, type, module.getModuleFactory().getModuleVersionNumber())); + String uniqueName = FactoryClassNameNormalizer.normalize(module.getModuleFactory().getClass().getCanonicalName()) + "-" + module.getModuleFactory().getModuleDisplayName() + "-" + type.getTypeName() + "-" + module.getModuleFactory().getModuleVersionNumber(); + ingestModules.add(skCase.addIngestModule(module.getModuleName(), uniqueName, type, module.getModuleFactory().getModuleVersionNumber())); } for (IngestModuleTemplate module : fileIngestModuleTemplates) { IngestModuleType type = module.isDataSourceIngestModuleTemplate() ? IngestModuleType.DATA_SOURCE_LEVEL : IngestModuleType.FILE_LEVEL; - ingestModules.add(skCase.addIngestModule(module.getModuleName(), currentFileIngestTask, type, module.getModuleFactory().getModuleVersionNumber())); + String uniqueName = FactoryClassNameNormalizer.normalize(module.getModuleFactory().getClass().getCanonicalName()) + "-" + module.getModuleFactory().getModuleDisplayName() + "-" + type.getTypeName() + "-" + module.getModuleFactory().getModuleVersionNumber(); + ingestModules.add(skCase.addIngestModule(module.getModuleName(), uniqueName, type, module.getModuleFactory().getModuleVersionNumber())); } for (IngestModuleTemplate module : secondStageDataSourceModuleTemplates) { IngestModuleType type = module.isDataSourceIngestModuleTemplate() ? IngestModuleType.DATA_SOURCE_LEVEL : IngestModuleType.FILE_LEVEL; - ingestModules.add(skCase.addIngestModule(module.getModuleName(), currentFileIngestTask, type, module.getModuleFactory().getModuleVersionNumber())); + String uniqueName = FactoryClassNameNormalizer.normalize(module.getModuleFactory().getClass().getCanonicalName()) + "-" + module.getModuleFactory().getModuleDisplayName() + "-" + type.getTypeName() + "-" + module.getModuleFactory().getModuleVersionNumber(); + ingestModules.add(skCase.addIngestModule(module.getModuleName(), uniqueName, type, module.getModuleFactory().getModuleVersionNumber())); } } catch (TskCoreException ex) { logger.log(Level.SEVERE, "Failed to add ingest modules to database.", ex); @@ -396,7 +398,7 @@ final class DataSourceIngestJob { try { this.ingestJob = Case.getCurrentCase().getSleuthkitCase().addIngestJob(dataSource, NetworkUtils.getLocalHostName(), ingestModules, this.createTime); } catch (TskCoreException ex) { - logger.log(Level.SEVERE, "Failed to add ingest job to database.", ex); + logger.log(Level.SEVERE, "Failed to add ingest job to database.", ex); } } return errors; @@ -674,14 +676,14 @@ final class DataSourceIngestJob { } } } - try { - this.ingestJob.setEndDate(new Date().toInstant().toEpochMilli()); - } catch (TskCoreException ex) { - logger.log(Level.SEVERE, "Failed to set end date for ingest job in database.", ex); - } catch (TskDataException ex) { - Exceptions.printStackTrace(ex); + if (!this.cancelled) { + try { + this.ingestJob.setEndDate(new Date().toInstant().toEpochMilli()); + } catch (TskCoreException | TskDataException ex) { + logger.log(Level.SEVERE, "Failed to set end date for ingest job in database.", ex); + } + this.parentJob.dataSourceJobFinished(this); } - this.parentJob.dataSourceJobFinished(this); } /** diff --git a/Core/src/org/sleuthkit/autopsy/ingest/FactoryClassNameNormalizer.java b/Core/src/org/sleuthkit/autopsy/ingest/FactoryClassNameNormalizer.java new file mode 100755 index 0000000000..be12540921 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/ingest/FactoryClassNameNormalizer.java @@ -0,0 +1,42 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ +package org.sleuthkit.autopsy.ingest; + +import org.sleuthkit.autopsy.coreutils.Logger; + +/** + * Used to strip python ids on factory class names. + */ +class FactoryClassNameNormalizer { + + private static final CharSequence pythonModuleSettingsPrefixCS = "org.python.proxies.".subSequence(0, "org.python.proxies.".length() - 1); //NON-NLS + private static final Logger logger = Logger.getLogger(FactoryClassNameNormalizer.class.getName()); + + static String normalize(String canonicalClassName) { + if (isPythonModuleSettingsFile(canonicalClassName)) { + // compiled python modules have variable instance number as a part of their file name. + // This block of code gets rid of that variable instance number and helps maitains constant module name over multiple runs. + String moduleClassName = canonicalClassName.replaceAll("[$][\\d]", "\\"); //NON-NLS NON-NLS + return moduleClassName; + } + return canonicalClassName; + } + + /** + * Determines if the moduleSettingsFilePath is that of a serialized jython + * instance. Serialized Jython instances (settings saved on the disk) + * contain "org.python.proxies." in their fileName based on the current + * implementation. + * + * @param moduleSettingsFilePath path to the module settings file. + * + * @return True or false + */ + private static boolean isPythonModuleSettingsFile(String moduleSettingsFilePath) { + return moduleSettingsFilePath.contains(pythonModuleSettingsPrefixCS); + } + +} diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobSettings.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobSettings.java index 827bbc016d..58644dc05a 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobSettings.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobSettings.java @@ -33,10 +33,10 @@ import java.util.logging.Level; import org.openide.util.NbBundle; import org.openide.util.io.NbObjectInputStream; import org.openide.util.io.NbObjectOutputStream; +import org.python.util.PythonObjectInputStream; +import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.ModuleSettings; import org.sleuthkit.autopsy.coreutils.PlatformUtil; -import org.sleuthkit.autopsy.coreutils.Logger; -import org.python.util.PythonObjectInputStream; /** * Encapsulates the ingest job settings for a particular execution context. @@ -467,17 +467,9 @@ public class IngestJobSettings { * @param settings The ingest job settings for the ingest module */ private void saveModuleSettings(IngestModuleFactory factory, IngestModuleIngestJobSettings settings) { - try { - String moduleSettingsFilePath = getModuleSettingsFilePath(factory); - // compiled python modules have substring org.python.proxies. It can be used to identify them. - if (isPythonModuleSettingsFile(moduleSettingsFilePath)) { - // compiled python modules have variable instance number as a part of their file name. - // This block of code gets rid of that variable instance number and helps maitains constant module name over multiple runs. - moduleSettingsFilePath = moduleSettingsFilePath.replaceAll("[$][\\d]+.settings$", "\\$.settings"); //NON-NLS NON-NLS - } - try (NbObjectOutputStream out = new NbObjectOutputStream(new FileOutputStream(moduleSettingsFilePath))) { - out.writeObject(settings); - } + String moduleSettingsFilePath = FactoryClassNameNormalizer.normalize(factory.getClass().getCanonicalName()) + MODULE_SETTINGS_FILE_EXT; + try (NbObjectOutputStream out = new NbObjectOutputStream(new FileOutputStream(moduleSettingsFilePath))) { + out.writeObject(settings); } catch (IOException ex) { String warning = NbBundle.getMessage(IngestJobSettings.class, "IngestJobSettings.moduleSettingsSave.warning", factory.getModuleDisplayName(), this.executionContext); //NON-NLS logger.log(Level.SEVERE, warning, ex); From 5e6b776761076189e5ea7d12cc8ce09cc0d791a2 Mon Sep 17 00:00:00 2001 From: Oliver Spohngellert Date: Tue, 31 May 2016 16:00:08 -0400 Subject: [PATCH 3/9] Added support for status. --- .../autopsy/ingest/DataSourceIngestJob.java | 22 +++++++++++++++---- .../ingest/FactoryClassNameNormalizer.java | 2 +- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java index 27ddb8bbb8..3de307a752 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java @@ -40,6 +40,7 @@ import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.IngestJobInfo; import org.sleuthkit.datamodel.IngestModuleInfo; import org.sleuthkit.datamodel.IngestModuleType; +import org.sleuthkit.datamodel.IngestStatusType; import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskDataException; @@ -676,14 +677,27 @@ final class DataSourceIngestJob { } } } - if (!this.cancelled) { + if (this.cancelled) { try { - this.ingestJob.setEndDate(new Date().toInstant().toEpochMilli()); + ingestJob.setIngestStatus(IngestStatusType.CANCELLED); } catch (TskCoreException | TskDataException ex) { - logger.log(Level.SEVERE, "Failed to set end date for ingest job in database.", ex); + logger.log(Level.SEVERE, "Failed to set ingest status for ingest job in database.", ex); } - this.parentJob.dataSourceJobFinished(this); } + else { + try { + ingestJob.setIngestStatus(IngestStatusType.COMPLETED); + } catch (TskCoreException | TskDataException ex) { + logger.log(Level.SEVERE, "Failed to set ingest status for ingest job in database.", ex); + } + } + try { + this.ingestJob.setEndDate(new Date().toInstant().toEpochMilli()); + } catch (TskCoreException | TskDataException ex) { + logger.log(Level.SEVERE, "Failed to set end date for ingest job in database.", ex); + } + this.parentJob.dataSourceJobFinished(this); + } /** diff --git a/Core/src/org/sleuthkit/autopsy/ingest/FactoryClassNameNormalizer.java b/Core/src/org/sleuthkit/autopsy/ingest/FactoryClassNameNormalizer.java index be12540921..a03abea38a 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/FactoryClassNameNormalizer.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/FactoryClassNameNormalizer.java @@ -19,7 +19,7 @@ class FactoryClassNameNormalizer { if (isPythonModuleSettingsFile(canonicalClassName)) { // compiled python modules have variable instance number as a part of their file name. // This block of code gets rid of that variable instance number and helps maitains constant module name over multiple runs. - String moduleClassName = canonicalClassName.replaceAll("[$][\\d]", "\\"); //NON-NLS NON-NLS + String moduleClassName = canonicalClassName.replaceAll("[$][\\d]", ""); //NON-NLS NON-NLS return moduleClassName; } return canonicalClassName; From 63647c76a78bfe2cb2b586b293658fe5c25a6e21 Mon Sep 17 00:00:00 2001 From: Oliver Spohngellert Date: Tue, 7 Jun 2016 13:03:21 -0400 Subject: [PATCH 4/9] Updated as per PR notes. --- .../autopsy/ingest/DataSourceIngestJob.java | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java index 3de307a752..964d26ace0 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java @@ -38,9 +38,9 @@ import org.sleuthkit.autopsy.coreutils.NetworkUtils; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.IngestJobInfo; +import org.sleuthkit.datamodel.IngestJobInfo.IngestJobStatusType; import org.sleuthkit.datamodel.IngestModuleInfo; -import org.sleuthkit.datamodel.IngestModuleType; -import org.sleuthkit.datamodel.IngestStatusType; +import org.sleuthkit.datamodel.IngestModuleInfo.IngestModuleType; import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskDataException; @@ -258,17 +258,17 @@ final class DataSourceIngestJob { try { for (IngestModuleTemplate module : firstStageDataSourceModuleTemplates) { IngestModuleType type = module.isDataSourceIngestModuleTemplate() ? IngestModuleType.DATA_SOURCE_LEVEL : IngestModuleType.FILE_LEVEL; - String uniqueName = FactoryClassNameNormalizer.normalize(module.getModuleFactory().getClass().getCanonicalName()) + "-" + module.getModuleFactory().getModuleDisplayName() + "-" + type.getTypeName() + "-" + module.getModuleFactory().getModuleVersionNumber(); + String uniqueName = FactoryClassNameNormalizer.normalize(module.getModuleFactory().getClass().getCanonicalName()) + "-" + module.getModuleFactory().getModuleDisplayName() + "-" + type.toString() + "-" + module.getModuleFactory().getModuleVersionNumber(); ingestModules.add(skCase.addIngestModule(module.getModuleName(), uniqueName, type, module.getModuleFactory().getModuleVersionNumber())); } for (IngestModuleTemplate module : fileIngestModuleTemplates) { IngestModuleType type = module.isDataSourceIngestModuleTemplate() ? IngestModuleType.DATA_SOURCE_LEVEL : IngestModuleType.FILE_LEVEL; - String uniqueName = FactoryClassNameNormalizer.normalize(module.getModuleFactory().getClass().getCanonicalName()) + "-" + module.getModuleFactory().getModuleDisplayName() + "-" + type.getTypeName() + "-" + module.getModuleFactory().getModuleVersionNumber(); + String uniqueName = FactoryClassNameNormalizer.normalize(module.getModuleFactory().getClass().getCanonicalName()) + "-" + module.getModuleFactory().getModuleDisplayName() + "-" + type.toString() + "-" + module.getModuleFactory().getModuleVersionNumber(); ingestModules.add(skCase.addIngestModule(module.getModuleName(), uniqueName, type, module.getModuleFactory().getModuleVersionNumber())); } for (IngestModuleTemplate module : secondStageDataSourceModuleTemplates) { IngestModuleType type = module.isDataSourceIngestModuleTemplate() ? IngestModuleType.DATA_SOURCE_LEVEL : IngestModuleType.FILE_LEVEL; - String uniqueName = FactoryClassNameNormalizer.normalize(module.getModuleFactory().getClass().getCanonicalName()) + "-" + module.getModuleFactory().getModuleDisplayName() + "-" + type.getTypeName() + "-" + module.getModuleFactory().getModuleVersionNumber(); + String uniqueName = FactoryClassNameNormalizer.normalize(module.getModuleFactory().getClass().getCanonicalName()) + "-" + module.getModuleFactory().getModuleDisplayName() + "-" + type.toString() + "-" + module.getModuleFactory().getModuleVersionNumber(); ingestModules.add(skCase.addIngestModule(module.getModuleName(), uniqueName, type, module.getModuleFactory().getModuleVersionNumber())); } } catch (TskCoreException ex) { @@ -397,7 +397,7 @@ final class DataSourceIngestJob { this.startSecondStage(); } try { - this.ingestJob = Case.getCurrentCase().getSleuthkitCase().addIngestJob(dataSource, NetworkUtils.getLocalHostName(), ingestModules, this.createTime); + this.ingestJob = Case.getCurrentCase().getSleuthkitCase().addIngestJob(dataSource, NetworkUtils.getLocalHostName(), ingestModules, new Date(this.createTime)); } catch (TskCoreException ex) { logger.log(Level.SEVERE, "Failed to add ingest job to database.", ex); } @@ -679,20 +679,20 @@ final class DataSourceIngestJob { } if (this.cancelled) { try { - ingestJob.setIngestStatus(IngestStatusType.CANCELLED); + ingestJob.setIngestJobStatus(IngestJobStatusType.CANCELLED); } catch (TskCoreException | TskDataException ex) { logger.log(Level.SEVERE, "Failed to set ingest status for ingest job in database.", ex); } } else { try { - ingestJob.setIngestStatus(IngestStatusType.COMPLETED); + ingestJob.setIngestJobStatus(IngestJobStatusType.COMPLETED); } catch (TskCoreException | TskDataException ex) { logger.log(Level.SEVERE, "Failed to set ingest status for ingest job in database.", ex); } } try { - this.ingestJob.setEndDate(new Date().toInstant().toEpochMilli()); + this.ingestJob.setEndDateTime(new Date()); } catch (TskCoreException | TskDataException ex) { logger.log(Level.SEVERE, "Failed to set end date for ingest job in database.", ex); } From f1fdd5408b5a99e684b26ee5e1c71cb21a7d3d84 Mon Sep 17 00:00:00 2001 From: Oliver Spohngellert Date: Wed, 8 Jun 2016 11:36:14 -0400 Subject: [PATCH 5/9] Updated to match sleuthkit update. --- .../sleuthkit/autopsy/ingest/DataSourceIngestJob.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java index 964d26ace0..4655ae90a7 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java @@ -43,7 +43,6 @@ import org.sleuthkit.datamodel.IngestModuleInfo; import org.sleuthkit.datamodel.IngestModuleInfo.IngestModuleType; import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.TskCoreException; -import org.sleuthkit.datamodel.TskDataException; /** * Encapsulates a data source and the ingest module pipelines used to process @@ -397,7 +396,7 @@ final class DataSourceIngestJob { this.startSecondStage(); } try { - this.ingestJob = Case.getCurrentCase().getSleuthkitCase().addIngestJob(dataSource, NetworkUtils.getLocalHostName(), ingestModules, new Date(this.createTime)); + this.ingestJob = Case.getCurrentCase().getSleuthkitCase().addIngestJob(dataSource, NetworkUtils.getLocalHostName(), ingestModules, new Date(this.createTime), new Date(0), IngestJobStatusType.STARTED, ""); } catch (TskCoreException ex) { logger.log(Level.SEVERE, "Failed to add ingest job to database.", ex); } @@ -680,20 +679,20 @@ final class DataSourceIngestJob { if (this.cancelled) { try { ingestJob.setIngestJobStatus(IngestJobStatusType.CANCELLED); - } catch (TskCoreException | TskDataException ex) { + } catch (TskCoreException ex) { logger.log(Level.SEVERE, "Failed to set ingest status for ingest job in database.", ex); } } else { try { ingestJob.setIngestJobStatus(IngestJobStatusType.COMPLETED); - } catch (TskCoreException | TskDataException ex) { + } catch (TskCoreException ex) { logger.log(Level.SEVERE, "Failed to set ingest status for ingest job in database.", ex); } } try { this.ingestJob.setEndDateTime(new Date()); - } catch (TskCoreException | TskDataException ex) { + } catch (TskCoreException ex) { logger.log(Level.SEVERE, "Failed to set end date for ingest job in database.", ex); } this.parentJob.dataSourceJobFinished(this); From e4e256d5b4db4b579b8a8414c1c7cfc7757764b0 Mon Sep 17 00:00:00 2001 From: Oliver Spohngellert Date: Thu, 9 Jun 2016 13:44:17 -0400 Subject: [PATCH 6/9] Reorganized adding of ingest modules. --- .../autopsy/ingest/DataSourceIngestJob.java | 29 ++++++++----------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java index 4655ae90a7..a5f8683f90 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java @@ -255,26 +255,22 @@ final class DataSourceIngestJob { } SleuthkitCase skCase = Case.getCurrentCase().getSleuthkitCase(); try { - for (IngestModuleTemplate module : firstStageDataSourceModuleTemplates) { - IngestModuleType type = module.isDataSourceIngestModuleTemplate() ? IngestModuleType.DATA_SOURCE_LEVEL : IngestModuleType.FILE_LEVEL; - String uniqueName = FactoryClassNameNormalizer.normalize(module.getModuleFactory().getClass().getCanonicalName()) + "-" + module.getModuleFactory().getModuleDisplayName() + "-" + type.toString() + "-" + module.getModuleFactory().getModuleVersionNumber(); - ingestModules.add(skCase.addIngestModule(module.getModuleName(), uniqueName, type, module.getModuleFactory().getModuleVersionNumber())); - } - for (IngestModuleTemplate module : fileIngestModuleTemplates) { - IngestModuleType type = module.isDataSourceIngestModuleTemplate() ? IngestModuleType.DATA_SOURCE_LEVEL : IngestModuleType.FILE_LEVEL; - String uniqueName = FactoryClassNameNormalizer.normalize(module.getModuleFactory().getClass().getCanonicalName()) + "-" + module.getModuleFactory().getModuleDisplayName() + "-" + type.toString() + "-" + module.getModuleFactory().getModuleVersionNumber(); - ingestModules.add(skCase.addIngestModule(module.getModuleName(), uniqueName, type, module.getModuleFactory().getModuleVersionNumber())); - } - for (IngestModuleTemplate module : secondStageDataSourceModuleTemplates) { - IngestModuleType type = module.isDataSourceIngestModuleTemplate() ? IngestModuleType.DATA_SOURCE_LEVEL : IngestModuleType.FILE_LEVEL; - String uniqueName = FactoryClassNameNormalizer.normalize(module.getModuleFactory().getClass().getCanonicalName()) + "-" + module.getModuleFactory().getModuleDisplayName() + "-" + type.toString() + "-" + module.getModuleFactory().getModuleVersionNumber(); - ingestModules.add(skCase.addIngestModule(module.getModuleName(), uniqueName, type, module.getModuleFactory().getModuleVersionNumber())); - } + this.addIngestModules(firstStageDataSourceModuleTemplates, skCase); + this.addIngestModules(fileIngestModuleTemplates, skCase); + this.addIngestModules(secondStageDataSourceModuleTemplates, skCase); } catch (TskCoreException ex) { logger.log(Level.SEVERE, "Failed to add ingest modules to database.", ex); } } + private void addIngestModules(List templates, SleuthkitCase skCase) throws TskCoreException { + for (IngestModuleTemplate module : templates) { + IngestModuleType type = module.isDataSourceIngestModuleTemplate() ? IngestModuleType.DATA_SOURCE_LEVEL : IngestModuleType.FILE_LEVEL; + String uniqueName = FactoryClassNameNormalizer.normalize(module.getModuleFactory().getClass().getCanonicalName()) + "-" + module.getModuleFactory().getModuleDisplayName() + "-" + type.toString() + "-" + module.getModuleFactory().getModuleVersionNumber(); + ingestModules.add(skCase.addIngestModule(module.getModuleName(), uniqueName, type, module.getModuleFactory().getModuleVersionNumber())); + } + } + /** * Uses an input collection of ingest module templates and a pipeline * configuration, i.e., an ordered list of ingest module factory class @@ -682,8 +678,7 @@ final class DataSourceIngestJob { } catch (TskCoreException ex) { logger.log(Level.SEVERE, "Failed to set ingest status for ingest job in database.", ex); } - } - else { + } else { try { ingestJob.setIngestJobStatus(IngestJobStatusType.COMPLETED); } catch (TskCoreException ex) { From f7198676788b6c8437748e14563e4550cc5b0433 Mon Sep 17 00:00:00 2001 From: Oliver Spohngellert Date: Thu, 9 Jun 2016 14:47:55 -0400 Subject: [PATCH 7/9] Properly accounted for modules of both types. --- .../sleuthkit/autopsy/ingest/DataSourceIngestJob.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java index a5f8683f90..7c2d7f3b61 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java @@ -255,17 +255,16 @@ final class DataSourceIngestJob { } SleuthkitCase skCase = Case.getCurrentCase().getSleuthkitCase(); try { - this.addIngestModules(firstStageDataSourceModuleTemplates, skCase); - this.addIngestModules(fileIngestModuleTemplates, skCase); - this.addIngestModules(secondStageDataSourceModuleTemplates, skCase); + this.addIngestModules(firstStageDataSourceModuleTemplates, IngestModuleType.DATA_SOURCE_LEVEL, skCase); + this.addIngestModules(fileIngestModuleTemplates, IngestModuleType.FILE_LEVEL, skCase); + this.addIngestModules(secondStageDataSourceModuleTemplates, IngestModuleType.DATA_SOURCE_LEVEL, skCase); } catch (TskCoreException ex) { logger.log(Level.SEVERE, "Failed to add ingest modules to database.", ex); } } - private void addIngestModules(List templates, SleuthkitCase skCase) throws TskCoreException { + private void addIngestModules(List templates, IngestModuleType type, SleuthkitCase skCase) throws TskCoreException { for (IngestModuleTemplate module : templates) { - IngestModuleType type = module.isDataSourceIngestModuleTemplate() ? IngestModuleType.DATA_SOURCE_LEVEL : IngestModuleType.FILE_LEVEL; String uniqueName = FactoryClassNameNormalizer.normalize(module.getModuleFactory().getClass().getCanonicalName()) + "-" + module.getModuleFactory().getModuleDisplayName() + "-" + type.toString() + "-" + module.getModuleFactory().getModuleVersionNumber(); ingestModules.add(skCase.addIngestModule(module.getModuleName(), uniqueName, type, module.getModuleFactory().getModuleVersionNumber())); } From 3fdedcd4db68bb10b9c8f695491b0b3f10ea2661 Mon Sep 17 00:00:00 2001 From: Oliver Spohngellert Date: Fri, 10 Jun 2016 12:28:52 -0400 Subject: [PATCH 8/9] Fixed ingest settings path. --- Core/src/org/sleuthkit/autopsy/ingest/IngestJobSettings.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobSettings.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobSettings.java index 58644dc05a..bea9d2a10b 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobSettings.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobSettings.java @@ -467,7 +467,7 @@ public class IngestJobSettings { * @param settings The ingest job settings for the ingest module */ private void saveModuleSettings(IngestModuleFactory factory, IngestModuleIngestJobSettings settings) { - String moduleSettingsFilePath = FactoryClassNameNormalizer.normalize(factory.getClass().getCanonicalName()) + MODULE_SETTINGS_FILE_EXT; + String moduleSettingsFilePath = Paths.get(this.moduleSettingsFolderPath, FactoryClassNameNormalizer.normalize(factory.getClass().getCanonicalName()) + MODULE_SETTINGS_FILE_EXT).toString(); try (NbObjectOutputStream out = new NbObjectOutputStream(new FileOutputStream(moduleSettingsFilePath))) { out.writeObject(settings); } catch (IOException ex) { From 0ea1cedf5d3ea5e62de6281adb708446a0966840 Mon Sep 17 00:00:00 2001 From: Oliver Spohngellert Date: Mon, 13 Jun 2016 11:08:12 -0400 Subject: [PATCH 9/9] Made unique name generated in sk. --- Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java index 7c2d7f3b61..1869be9169 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java @@ -265,8 +265,7 @@ final class DataSourceIngestJob { private void addIngestModules(List templates, IngestModuleType type, SleuthkitCase skCase) throws TskCoreException { for (IngestModuleTemplate module : templates) { - String uniqueName = FactoryClassNameNormalizer.normalize(module.getModuleFactory().getClass().getCanonicalName()) + "-" + module.getModuleFactory().getModuleDisplayName() + "-" + type.toString() + "-" + module.getModuleFactory().getModuleVersionNumber(); - ingestModules.add(skCase.addIngestModule(module.getModuleName(), uniqueName, type, module.getModuleFactory().getModuleVersionNumber())); + ingestModules.add(skCase.addIngestModule(module.getModuleName(), FactoryClassNameNormalizer.normalize(module.getModuleFactory().getClass().getCanonicalName()), type, module.getModuleFactory().getModuleVersionNumber())); } }