diff --git a/Core/ivy.xml b/Core/ivy.xml index cbe45d8c33..5949f5a777 100644 --- a/Core/ivy.xml +++ b/Core/ivy.xml @@ -31,6 +31,8 @@ + + diff --git a/Core/nbproject/project.properties b/Core/nbproject/project.properties index 89bf4a93d7..b18bef9b6a 100644 --- a/Core/nbproject/project.properties +++ b/Core/nbproject/project.properties @@ -4,6 +4,7 @@ file.reference.commons-compress-1.14.jar=release/modules/ext/commons-compress-1. file.reference.commons-dbcp2-2.1.1.jar=release\\modules\\ext\\commons-dbcp2-2.1.1.jar file.reference.commons-pool2-2.4.2.jar=release\\modules\\ext\\commons-pool2-2.4.2.jar file.reference.dd-plist-1.20.jar=release/modules/ext/dd-plist-1.20.jar +file.reference.jackson-core-2.9.7.jar=release/modules/ext/jackson-core-2.9.7.jar file.reference.jdom-2.0.5-contrib.jar=release/modules/ext/jdom-2.0.5-contrib.jar file.reference.jdom-2.0.5.jar=release/modules/ext/jdom-2.0.5.jar file.reference.jgraphx-v3.8.0.jar=release/modules/ext/jgraphx-v3.8.0.jar diff --git a/Core/nbproject/project.xml b/Core/nbproject/project.xml index 04732e7873..d6f3562663 100644 --- a/Core/nbproject/project.xml +++ b/Core/nbproject/project.xml @@ -433,6 +433,10 @@ ext/curator-client-2.8.0.jar release/modules/ext/curator-client-2.8.0.jar + + ext/jackson-core-2.9.7.jar + release/modules/ext/jackson-core-2.9.7.jar + ext/cxf-rt-frontend-jaxrs-3.0.16.jar release/modules/ext/cxf-rt-frontend-jaxrs-3.0.16.jar diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeNormalizer.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeNormalizer.java index 11a8e2249b..651c07f74c 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeNormalizer.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeNormalizer.java @@ -52,27 +52,29 @@ final public class CorrelationAttributeNormalizer { throw new CorrelationAttributeNormalizationException("Data was null."); } + String trimmedData = data.trim(); + switch(attributeType.getId()){ case CorrelationAttributeInstance.FILES_TYPE_ID: - return normalizeMd5(data); + return normalizeMd5(trimmedData); case CorrelationAttributeInstance.DOMAIN_TYPE_ID: - return normalizeDomain(data); + return normalizeDomain(trimmedData); case CorrelationAttributeInstance.EMAIL_TYPE_ID: - return normalizeEmail(data); + return normalizeEmail(trimmedData); case CorrelationAttributeInstance.PHONE_TYPE_ID: - return normalizePhone(data); + return normalizePhone(trimmedData); case CorrelationAttributeInstance.USBID_TYPE_ID: - return normalizeUsbId(data); + return normalizeUsbId(trimmedData); case CorrelationAttributeInstance.SSID_TYPE_ID: - return data; + return trimmedData; case CorrelationAttributeInstance.MAC_TYPE_ID: - return data; + return trimmedData; case CorrelationAttributeInstance.IMEI_TYPE_ID: - return data; + return trimmedData; case CorrelationAttributeInstance.IMSI_TYPE_ID: - return data; + return trimmedData; case CorrelationAttributeInstance.ICCID_TYPE_ID: - return data; + return trimmedData; default: final String errorMessage = String.format( "Validator function not found for attribute type: %s", diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java index 841b4b0958..bfb0ad613a 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java @@ -62,6 +62,7 @@ public class IngestEventsListener { final Collection recentlyAddedCeArtifacts = new LinkedHashSet<>(); private static int correlationModuleInstanceCount; private static boolean flagNotableItems; + private static boolean flagSeenDevices; private final ExecutorService jobProcessingExecutor; private static final String INGEST_EVENT_THREAD_NAME = "Ingest-Event-Listener-%d"; private final PropertyChangeListener pcl1 = new IngestModuleEventListener(); @@ -136,6 +137,15 @@ public class IngestEventsListener { return flagNotableItems; } + /** + * Are previously seen devices being flagged? + * + * @return True if flagging seen devices; otherwise false. + */ + public synchronized static boolean isFlagSeenDevices() { + return flagSeenDevices; + } + /** * Configure the listener to flag notable items or not. * @@ -145,6 +155,15 @@ public class IngestEventsListener { flagNotableItems = value; } + /** + * Configure the listener to flag previously seen devices or not. + * + * @param value True to flag seen devices; otherwise false. + */ + public synchronized static void setFlagSeenDevices(boolean value) { + flagSeenDevices = value; + } + @NbBundle.Messages({"IngestEventsListener.prevTaggedSet.text=Previously Tagged As Notable (Central Repository)", "IngestEventsListener.prevCaseComment.text=Previous Case: ", "IngestEventsListener.ingestmodule.name=Correlation Engine"}) @@ -186,6 +205,46 @@ public class IngestEventsListener { } } + /** + * Create an Interesting Aritfact hit for a device which was previously seen + * in the central repository. + * + * @param bbArtifact the artifact to create the interesting item for + */ + @NbBundle.Messages({"IngestEventsListener.prevExists.text=Previously Seen Devices (Central Repository)", + "# {0} - typeName", + "# {1} - count", + "IngestEventsListener.prevCount.text=Number of previous {0}: {1}"}) + static private void postCorrelatedPreviousArtifactToBlackboard(BlackboardArtifact bbArtifact) { + + try { + AbstractFile af = bbArtifact.getSleuthkitCase().getAbstractFileById(bbArtifact.getObjectID()); + Collection attributes = new ArrayList<>(); + String MODULE_NAME = Bundle.IngestEventsListener_ingestmodule_name(); + BlackboardArtifact tifArtifact = af.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_ARTIFACT_HIT); + BlackboardAttribute att = new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME, MODULE_NAME, + Bundle.IngestEventsListener_prevExists_text()); + attributes.add(att); + attributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_ASSOCIATED_ARTIFACT, MODULE_NAME, bbArtifact.getArtifactID())); + + tifArtifact.addAttributes(attributes); + try { + // index the artifact for keyword search + Blackboard blackboard = Case.getCurrentCaseThrows().getServices().getBlackboard(); + blackboard.indexArtifact(tifArtifact); + } catch (Blackboard.BlackboardException | NoCurrentCaseException ex) { + LOGGER.log(Level.SEVERE, "Unable to index blackboard artifact " + tifArtifact.getArtifactID(), ex); //NON-NLS + } + + // fire event to notify UI of this new artifact + IngestServices.getInstance().fireModuleDataEvent(new ModuleDataEvent(MODULE_NAME, BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_ARTIFACT_HIT)); + } catch (TskCoreException ex) { + LOGGER.log(Level.SEVERE, "Failed to create BlackboardArtifact.", ex); // NON-NLS + } catch (IllegalStateException ex) { + LOGGER.log(Level.SEVERE, "Failed to create BlackboardAttribute.", ex); // NON-NLS + } + } + private class IngestModuleEventListener implements PropertyChangeListener { @Override @@ -205,7 +264,8 @@ public class IngestEventsListener { case DATA_ADDED: { //if ingest isn't running create the interesting items otherwise use the ingest module setting to determine if we create interesting items boolean flagNotable = !IngestManager.getInstance().isIngestRunning() || isFlagNotableItems(); - jobProcessingExecutor.submit(new DataAddedTask(dbManager, evt, flagNotable)); + boolean flagPrevious = !IngestManager.getInstance().isIngestRunning() || isFlagSeenDevices(); + jobProcessingExecutor.submit(new DataAddedTask(dbManager, evt, flagNotable, flagPrevious)); break; } } @@ -244,11 +304,13 @@ public class IngestEventsListener { private final EamDb dbManager; private final PropertyChangeEvent event; private final boolean flagNotableItemsEnabled; + private final boolean flagPreviousItemsEnabled; - private DataAddedTask(EamDb db, PropertyChangeEvent evt, boolean flagNotableItemsEnabled) { + private DataAddedTask(EamDb db, PropertyChangeEvent evt, boolean flagNotableItemsEnabled, boolean flagPreviousItemsEnabled) { dbManager = db; event = evt; this.flagNotableItemsEnabled = flagNotableItemsEnabled; + this.flagPreviousItemsEnabled = flagPreviousItemsEnabled; } @Override @@ -286,6 +348,21 @@ public class IngestEventsListener { LOGGER.log(Level.INFO, String.format("Unable to flag notable item: %s.", eamArtifact.toString()), ex); } } + if (flagPreviousItemsEnabled + && (eamArtifact.getCorrelationType().getId() == CorrelationAttributeInstance.USBID_TYPE_ID + || eamArtifact.getCorrelationType().getId() == CorrelationAttributeInstance.ICCID_TYPE_ID + || eamArtifact.getCorrelationType().getId() == CorrelationAttributeInstance.IMEI_TYPE_ID + || eamArtifact.getCorrelationType().getId() == CorrelationAttributeInstance.IMSI_TYPE_ID + || eamArtifact.getCorrelationType().getId() == CorrelationAttributeInstance.MAC_TYPE_ID)) { + try { + Long countPreviousOccurences = dbManager.getCountArtifactInstancesByTypeValue(eamArtifact.getCorrelationType(), eamArtifact.getCorrelationValue()); + if (countPreviousOccurences > 0) { + postCorrelatedPreviousArtifactToBlackboard(bbArtifact); + } + } catch (CorrelationAttributeNormalizationException ex) { + LOGGER.log(Level.INFO, String.format("Unable to flag notable item: %s.", eamArtifact.toString()), ex); + } + } eamArtifacts.add(eamArtifact); } } catch (EamDbException ex) { diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/Bundle.properties b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/Bundle.properties index a525713f7c..f99db1edb6 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/Bundle.properties @@ -1,2 +1,3 @@ IngestSettingsPanel.ingestSettingsLabel.text=Ingest Settings IngestSettingsPanel.flagTaggedNotableItemsCheckbox.text=Flag items previously tagged as notable +IngestSettingsPanel.flagPreviouslySeenDevicesCheckbox.text=Flag previously seen devices diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java index 95491f9cad..cb3de00d3b 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java @@ -65,6 +65,7 @@ import org.sleuthkit.datamodel.SleuthkitCase; final class IngestModule implements FileIngestModule { static final boolean DEFAULT_FLAG_TAGGED_NOTABLE_ITEMS = true; + static final boolean DEFAULT_FLAG_PREVIOUS_DEVICES = true; private final static Logger logger = Logger.getLogger(IngestModule.class.getName()); private final IngestServices services = IngestServices.getInstance(); @@ -76,6 +77,7 @@ final class IngestModule implements FileIngestModule { private Blackboard blackboard; private CorrelationAttributeInstance.Type filesType; private final boolean flagTaggedNotableItems; + private final boolean flagPreviouslySeenDevices; /** * Instantiate the Correlation Engine ingest module. @@ -84,6 +86,7 @@ final class IngestModule implements FileIngestModule { */ IngestModule(IngestSettings settings) { flagTaggedNotableItems = settings.isFlagTaggedNotableItems(); + flagPreviouslySeenDevices = settings.isFlagPreviousDevices(); } @Override @@ -232,6 +235,9 @@ final class IngestModule implements FileIngestModule { if (IngestEventsListener.getCeModuleInstanceCount() == 1 || !IngestEventsListener.isFlagNotableItems()) { IngestEventsListener.setFlagNotableItems(flagTaggedNotableItems); } + if (IngestEventsListener.getCeModuleInstanceCount() == 1 || !IngestEventsListener.isFlagSeenDevices()) { + IngestEventsListener.setFlagSeenDevices(flagPreviouslySeenDevices); + } if (EamDb.isEnabled() == false) { /* diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettings.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettings.java index 32ab9e9f2d..5a0580adf4 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettings.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettings.java @@ -28,21 +28,25 @@ final class IngestSettings implements IngestModuleIngestJobSettings { private static final long serialVersionUID = 1L; private boolean flagTaggedNotableItems; + private boolean flagPreviousDevices; /** * Instantiate the ingest job settings with default values. */ IngestSettings() { this.flagTaggedNotableItems = IngestModule.DEFAULT_FLAG_TAGGED_NOTABLE_ITEMS; + this.flagPreviousDevices = IngestModule.DEFAULT_FLAG_PREVIOUS_DEVICES; } /** * Instantiate the ingest job settings. * * @param flagTaggedNotableItems Flag previously tagged notable items. + * @param flagPreviousDevices Flag devices which exist already in the Central Repository */ - IngestSettings(boolean flagTaggedNotableItems) { + IngestSettings(boolean flagTaggedNotableItems, boolean flagPreviousDevices) { this.flagTaggedNotableItems = flagTaggedNotableItems; + this.flagPreviousDevices = flagPreviousDevices; } @Override @@ -60,12 +64,11 @@ final class IngestSettings implements IngestModuleIngestJobSettings { } /** - * Flag or ignore previously identified notable items. + * Are previously seen devices to be flagged? * - * @param ignorePreviousNotableItems Are previously tagged notable items to - * be flagged? + * @return True if flagging; otherwise false. */ - void setFlagTaggedNotableItems(boolean flagTaggedNotableItems) { - this.flagTaggedNotableItems = flagTaggedNotableItems; + boolean isFlagPreviousDevices() { + return flagPreviousDevices; } } diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.form b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.form index 564031cb72..3c2fddca0f 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.form +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.form @@ -19,11 +19,14 @@ - - - - + + + + + + + @@ -36,7 +39,9 @@ - + + + @@ -59,5 +64,12 @@ + + + + + + + diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.java index ed36c71287..159f925355 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.java @@ -42,11 +42,12 @@ final class IngestSettingsPanel extends IngestModuleIngestJobSettingsPanel { */ private void customizeComponents(IngestSettings settings) { flagTaggedNotableItemsCheckbox.setSelected(settings.isFlagTaggedNotableItems()); + flagPreviouslySeenDevicesCheckbox.setSelected(settings.isFlagPreviousDevices()); } @Override public IngestModuleIngestJobSettings getSettings() { - return new IngestSettings(flagTaggedNotableItemsCheckbox.isSelected()); + return new IngestSettings(flagTaggedNotableItemsCheckbox.isSelected(), flagPreviouslySeenDevicesCheckbox.isSelected()); } /** @@ -60,12 +61,15 @@ final class IngestSettingsPanel extends IngestModuleIngestJobSettingsPanel { ingestSettingsLabel = new javax.swing.JLabel(); flagTaggedNotableItemsCheckbox = new javax.swing.JCheckBox(); + flagPreviouslySeenDevicesCheckbox = new javax.swing.JCheckBox(); ingestSettingsLabel.setFont(new java.awt.Font("Tahoma", 1, 11)); // NOI18N org.openide.awt.Mnemonics.setLocalizedText(ingestSettingsLabel, org.openide.util.NbBundle.getMessage(IngestSettingsPanel.class, "IngestSettingsPanel.ingestSettingsLabel.text")); // NOI18N org.openide.awt.Mnemonics.setLocalizedText(flagTaggedNotableItemsCheckbox, org.openide.util.NbBundle.getMessage(IngestSettingsPanel.class, "IngestSettingsPanel.flagTaggedNotableItemsCheckbox.text")); // NOI18N + org.openide.awt.Mnemonics.setLocalizedText(flagPreviouslySeenDevicesCheckbox, org.openide.util.NbBundle.getMessage(IngestSettingsPanel.class, "IngestSettingsPanel.flagPreviouslySeenDevicesCheckbox.text")); // NOI18N + javax.swing.GroupLayout layout = new javax.swing.GroupLayout(this); this.setLayout(layout); layout.setHorizontalGroup( @@ -73,10 +77,12 @@ final class IngestSettingsPanel extends IngestModuleIngestJobSettingsPanel { .addGroup(layout.createSequentialGroup() .addContainerGap() .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addComponent(ingestSettingsLabel) .addGroup(layout.createSequentialGroup() .addGap(10, 10, 10) - .addComponent(flagTaggedNotableItemsCheckbox)) - .addComponent(ingestSettingsLabel)) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addComponent(flagPreviouslySeenDevicesCheckbox) + .addComponent(flagTaggedNotableItemsCheckbox)))) .addContainerGap(65, Short.MAX_VALUE)) ); layout.setVerticalGroup( @@ -86,11 +92,14 @@ final class IngestSettingsPanel extends IngestModuleIngestJobSettingsPanel { .addComponent(ingestSettingsLabel) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) .addComponent(flagTaggedNotableItemsCheckbox) - .addContainerGap(245, Short.MAX_VALUE)) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addComponent(flagPreviouslySeenDevicesCheckbox) + .addContainerGap(222, Short.MAX_VALUE)) ); }// //GEN-END:initComponents // Variables declaration - do not modify//GEN-BEGIN:variables + private javax.swing.JCheckBox flagPreviouslySeenDevicesCheckbox; private javax.swing.JCheckBox flagTaggedNotableItemsCheckbox; private javax.swing.JLabel ingestSettingsLabel; // End of variables declaration//GEN-END:variables diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonAttributePanel.form b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonAttributePanel.form index 3c712368ef..02353d8835 100644 --- a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonAttributePanel.form +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonAttributePanel.form @@ -58,7 +58,7 @@ - + @@ -77,7 +77,7 @@ - + diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonAttributePanel.java b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonAttributePanel.java index f73eaf020f..9ad914a468 100644 --- a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonAttributePanel.java +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonAttributePanel.java @@ -18,6 +18,8 @@ */ package org.sleuthkit.autopsy.commonfilesearch; +import org.sleuthkit.autopsy.guiutils.DataSourceComboBoxModel; +import org.sleuthkit.autopsy.guiutils.DataSourceLoader; import java.awt.Dimension; import java.sql.SQLException; import java.util.ArrayList; diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/InterCasePanel.java b/Core/src/org/sleuthkit/autopsy/commonfilesearch/InterCasePanel.java index 22454979fb..2cb94571ae 100644 --- a/Core/src/org/sleuthkit/autopsy/commonfilesearch/InterCasePanel.java +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/InterCasePanel.java @@ -19,6 +19,7 @@ */ package org.sleuthkit.autopsy.commonfilesearch; +import org.sleuthkit.autopsy.guiutils.DataSourceComboBoxModel; import java.util.Collections; import java.util.HashMap; import java.util.List; diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/IntraCasePanel.java b/Core/src/org/sleuthkit/autopsy/commonfilesearch/IntraCasePanel.java index 4841d39e84..fd8f9becac 100644 --- a/Core/src/org/sleuthkit/autopsy/commonfilesearch/IntraCasePanel.java +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/IntraCasePanel.java @@ -19,6 +19,7 @@ */ package org.sleuthkit.autopsy.commonfilesearch; +import org.sleuthkit.autopsy.guiutils.DataSourceComboBoxModel; import java.util.Collections; import java.util.HashMap; import java.util.Map; diff --git a/Core/src/org/sleuthkit/autopsy/core/layer.xml b/Core/src/org/sleuthkit/autopsy/core/layer.xml index 8bdde0f317..3c87a3df9f 100644 --- a/Core/src/org/sleuthkit/autopsy/core/layer.xml +++ b/Core/src/org/sleuthkit/autopsy/core/layer.xml @@ -337,6 +337,11 @@ + + + + +