From 2669e2cc60528c20797440a8a5b3998bdcd25682 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Mon, 26 Feb 2018 11:49:26 -0500 Subject: [PATCH 01/20] Initial changes. --- .../ingestmodule/Bundle.properties | 2 + .../ingestmodule/IngestModule.java | 46 ++++---- .../ingestmodule/IngestModuleFactory.java | 28 ++++- .../ingestmodule/IngestSettings.java | 71 ++++++++++++ .../ingestmodule/IngestSettingsPanel.form | 70 ++++++++++++ .../ingestmodule/IngestSettingsPanel.java | 101 ++++++++++++++++++ 6 files changed, 297 insertions(+), 21 deletions(-) create mode 100755 Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/Bundle.properties create mode 100755 Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettings.java create mode 100755 Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.form create mode 100755 Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.java diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/Bundle.properties b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/Bundle.properties new file mode 100755 index 0000000000..c903c40421 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/Bundle.properties @@ -0,0 +1,2 @@ +IngestSettingsPanel.ingestSettingsLabel.text=Ingest Settings +IngestSettingsPanel.ignorePreviousNotableItemsCheckbox.text=Ignore previously seen notable items. diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java index 09f3c63449..b3bf0505e6 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java @@ -1,7 +1,7 @@ /* * Central Repository * - * Copyright 2011-2017 Basis Technology Corp. + * Copyright 2011-2018 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -55,9 +55,11 @@ import org.sleuthkit.autopsy.centralrepository.eventlisteners.IngestEventsListen */ @Messages({"IngestModule.prevTaggedSet.text=Previously Tagged As Notable (Central Repository)", "IngestModule.prevCaseComment.text=Previous Case: "}) -class IngestModule implements FileIngestModule { +final class IngestModule implements FileIngestModule { - private final static Logger LOGGER = Logger.getLogger(IngestModule.class.getName()); + static final boolean DEFAULT_IGNORE_PREVIOUS_NOTABLE_ITEMS = false; + + private final static Logger logger = Logger.getLogger(IngestModule.class.getName()); private final IngestServices services = IngestServices.getInstance(); private static final IngestModuleReferenceCounter refCounter = new IngestModuleReferenceCounter(); private static final IngestModuleReferenceCounter warningMsgRefCounter = new IngestModuleReferenceCounter(); @@ -66,6 +68,12 @@ class IngestModule implements FileIngestModule { private CorrelationDataSource eamDataSource; private Blackboard blackboard; private CorrelationAttribute.Type filesType; + + private final boolean ignorePreviousNotableItems; + + IngestModule(IngestSettings settings) { + ignorePreviousNotableItems = settings.isIgnorePreviousNotableItems(); + } @Override public ProcessResult process(AbstractFile af) { @@ -89,7 +97,7 @@ class IngestModule implements FileIngestModule { try { dbManager = EamDb.getInstance(); } catch (EamDbException ex) { - LOGGER.log(Level.SEVERE, "Error connecting to Central Repository database.", ex); + logger.log(Level.SEVERE, "Error connecting to Central Repository database.", ex); return ProcessResult.ERROR; } @@ -113,7 +121,7 @@ class IngestModule implements FileIngestModule { postCorrelatedBadFileToBlackboard(af, caseDisplayNames); } } catch (EamDbException ex) { - LOGGER.log(Level.SEVERE, "Error searching database for artifact.", ex); // NON-NLS + logger.log(Level.SEVERE, "Error searching database for artifact.", ex); // NON-NLS return ProcessResult.ERROR; } } @@ -131,7 +139,7 @@ class IngestModule implements FileIngestModule { eamArtifact.addInstance(cefi); dbManager.prepareBulkArtifact(eamArtifact); } catch (EamDbException ex) { - LOGGER.log(Level.SEVERE, "Error adding artifact to bulk artifacts.", ex); // NON-NLS + logger.log(Level.SEVERE, "Error adding artifact to bulk artifacts.", ex); // NON-NLS return ProcessResult.ERROR; } @@ -148,19 +156,19 @@ class IngestModule implements FileIngestModule { try { dbManager = EamDb.getInstance(); } catch (EamDbException ex) { - LOGGER.log(Level.SEVERE, "Error connecting to Central Repository database.", ex); + logger.log(Level.SEVERE, "Error connecting to Central Repository database.", ex); return; } try { dbManager.bulkInsertArtifacts(); } catch (EamDbException ex) { - LOGGER.log(Level.SEVERE, "Error doing bulk insert of artifacts.", ex); // NON-NLS + logger.log(Level.SEVERE, "Error doing bulk insert of artifacts.", ex); // NON-NLS } try { Long count = dbManager.getCountArtifactInstancesByCaseDataSource(eamCase.getCaseUUID(), eamDataSource.getDeviceID()); - LOGGER.log(Level.INFO, "{0} artifacts in db for case: {1} ds:{2}", new Object[]{count, eamCase.getDisplayName(), eamDataSource.getName()}); // NON-NLS + logger.log(Level.INFO, "{0} artifacts in db for case: {1} ds:{2}", new Object[]{count, eamCase.getDisplayName(), eamDataSource.getName()}); // NON-NLS } catch (EamDbException ex) { - LOGGER.log(Level.SEVERE, "Error counting artifacts.", ex); // NON-NLS + logger.log(Level.SEVERE, "Error counting artifacts.", ex); // NON-NLS } // TODO: once we implement shared cache, if refCounter is 1, then submit data in bulk. @@ -193,7 +201,7 @@ class IngestModule implements FileIngestModule { // Don't allow sqlite central repo databases to be used for multi user cases if ((Case.getCurrentCase().getCaseType() == Case.CaseType.MULTI_USER_CASE) && (EamDbPlatformEnum.getSelectedPlatform() == EamDbPlatformEnum.SQLITE)) { - LOGGER.log(Level.SEVERE, "Cannot run correlation engine on a multi-user case with a SQLite central repository."); + logger.log(Level.SEVERE, "Cannot run correlation engine on a multi-user case with a SQLite central repository."); throw new IngestModuleException("Cannot run on a multi-user case with a SQLite central repository."); // NON-NLS } jobId = context.getJobId(); @@ -202,14 +210,14 @@ class IngestModule implements FileIngestModule { try { centralRepoDb = EamDb.getInstance(); } catch (EamDbException ex) { - LOGGER.log(Level.SEVERE, "Error connecting to central repository database.", ex); // NON-NLS + logger.log(Level.SEVERE, "Error connecting to central repository database.", ex); // NON-NLS throw new IngestModuleException("Error connecting to central repository database.", ex); // NON-NLS } try { filesType = centralRepoDb.getCorrelationTypeById(CorrelationAttribute.FILES_TYPE_ID); } catch (EamDbException ex) { - LOGGER.log(Level.SEVERE, "Error getting correlation type FILES in ingest module start up.", ex); // NON-NLS + logger.log(Level.SEVERE, "Error getting correlation type FILES in ingest module start up.", ex); // NON-NLS throw new IngestModuleException("Error getting correlation type FILES in ingest module start up.", ex); // NON-NLS } Case autopsyCase = Case.getCurrentCase(); @@ -223,7 +231,7 @@ class IngestModule implements FileIngestModule { try { eamCase = centralRepoDb.newCase(autopsyCase); } catch (EamDbException ex) { - LOGGER.log(Level.SEVERE, "Error creating new case in ingest module start up.", ex); // NON-NLS + logger.log(Level.SEVERE, "Error creating new case in ingest module start up.", ex); // NON-NLS throw new IngestModuleException("Error creating new case in ingest module start up.", ex); // NON-NLS } } @@ -231,7 +239,7 @@ class IngestModule implements FileIngestModule { try { eamDataSource = CorrelationDataSource.fromTSKDataSource(eamCase, context.getDataSource()); } catch (EamDbException ex) { - LOGGER.log(Level.SEVERE, "Error getting data source info.", ex); // NON-NLS + logger.log(Level.SEVERE, "Error getting data source info.", ex); // NON-NLS throw new IngestModuleException("Error getting data source info.", ex); // NON-NLS } // TODO: once we implement a shared cache, load/init it here w/ syncronized and define reference counter @@ -245,7 +253,7 @@ class IngestModule implements FileIngestModule { centralRepoDb.newDataSource(eamDataSource); } } catch (EamDbException ex) { - LOGGER.log(Level.SEVERE, "Error adding data source to Central Repository.", ex); // NON-NLS + logger.log(Level.SEVERE, "Error adding data source to Central Repository.", ex); // NON-NLS throw new IngestModuleException("Error adding data source to Central Repository.", ex); // NON-NLS } @@ -268,7 +276,7 @@ class IngestModule implements FileIngestModule { // index the artifact for keyword search blackboard.indexArtifact(tifArtifact); } catch (Blackboard.BlackboardException ex) { - LOGGER.log(Level.SEVERE, "Unable to index blackboard artifact " + tifArtifact.getArtifactID(), ex); //NON-NLS + logger.log(Level.SEVERE, "Unable to index blackboard artifact " + tifArtifact.getArtifactID(), ex); //NON-NLS } // send inbox message @@ -277,9 +285,9 @@ class IngestModule implements FileIngestModule { // fire event to notify UI of this new artifact services.fireModuleDataEvent(new ModuleDataEvent(MODULE_NAME, BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT)); } catch (TskCoreException ex) { - LOGGER.log(Level.SEVERE, "Failed to create BlackboardArtifact.", ex); // NON-NLS + logger.log(Level.SEVERE, "Failed to create BlackboardArtifact.", ex); // NON-NLS } catch (IllegalStateException ex) { - LOGGER.log(Level.SEVERE, "Failed to create BlackboardAttribute.", ex); // NON-NLS + logger.log(Level.SEVERE, "Failed to create BlackboardAttribute.", ex); // NON-NLS } } diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModuleFactory.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModuleFactory.java index ed3d4f0915..a0ec1f4329 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModuleFactory.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModuleFactory.java @@ -1,7 +1,7 @@ /* * Central Repository * - * Copyright 2015-2017 Basis Technology Corp. + * Copyright 2015-2018 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -25,6 +25,7 @@ import org.sleuthkit.autopsy.ingest.IngestModuleFactoryAdapter; import org.sleuthkit.autopsy.ingest.IngestModuleGlobalSettingsPanel; import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettings; import org.sleuthkit.autopsy.centralrepository.optionspanel.GlobalSettingsPanel; +import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettingsPanel; /** * Factory for Central Repository ingest modules @@ -34,8 +35,13 @@ import org.sleuthkit.autopsy.centralrepository.optionspanel.GlobalSettingsPanel; "IngestModuleFactory.ingestmodule.desc=Saves properties to the central repository for later correlation"}) public class IngestModuleFactory extends IngestModuleFactoryAdapter { - private static final String VERSION_NUMBER = "0.8.0"; + private static final String VERSION_NUMBER = "0.9.0"; + /** + * Get the name of the module. + * + * @return The module name. + */ static String getModuleName() { return Bundle.IngestModuleFactory_ingestmodule_name(); } @@ -76,5 +82,23 @@ public class IngestModuleFactory extends IngestModuleFactoryAdapter { globalOptionsPanel.load(); return globalOptionsPanel; } + + @Override + public IngestModuleIngestJobSettings getDefaultIngestJobSettings() { + return new IngestSettings(); + } + + @Override + public boolean hasIngestJobSettingsPanel() { + return true; + } + + @Override + public IngestModuleIngestJobSettingsPanel getIngestJobSettingsPanel(IngestModuleIngestJobSettings settings) { + if (!(settings instanceof IngestSettings)) { + throw new IllegalArgumentException("Expected settings argument to be an instance of IngestSettings"); + } + return new IngestSettingsPanel((IngestSettings) settings); + } } diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettings.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettings.java new file mode 100755 index 0000000000..e69e625b85 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettings.java @@ -0,0 +1,71 @@ +/* + * Central Repository + * + * Copyright 2018 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.centralrepository.ingestmodule; + +import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettings; + +/** + * Ingest job settings for the Correlation Engine module. + */ +final class IngestSettings implements IngestModuleIngestJobSettings { + + private static final long serialVersionUID = 1L; + + private boolean ignorePreviousNotableItems; + + /** + * Instantiate the ingest job settings with default values. + */ + IngestSettings() { + this.ignorePreviousNotableItems = IngestModule.DEFAULT_IGNORE_PREVIOUS_NOTABLE_ITEMS; + } + + /** + * Instantiate the ingest job settings. + * + * @param ignorePreviousNotableItems Ignore previously seen notable items. + */ + IngestSettings(boolean ignorePreviousNotableItems) { + this.ignorePreviousNotableItems = ignorePreviousNotableItems; + } + + @Override + public long getVersionNumber() { + return serialVersionUID; + } + + /** + * Are previously identified notable items ignored? + * + * @return True if ignored; otherwise false. + */ + boolean isIgnorePreviousNotableItems() { + return ignorePreviousNotableItems; + } + + /** + * Consider or ignore previously identified notable items. + * + * @param ignorePreviousNotableItems Are previously identified notable items + * ignored? + */ + void setIgnorePreviousNotableItems(boolean ignorePreviousNotableItems) { + this.ignorePreviousNotableItems = ignorePreviousNotableItems; + } +} diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.form b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.form new file mode 100755 index 0000000000..c60abee00d --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.form @@ -0,0 +1,70 @@ + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.java new file mode 100755 index 0000000000..77e04d1528 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.java @@ -0,0 +1,101 @@ +/* + * Central Repository + * + * Copyright 2018 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.centralrepository.ingestmodule; + +import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettings; +import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettingsPanel; + +/** + * Ingest job settings panel for the Correlation Engine module. + */ +final class IngestSettingsPanel extends IngestModuleIngestJobSettingsPanel { + + /** + * Creates new form IngestModulePanel + */ + public IngestSettingsPanel(IngestSettings settings) { + initComponents(); + customizeComponents(settings); + } + + /** + * Update components with values from the ingest job settings. + * + * @param settings The ingest job settings. + */ + private void customizeComponents(IngestSettings settings) { + ignorePreviousNotableItemsCheckbox.setSelected(settings.isIgnorePreviousNotableItems()); + } + + @Override + public IngestModuleIngestJobSettings getSettings() { + return new IngestSettings(ignorePreviousNotableItemsCheckbox.isSelected()); + } + + /** + * This method is called from within the constructor to initialize the form. + * WARNING: Do NOT modify this code. The content of this method is always + * regenerated by the Form Editor. + */ + @SuppressWarnings("unchecked") + // //GEN-BEGIN:initComponents + private void initComponents() { + + ingestSettingsLabel = new javax.swing.JLabel(); + ignorePreviousNotableItemsCheckbox = new javax.swing.JCheckBox(); + + setDefaultCloseOperation(javax.swing.WindowConstants.EXIT_ON_CLOSE); + + ingestSettingsLabel.setFont(new java.awt.Font("Tahoma", 1, 11)); // NOI18N + org.openide.awt.Mnemonics.setLocalizedText(ingestSettingsLabel, org.openide.util.NbBundle.getMessage(IngestSettingsPanel.class, "IngestSettingsPanel.ingestSettingsLabel.text")); // NOI18N + + org.openide.awt.Mnemonics.setLocalizedText(ignorePreviousNotableItemsCheckbox, org.openide.util.NbBundle.getMessage(IngestSettingsPanel.class, "IngestSettingsPanel.ignorePreviousNotableItemsCheckbox.text")); // NOI18N + + javax.swing.GroupLayout layout = new javax.swing.GroupLayout(getContentPane()); + getContentPane().setLayout(layout); + layout.setHorizontalGroup( + layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addGroup(layout.createSequentialGroup() + .addContainerGap() + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addGroup(layout.createSequentialGroup() + .addGap(10, 10, 10) + .addComponent(ignorePreviousNotableItemsCheckbox)) + .addComponent(ingestSettingsLabel)) + .addContainerGap(83, Short.MAX_VALUE)) + ); + layout.setVerticalGroup( + layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addGroup(layout.createSequentialGroup() + .addContainerGap() + .addComponent(ingestSettingsLabel) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) + .addComponent(ignorePreviousNotableItemsCheckbox) + .addContainerGap(245, Short.MAX_VALUE)) + ); + + pack(); + }// //GEN-END:initComponents + + // Variables declaration - do not modify//GEN-BEGIN:variables + private javax.swing.JCheckBox ignorePreviousNotableItemsCheckbox; + private javax.swing.JLabel ingestSettingsLabel; + // End of variables declaration//GEN-END:variables + +} From bd1f6346a13f8db591295db8a8603eea3ddf6d6a Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Mon, 26 Feb 2018 13:13:19 -0500 Subject: [PATCH 02/20] Added flag toggle. --- .../ingestmodule/IngestModule.java | 27 ++++++++++++++----- 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java index b3bf0505e6..49a89555a0 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java @@ -23,10 +23,12 @@ import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbException; import java.util.List; import java.util.logging.Level; import java.util.stream.Collectors; +import org.openide.util.Exceptions; import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.services.Blackboard; +import org.sleuthkit.autopsy.casemodule.services.TagsManager; import org.sleuthkit.autopsy.core.RuntimeProperties; import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; import org.sleuthkit.autopsy.ingest.FileIngestModule; @@ -48,6 +50,7 @@ import org.sleuthkit.datamodel.HashUtility; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskData; import org.sleuthkit.autopsy.centralrepository.eventlisteners.IngestEventsListener; +import org.sleuthkit.datamodel.ContentTag; /** * Ingest module for inserting entries into the Central Repository database on @@ -76,7 +79,7 @@ final class IngestModule implements FileIngestModule { } @Override - public ProcessResult process(AbstractFile af) { + public ProcessResult process(AbstractFile abstractFile) { if (EamDb.isEnabled() == false) { /* * Not signaling an error for now. This is a workaround for the way @@ -86,10 +89,22 @@ final class IngestModule implements FileIngestModule { */ return ProcessResult.OK; } + + if(ignorePreviousNotableItems) { //DLG: + CorrelationAttribute attribute = EamArtifactUtil.getCorrelationAttributeFromContent(abstractFile, TskData.FileKnown.BAD, null); //DLG: + //DLG: try { + //DLG: List contentTagsList = Case.getCurrentCase().getServices().getTagsManager().getContentTagsByContent(abstractFile); + //DLG: ContentTag tag = contentTagsList.get(0); + //DLG: tag.getId(); + //DLG: } catch (TskCoreException ex) { + //DLG: Exceptions.printStackTrace(ex); //DLG: + //DLG: return ProcessResult.ERROR; + //DLG: } + } //DLG: blackboard = Case.getCurrentCase().getServices().getBlackboard(); - if (!EamArtifactUtil.isValidCentralRepoFile(af)) { + if (!EamArtifactUtil.isValidCentralRepoFile(abstractFile)) { return ProcessResult.OK; } @@ -107,18 +122,18 @@ final class IngestModule implements FileIngestModule { } // get the hash because we're going to correlate it - String md5 = af.getMd5Hash(); + String md5 = abstractFile.getMd5Hash(); if ((md5 == null) || (HashUtility.isNoDataMd5(md5))) { return ProcessResult.OK; } /* Search the central repo to see if this file was previously * marked as being bad. Create artifact if it was. */ - if (af.getKnown() != TskData.FileKnown.KNOWN) { + if (abstractFile.getKnown() != TskData.FileKnown.KNOWN && !ignorePreviousNotableItems) { try { List caseDisplayNames = dbManager.getListCasesHavingArtifactInstancesKnownBad(filesType, md5); if (!caseDisplayNames.isEmpty()) { - postCorrelatedBadFileToBlackboard(af, caseDisplayNames); + postCorrelatedBadFileToBlackboard(abstractFile, caseDisplayNames); } } catch (EamDbException ex) { logger.log(Level.SEVERE, "Error searching database for artifact.", ex); // NON-NLS @@ -132,7 +147,7 @@ final class IngestModule implements FileIngestModule { CorrelationAttributeInstance cefi = new CorrelationAttributeInstance( eamCase, eamDataSource, - af.getParentPath() + af.getName(), + abstractFile.getParentPath() + abstractFile.getName(), null, TskData.FileKnown.UNKNOWN // NOTE: Known status in the CR is based on tagging, not hashes like the Case Database. ); From a569657a05e09f01a3cd795c2ba5a937d8f6dacb Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Mon, 26 Feb 2018 13:53:49 -0500 Subject: [PATCH 03/20] Fixed compile issues with panel. --- .../ingestmodule/IngestModuleFactory.java | 4 ++-- .../ingestmodule/IngestSettingsPanel.form | 7 ------- .../ingestmodule/IngestSettingsPanel.java | 8 ++------ 3 files changed, 4 insertions(+), 15 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModuleFactory.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModuleFactory.java index a0ec1f4329..e9c9a5a88f 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModuleFactory.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModuleFactory.java @@ -67,8 +67,8 @@ public class IngestModuleFactory extends IngestModuleFactoryAdapter { } @Override - public FileIngestModule createFileIngestModule(IngestModuleIngestJobSettings ingestOptions) { - return new IngestModule(); + public FileIngestModule createFileIngestModule(IngestModuleIngestJobSettings settings) { + return new IngestModule((IngestSettings) settings); } @Override diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.form b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.form index c60abee00d..fbcf49c00d 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.form +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.form @@ -1,13 +1,6 @@
- - - - - - - diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.java index 77e04d1528..e7b559e0e5 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.java @@ -60,15 +60,13 @@ final class IngestSettingsPanel extends IngestModuleIngestJobSettingsPanel { ingestSettingsLabel = new javax.swing.JLabel(); ignorePreviousNotableItemsCheckbox = new javax.swing.JCheckBox(); - setDefaultCloseOperation(javax.swing.WindowConstants.EXIT_ON_CLOSE); - ingestSettingsLabel.setFont(new java.awt.Font("Tahoma", 1, 11)); // NOI18N org.openide.awt.Mnemonics.setLocalizedText(ingestSettingsLabel, org.openide.util.NbBundle.getMessage(IngestSettingsPanel.class, "IngestSettingsPanel.ingestSettingsLabel.text")); // NOI18N org.openide.awt.Mnemonics.setLocalizedText(ignorePreviousNotableItemsCheckbox, org.openide.util.NbBundle.getMessage(IngestSettingsPanel.class, "IngestSettingsPanel.ignorePreviousNotableItemsCheckbox.text")); // NOI18N - javax.swing.GroupLayout layout = new javax.swing.GroupLayout(getContentPane()); - getContentPane().setLayout(layout); + javax.swing.GroupLayout layout = new javax.swing.GroupLayout(this); + this.setLayout(layout); layout.setHorizontalGroup( layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addGroup(layout.createSequentialGroup() @@ -89,8 +87,6 @@ final class IngestSettingsPanel extends IngestModuleIngestJobSettingsPanel { .addComponent(ignorePreviousNotableItemsCheckbox) .addContainerGap(245, Short.MAX_VALUE)) ); - - pack(); }// //GEN-END:initComponents // Variables declaration - do not modify//GEN-BEGIN:variables From a518eb721c879e9e51d1c5efa0340b3e50dcca42 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Tue, 27 Feb 2018 15:21:34 -0500 Subject: [PATCH 04/20] Cleanup. --- .../ingestmodule/IngestModule.java | 26 ++++++------------- 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java index 49a89555a0..041c6b9d85 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java @@ -71,9 +71,9 @@ final class IngestModule implements FileIngestModule { private CorrelationDataSource eamDataSource; private Blackboard blackboard; private CorrelationAttribute.Type filesType; - + private final boolean ignorePreviousNotableItems; - + IngestModule(IngestSettings settings) { ignorePreviousNotableItems = settings.isIgnorePreviousNotableItems(); } @@ -89,18 +89,6 @@ final class IngestModule implements FileIngestModule { */ return ProcessResult.OK; } - - if(ignorePreviousNotableItems) { //DLG: - CorrelationAttribute attribute = EamArtifactUtil.getCorrelationAttributeFromContent(abstractFile, TskData.FileKnown.BAD, null); //DLG: - //DLG: try { - //DLG: List contentTagsList = Case.getCurrentCase().getServices().getTagsManager().getContentTagsByContent(abstractFile); - //DLG: ContentTag tag = contentTagsList.get(0); - //DLG: tag.getId(); - //DLG: } catch (TskCoreException ex) { - //DLG: Exceptions.printStackTrace(ex); //DLG: - //DLG: return ProcessResult.ERROR; - //DLG: } - } //DLG: blackboard = Case.getCurrentCase().getServices().getBlackboard(); @@ -127,8 +115,10 @@ final class IngestModule implements FileIngestModule { return ProcessResult.OK; } - /* Search the central repo to see if this file was previously - * marked as being bad. Create artifact if it was. */ + /* + * Search the central repo to see if this file was previously marked as + * being bad. Create artifact if it was. + */ if (abstractFile.getKnown() != TskData.FileKnown.KNOWN && !ignorePreviousNotableItems) { try { List caseDisplayNames = dbManager.getListCasesHavingArtifactInstancesKnownBad(filesType, md5); @@ -149,7 +139,7 @@ final class IngestModule implements FileIngestModule { eamDataSource, abstractFile.getParentPath() + abstractFile.getName(), null, - TskData.FileKnown.UNKNOWN // NOTE: Known status in the CR is based on tagging, not hashes like the Case Database. + TskData.FileKnown.UNKNOWN // NOTE: Known status in the CR is based on tagging, not hashes like the Case Database. ); eamArtifact.addInstance(cefi); dbManager.prepareBulkArtifact(eamArtifact); @@ -250,7 +240,7 @@ final class IngestModule implements FileIngestModule { throw new IngestModuleException("Error creating new case in ingest module start up.", ex); // NON-NLS } } - + try { eamDataSource = CorrelationDataSource.fromTSKDataSource(eamCase, context.getDataSource()); } catch (EamDbException ex) { From 7553c3418c5a41794195fa09a2b24754a7a29ae9 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Tue, 27 Feb 2018 16:16:21 -0500 Subject: [PATCH 05/20] Cleanup. --- .../autopsy/centralrepository/ingestmodule/IngestModule.java | 3 --- .../centralrepository/ingestmodule/IngestSettingsPanel.java | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java index 041c6b9d85..829f0b006e 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java @@ -23,12 +23,10 @@ import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbException; import java.util.List; import java.util.logging.Level; import java.util.stream.Collectors; -import org.openide.util.Exceptions; import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.services.Blackboard; -import org.sleuthkit.autopsy.casemodule.services.TagsManager; import org.sleuthkit.autopsy.core.RuntimeProperties; import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; import org.sleuthkit.autopsy.ingest.FileIngestModule; @@ -50,7 +48,6 @@ import org.sleuthkit.datamodel.HashUtility; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskData; import org.sleuthkit.autopsy.centralrepository.eventlisteners.IngestEventsListener; -import org.sleuthkit.datamodel.ContentTag; /** * Ingest module for inserting entries into the Central Repository database on diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.java index e7b559e0e5..f7afdec94e 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.java @@ -27,7 +27,7 @@ import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettingsPanel; final class IngestSettingsPanel extends IngestModuleIngestJobSettingsPanel { /** - * Creates new form IngestModulePanel + * Creates new form IngestSettingsPanel */ public IngestSettingsPanel(IngestSettings settings) { initComponents(); From 32209514efd13b34421dbd4da836d718df493cc1 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Mon, 5 Mar 2018 10:13:22 -0500 Subject: [PATCH 06/20] Revised. --- .../eventlisteners/CaseEventListener.java | 4 +- .../eventlisteners/IngestEventsListener.java | 60 +++++++++++++++---- .../ingestmodule/Bundle.properties | 2 +- .../ingestmodule/IngestModule.java | 22 +++++-- .../ingestmodule/IngestSettings.java | 28 ++++----- .../ingestmodule/IngestSettingsPanel.form | 12 ++-- .../ingestmodule/IngestSettingsPanel.java | 16 ++--- 7 files changed, 97 insertions(+), 47 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/CaseEventListener.java b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/CaseEventListener.java index b053d9df17..807cfef4bd 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/CaseEventListener.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/CaseEventListener.java @@ -1,7 +1,7 @@ /* * Central Repository * - * Copyright 2015-2017 Basis Technology Corp. + * Copyright 2015-2018 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -41,7 +41,6 @@ import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationCase; import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationDataSource; import org.sleuthkit.autopsy.centralrepository.datamodel.EamDb; import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbException; -import org.sleuthkit.autopsy.centralrepository.datamodel.EamOrganization; import org.sleuthkit.autopsy.coreutils.ThreadUtils; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.BlackboardArtifact; @@ -464,6 +463,7 @@ final class CaseEventListener implements PropertyChangeListener { if ((null == event.getOldValue()) && (event.getNewValue() instanceof Case)) { Case curCase = (Case) event.getNewValue(); IngestEventsListener.resetCeModuleInstanceCount(); + IngestEventsListener.resetCorrelationModulesFlaggingNotableCount(); if (!EamDb.isEnabled()) { return; diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java index 0877bc1685..2f1efd3785 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java @@ -1,7 +1,7 @@ /* * Central Repository * - * Copyright 2015-2017 Basis Technology Corp. + * Copyright 2015-2018 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -56,7 +56,8 @@ public class IngestEventsListener { private static final Logger LOGGER = Logger.getLogger(CorrelationAttribute.class.getName()); final Collection recentlyAddedCeArtifacts = new LinkedHashSet<>(); - private static int ceModuleInstanceCount = 0; + private static int correlationModuleInstanceCount = 0; + private static int correlationModulesFlaggingNotableCount = 0; private final ExecutorService jobProcessingExecutor; private static final String INGEST_EVENT_THREAD_NAME = "Ingest-Event-Listener-%d"; private final PropertyChangeListener pcl1 = new IngestModuleEventListener(); @@ -87,21 +88,20 @@ public class IngestEventsListener { } /** - * Enable this IngestEventsListener to add contents to the Correlation - * Engine. - * + * Increase the number of IngestEventsListeners adding contents to the + * Correlation Engine. */ public synchronized static void incrementCorrelationEngineModuleCount() { - ceModuleInstanceCount++; //Should be called once in the Correlation Engine module's startup method. + correlationModuleInstanceCount++; //Should be called once in the Correlation Engine module's startup method. } /** - * Disable this IngestEventsListener from adding contents to the Correlation - * Engine. + * Decrease the number of IngestEventsListeners adding contents to the + * Correlation Engine. */ public synchronized static void decrementCorrelationEngineModuleCount() { if (getCeModuleInstanceCount() > 0) { //prevent it ingestJobCounter from going negative - ceModuleInstanceCount--; //Should be called once in the Correlation Engine module's shutdown method. + correlationModuleInstanceCount--; //Should be called once in the Correlation Engine module's shutdown method. } } @@ -110,7 +110,7 @@ public class IngestEventsListener { * is being run during injest to 0. */ synchronized static void resetCeModuleInstanceCount() { - ceModuleInstanceCount = 0; //called when a case is opened in case for some reason counter was not reset + correlationModuleInstanceCount = 0; //called when a case is opened in case for some reason counter was not reset } /** @@ -120,7 +120,43 @@ public class IngestEventsListener { * @return boolean True for Correlation Engine enabled, False for disabled */ private synchronized static int getCeModuleInstanceCount() { - return ceModuleInstanceCount; + return correlationModuleInstanceCount; + } + + /** + * Increase the number of IngestEventsListeners adding contents to the + * Correlation Engine with notable item flagging enabled. + */ + public synchronized static void incrementCorrelationModulesFlaggingNotableCount() { + correlationModulesFlaggingNotableCount++; + } + + /** + * Decrease the number of IngestEventsListeners adding contents to the + * Correlation Engine with notable item flagging enabled. + */ + public synchronized static void decrementCorrelationModulesFlaggingNotableCount() { + if (correlationModulesFlaggingNotableCount > 0) { + correlationModulesFlaggingNotableCount--; + } + } + + /** + * Reset the counter which keeps track of if the Correlation Engine Module + * is being run during injest and flagging notable items to 0. + */ + synchronized static void resetCorrelationModulesFlaggingNotableCount() { + correlationModulesFlaggingNotableCount = 0; + } + + /** + * Wether or not the Correlation Engine Module is enabled for any of the + * currently running ingest jobs and flagging notable items. + * + * @return boolean True for Correlation Engine enabled, False for disabled + */ + private synchronized static int getCorrelationModulesFlaggingNotableCount() { + return correlationModulesFlaggingNotableCount; } @NbBundle.Messages({"IngestEventsListener.prevTaggedSet.text=Previously Tagged As Notable (Central Repository)", @@ -219,7 +255,7 @@ public class IngestEventsListener { @Override public void run() { - if (!EamDb.isEnabled()) { + if (!EamDb.isEnabled() || getCorrelationModulesFlaggingNotableCount() == 0) { return; } final ModuleDataEvent mde = (ModuleDataEvent) event.getOldValue(); diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/Bundle.properties b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/Bundle.properties index c903c40421..a525713f7c 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/Bundle.properties @@ -1,2 +1,2 @@ IngestSettingsPanel.ingestSettingsLabel.text=Ingest Settings -IngestSettingsPanel.ignorePreviousNotableItemsCheckbox.text=Ignore previously seen notable items. +IngestSettingsPanel.flagTaggedNotableItemsCheckbox.text=Flag items previously tagged as notable diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java index 829f0b006e..20fbf86753 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java @@ -57,7 +57,7 @@ import org.sleuthkit.autopsy.centralrepository.eventlisteners.IngestEventsListen "IngestModule.prevCaseComment.text=Previous Case: "}) final class IngestModule implements FileIngestModule { - static final boolean DEFAULT_IGNORE_PREVIOUS_NOTABLE_ITEMS = false; + static final boolean DEFAULT_FLAG_TAGGED_NOTABLE_ITEMS = true; private final static Logger logger = Logger.getLogger(IngestModule.class.getName()); private final IngestServices services = IngestServices.getInstance(); @@ -69,10 +69,14 @@ final class IngestModule implements FileIngestModule { private Blackboard blackboard; private CorrelationAttribute.Type filesType; - private final boolean ignorePreviousNotableItems; + private final boolean flagTaggedNotableItems; + /** + * //DLG: + * @param settings + */ IngestModule(IngestSettings settings) { - ignorePreviousNotableItems = settings.isIgnorePreviousNotableItems(); + flagTaggedNotableItems = settings.isFlagTaggedNotableItems(); } @Override @@ -116,7 +120,7 @@ final class IngestModule implements FileIngestModule { * Search the central repo to see if this file was previously marked as * being bad. Create artifact if it was. */ - if (abstractFile.getKnown() != TskData.FileKnown.KNOWN && !ignorePreviousNotableItems) { + if (abstractFile.getKnown() != TskData.FileKnown.KNOWN && flagTaggedNotableItems) { try { List caseDisplayNames = dbManager.getListCasesHavingArtifactInstancesKnownBad(filesType, md5); if (!caseDisplayNames.isEmpty()) { @@ -151,6 +155,11 @@ final class IngestModule implements FileIngestModule { @Override public void shutDown() { IngestEventsListener.decrementCorrelationEngineModuleCount(); + + if (flagTaggedNotableItems) { + IngestEventsListener.decrementCorrelationModulesFlaggingNotableCount(); + } + if ((EamDb.isEnabled() == false) || (eamCase == null) || (eamDataSource == null)) { return; } @@ -185,6 +194,11 @@ final class IngestModule implements FileIngestModule { @Override public void startUp(IngestJobContext context) throws IngestModuleException { IngestEventsListener.incrementCorrelationEngineModuleCount(); + + if (flagTaggedNotableItems) { + IngestEventsListener.incrementCorrelationModulesFlaggingNotableCount(); + } + if (EamDb.isEnabled() == false) { /* * Not throwing the customary exception for now. This is a diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettings.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettings.java index e69e625b85..32ab9e9f2d 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettings.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettings.java @@ -27,22 +27,22 @@ final class IngestSettings implements IngestModuleIngestJobSettings { private static final long serialVersionUID = 1L; - private boolean ignorePreviousNotableItems; + private boolean flagTaggedNotableItems; /** * Instantiate the ingest job settings with default values. */ IngestSettings() { - this.ignorePreviousNotableItems = IngestModule.DEFAULT_IGNORE_PREVIOUS_NOTABLE_ITEMS; + this.flagTaggedNotableItems = IngestModule.DEFAULT_FLAG_TAGGED_NOTABLE_ITEMS; } /** * Instantiate the ingest job settings. * - * @param ignorePreviousNotableItems Ignore previously seen notable items. + * @param flagTaggedNotableItems Flag previously tagged notable items. */ - IngestSettings(boolean ignorePreviousNotableItems) { - this.ignorePreviousNotableItems = ignorePreviousNotableItems; + IngestSettings(boolean flagTaggedNotableItems) { + this.flagTaggedNotableItems = flagTaggedNotableItems; } @Override @@ -51,21 +51,21 @@ final class IngestSettings implements IngestModuleIngestJobSettings { } /** - * Are previously identified notable items ignored? + * Are previously tagged notable items to be flagged? * - * @return True if ignored; otherwise false. + * @return True if flagging; otherwise false. */ - boolean isIgnorePreviousNotableItems() { - return ignorePreviousNotableItems; + boolean isFlagTaggedNotableItems() { + return flagTaggedNotableItems; } /** - * Consider or ignore previously identified notable items. + * Flag or ignore previously identified notable items. * - * @param ignorePreviousNotableItems Are previously identified notable items - * ignored? + * @param ignorePreviousNotableItems Are previously tagged notable items to + * be flagged? */ - void setIgnorePreviousNotableItems(boolean ignorePreviousNotableItems) { - this.ignorePreviousNotableItems = ignorePreviousNotableItems; + void setFlagTaggedNotableItems(boolean flagTaggedNotableItems) { + this.flagTaggedNotableItems = flagTaggedNotableItems; } } diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.form b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.form index fbcf49c00d..2abe207b5a 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.form +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.form @@ -1,6 +1,6 @@ - + @@ -21,11 +21,11 @@ - + - + @@ -35,7 +35,7 @@ - + @@ -52,10 +52,10 @@ - + - + diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.java index f7afdec94e..46538f41c2 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.java @@ -40,12 +40,12 @@ final class IngestSettingsPanel extends IngestModuleIngestJobSettingsPanel { * @param settings The ingest job settings. */ private void customizeComponents(IngestSettings settings) { - ignorePreviousNotableItemsCheckbox.setSelected(settings.isIgnorePreviousNotableItems()); + flagTaggedNotableItemsCheckbox.setSelected(settings.isFlagTaggedNotableItems()); } @Override public IngestModuleIngestJobSettings getSettings() { - return new IngestSettings(ignorePreviousNotableItemsCheckbox.isSelected()); + return new IngestSettings(flagTaggedNotableItemsCheckbox.isSelected()); } /** @@ -58,12 +58,12 @@ final class IngestSettingsPanel extends IngestModuleIngestJobSettingsPanel { private void initComponents() { ingestSettingsLabel = new javax.swing.JLabel(); - ignorePreviousNotableItemsCheckbox = new javax.swing.JCheckBox(); + flagTaggedNotableItemsCheckbox = new javax.swing.JCheckBox(); ingestSettingsLabel.setFont(new java.awt.Font("Tahoma", 1, 11)); // NOI18N org.openide.awt.Mnemonics.setLocalizedText(ingestSettingsLabel, org.openide.util.NbBundle.getMessage(IngestSettingsPanel.class, "IngestSettingsPanel.ingestSettingsLabel.text")); // NOI18N - org.openide.awt.Mnemonics.setLocalizedText(ignorePreviousNotableItemsCheckbox, org.openide.util.NbBundle.getMessage(IngestSettingsPanel.class, "IngestSettingsPanel.ignorePreviousNotableItemsCheckbox.text")); // NOI18N + org.openide.awt.Mnemonics.setLocalizedText(flagTaggedNotableItemsCheckbox, org.openide.util.NbBundle.getMessage(IngestSettingsPanel.class, "IngestSettingsPanel.flagTaggedNotableItemsCheckbox.text")); // NOI18N javax.swing.GroupLayout layout = new javax.swing.GroupLayout(this); this.setLayout(layout); @@ -74,9 +74,9 @@ final class IngestSettingsPanel extends IngestModuleIngestJobSettingsPanel { .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addGroup(layout.createSequentialGroup() .addGap(10, 10, 10) - .addComponent(ignorePreviousNotableItemsCheckbox)) + .addComponent(flagTaggedNotableItemsCheckbox)) .addComponent(ingestSettingsLabel)) - .addContainerGap(83, Short.MAX_VALUE)) + .addContainerGap(75, Short.MAX_VALUE)) ); layout.setVerticalGroup( layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) @@ -84,13 +84,13 @@ final class IngestSettingsPanel extends IngestModuleIngestJobSettingsPanel { .addContainerGap() .addComponent(ingestSettingsLabel) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) - .addComponent(ignorePreviousNotableItemsCheckbox) + .addComponent(flagTaggedNotableItemsCheckbox) .addContainerGap(245, Short.MAX_VALUE)) ); }// //GEN-END:initComponents // Variables declaration - do not modify//GEN-BEGIN:variables - private javax.swing.JCheckBox ignorePreviousNotableItemsCheckbox; + private javax.swing.JCheckBox flagTaggedNotableItemsCheckbox; private javax.swing.JLabel ingestSettingsLabel; // End of variables declaration//GEN-END:variables From b9255ff9c60e676a54f0a31c1345ee0291ad424c Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Tue, 6 Mar 2018 11:06:02 -0500 Subject: [PATCH 07/20] Added logic for comparing previous instances. --- .../ingestmodule/IngestModule.java | 33 ++++++++++++++++--- .../ingestmodule/IngestSettingsPanel.form | 2 +- .../ingestmodule/IngestSettingsPanel.java | 2 +- 3 files changed, 30 insertions(+), 7 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java index 20fbf86753..3396eb8ff2 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java @@ -23,6 +23,7 @@ import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbException; import java.util.List; import java.util.logging.Level; import java.util.stream.Collectors; +import org.openide.util.Exceptions; import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.casemodule.Case; @@ -120,14 +121,36 @@ final class IngestModule implements FileIngestModule { * Search the central repo to see if this file was previously marked as * being bad. Create artifact if it was. */ - if (abstractFile.getKnown() != TskData.FileKnown.KNOWN && flagTaggedNotableItems) { + + if (abstractFile.getKnown() != TskData.FileKnown.KNOWN) { + CorrelationAttribute contentCorrelationAttribute = EamArtifactUtil.getCorrelationAttributeFromContent(abstractFile, TskData.FileKnown.BAD, null); try { - List caseDisplayNames = dbManager.getListCasesHavingArtifactInstancesKnownBad(filesType, md5); - if (!caseDisplayNames.isEmpty()) { - postCorrelatedBadFileToBlackboard(abstractFile, caseDisplayNames); + List caseDisplayNamesList = EamDb.getInstance().getListCasesHavingArtifactInstancesKnownBad( + contentCorrelationAttribute.getCorrelationType(), contentCorrelationAttribute.getCorrelationValue()); + String currentCaseDisplayName = Case.getCurrentCase().getDisplayName(); + boolean taggedOutsideCurrentCase = false; + if (!caseDisplayNamesList.isEmpty()) { + for (String name : caseDisplayNamesList) { + if (!name.equals(currentCaseDisplayName)) { + taggedOutsideCurrentCase = true; + break; + } + } + } + + if(flagTaggedNotableItems || !taggedOutsideCurrentCase) { + try { + caseDisplayNamesList = dbManager.getListCasesHavingArtifactInstancesKnownBad(filesType, md5); + if (!caseDisplayNamesList.isEmpty()) { + postCorrelatedBadFileToBlackboard(abstractFile, caseDisplayNamesList); + } + } catch (EamDbException ex) { + logger.log(Level.SEVERE, "Error searching database for artifact.", ex); // NON-NLS + return ProcessResult.ERROR; + } } } catch (EamDbException ex) { - logger.log(Level.SEVERE, "Error searching database for artifact.", ex); // NON-NLS + logger.log(Level.SEVERE, "Error searching database for content.", ex); // NON-NLS return ProcessResult.ERROR; } } diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.form b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.form index 2abe207b5a..564031cb72 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.form +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.form @@ -25,7 +25,7 @@ - + diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.java index 46538f41c2..57d4f0a098 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.java @@ -76,7 +76,7 @@ final class IngestSettingsPanel extends IngestModuleIngestJobSettingsPanel { .addGap(10, 10, 10) .addComponent(flagTaggedNotableItemsCheckbox)) .addComponent(ingestSettingsLabel)) - .addContainerGap(75, Short.MAX_VALUE)) + .addContainerGap(65, Short.MAX_VALUE)) ); layout.setVerticalGroup( layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) From 3d5ac054226f2fc926fc78044115422cf2005141 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Tue, 6 Mar 2018 17:33:14 -0500 Subject: [PATCH 08/20] Corrected logic to match story criteria. --- .../eventlisteners/IngestEventsListener.java | 12 ++++--- .../ingestmodule/IngestModule.java | 35 +++++-------------- 2 files changed, 16 insertions(+), 31 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java index 2f1efd3785..f4e0fd9fbe 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java @@ -255,7 +255,7 @@ public class IngestEventsListener { @Override public void run() { - if (!EamDb.isEnabled() || getCorrelationModulesFlaggingNotableCount() == 0) { + if (!EamDb.isEnabled()) { return; } final ModuleDataEvent mde = (ModuleDataEvent) event.getOldValue(); @@ -276,10 +276,12 @@ public class IngestEventsListener { // query db for artifact instances having this TYPE/VALUE and knownStatus = "Bad". // if gettKnownStatus() is "Unknown" and this artifact instance was marked bad in a previous case, // create TSK_INTERESTING_ARTIFACT_HIT artifact on BB. - List caseDisplayNames = dbManager.getListCasesHavingArtifactInstancesKnownBad(eamArtifact.getCorrelationType(), eamArtifact.getCorrelationValue()); - if (!caseDisplayNames.isEmpty()) { - postCorrelatedBadArtifactToBlackboard(bbArtifact, - caseDisplayNames); + if (getCorrelationModulesFlaggingNotableCount() > 0) { + List caseDisplayNames = dbManager.getListCasesHavingArtifactInstancesKnownBad(eamArtifact.getCorrelationType(), eamArtifact.getCorrelationValue()); + if (!caseDisplayNames.isEmpty()) { + postCorrelatedBadArtifactToBlackboard(bbArtifact, + caseDisplayNames); + } } eamArtifacts.add(eamArtifact); } diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java index 3396eb8ff2..d1362fce63 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java @@ -124,34 +124,17 @@ final class IngestModule implements FileIngestModule { if (abstractFile.getKnown() != TskData.FileKnown.KNOWN) { CorrelationAttribute contentCorrelationAttribute = EamArtifactUtil.getCorrelationAttributeFromContent(abstractFile, TskData.FileKnown.BAD, null); - try { - List caseDisplayNamesList = EamDb.getInstance().getListCasesHavingArtifactInstancesKnownBad( - contentCorrelationAttribute.getCorrelationType(), contentCorrelationAttribute.getCorrelationValue()); - String currentCaseDisplayName = Case.getCurrentCase().getDisplayName(); - boolean taggedOutsideCurrentCase = false; - if (!caseDisplayNamesList.isEmpty()) { - for (String name : caseDisplayNamesList) { - if (!name.equals(currentCaseDisplayName)) { - taggedOutsideCurrentCase = true; - break; - } + if (flagTaggedNotableItems) { + try { + List caseDisplayNamesList = EamDb.getInstance().getListCasesHavingArtifactInstancesKnownBad( + contentCorrelationAttribute.getCorrelationType(), contentCorrelationAttribute.getCorrelationValue()); + if (!caseDisplayNamesList.isEmpty()) { + postCorrelatedBadFileToBlackboard(abstractFile, caseDisplayNamesList); } + } catch (EamDbException ex) { + logger.log(Level.SEVERE, "Error searching database for content.", ex); // NON-NLS + return ProcessResult.ERROR; } - - if(flagTaggedNotableItems || !taggedOutsideCurrentCase) { - try { - caseDisplayNamesList = dbManager.getListCasesHavingArtifactInstancesKnownBad(filesType, md5); - if (!caseDisplayNamesList.isEmpty()) { - postCorrelatedBadFileToBlackboard(abstractFile, caseDisplayNamesList); - } - } catch (EamDbException ex) { - logger.log(Level.SEVERE, "Error searching database for artifact.", ex); // NON-NLS - return ProcessResult.ERROR; - } - } - } catch (EamDbException ex) { - logger.log(Level.SEVERE, "Error searching database for content.", ex); // NON-NLS - return ProcessResult.ERROR; } } From 7a065600f9e98bd4bdb6b73f0f123774e2552d9a Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Tue, 6 Mar 2018 17:58:42 -0500 Subject: [PATCH 09/20] Fixed a few typos. --- .../eventlisteners/IngestEventsListener.java | 5 +++-- .../centralrepository/ingestmodule/IngestModule.java | 7 ++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java index f4e0fd9fbe..5815e6b527 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java @@ -114,7 +114,7 @@ public class IngestEventsListener { } /** - * Wether or not the Correlation Engine Module is enabled for any of the + * Whether or not the Correlation Engine Module is enabled for any of the * currently running ingest jobs. * * @return boolean True for Correlation Engine enabled, False for disabled @@ -153,7 +153,8 @@ public class IngestEventsListener { * Wether or not the Correlation Engine Module is enabled for any of the * currently running ingest jobs and flagging notable items. * - * @return boolean True for Correlation Engine enabled, False for disabled + * @return boolean True for Correlation Engine flagging enabled, False for + * disabled */ private synchronized static int getCorrelationModulesFlaggingNotableCount() { return correlationModulesFlaggingNotableCount; diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java index d1362fce63..db802950c3 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java @@ -73,8 +73,9 @@ final class IngestModule implements FileIngestModule { private final boolean flagTaggedNotableItems; /** - * //DLG: - * @param settings + * Instantiate the Correlation Engine ingest module. + * + * @param settings The ingest settings for the module instance. */ IngestModule(IngestSettings settings) { flagTaggedNotableItems = settings.isFlagTaggedNotableItems(); @@ -123,9 +124,9 @@ final class IngestModule implements FileIngestModule { */ if (abstractFile.getKnown() != TskData.FileKnown.KNOWN) { - CorrelationAttribute contentCorrelationAttribute = EamArtifactUtil.getCorrelationAttributeFromContent(abstractFile, TskData.FileKnown.BAD, null); if (flagTaggedNotableItems) { try { + CorrelationAttribute contentCorrelationAttribute = EamArtifactUtil.getCorrelationAttributeFromContent(abstractFile, TskData.FileKnown.BAD, null); List caseDisplayNamesList = EamDb.getInstance().getListCasesHavingArtifactInstancesKnownBad( contentCorrelationAttribute.getCorrelationType(), contentCorrelationAttribute.getCorrelationValue()); if (!caseDisplayNamesList.isEmpty()) { From eacd201dcd5f9b5985350934ce6ead76f901a95d Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Tue, 6 Mar 2018 18:00:10 -0500 Subject: [PATCH 10/20] Fixed typo. --- .../centralrepository/eventlisteners/IngestEventsListener.java | 2 +- .../autopsy/centralrepository/ingestmodule/IngestModule.java | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java index 5815e6b527..9e6d488069 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java @@ -150,7 +150,7 @@ public class IngestEventsListener { } /** - * Wether or not the Correlation Engine Module is enabled for any of the + * Whether or not the Correlation Engine Module is enabled for any of the * currently running ingest jobs and flagging notable items. * * @return boolean True for Correlation Engine flagging enabled, False for diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java index db802950c3..e0e7c2d7bf 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java @@ -23,7 +23,6 @@ import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbException; import java.util.List; import java.util.logging.Level; import java.util.stream.Collectors; -import org.openide.util.Exceptions; import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.casemodule.Case; From 43fcae798d1b3992a70518c2e808bbde731ad03a Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Tue, 6 Mar 2018 22:25:29 -0500 Subject: [PATCH 11/20] Simplified 'process()' logic. --- .../ingestmodule/IngestModule.java | 20 ++++++++----------- 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java index e0e7c2d7bf..865a5d7086 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java @@ -122,19 +122,15 @@ final class IngestModule implements FileIngestModule { * being bad. Create artifact if it was. */ - if (abstractFile.getKnown() != TskData.FileKnown.KNOWN) { - if (flagTaggedNotableItems) { - try { - CorrelationAttribute contentCorrelationAttribute = EamArtifactUtil.getCorrelationAttributeFromContent(abstractFile, TskData.FileKnown.BAD, null); - List caseDisplayNamesList = EamDb.getInstance().getListCasesHavingArtifactInstancesKnownBad( - contentCorrelationAttribute.getCorrelationType(), contentCorrelationAttribute.getCorrelationValue()); - if (!caseDisplayNamesList.isEmpty()) { - postCorrelatedBadFileToBlackboard(abstractFile, caseDisplayNamesList); - } - } catch (EamDbException ex) { - logger.log(Level.SEVERE, "Error searching database for content.", ex); // NON-NLS - return ProcessResult.ERROR; + if (abstractFile.getKnown() != TskData.FileKnown.KNOWN && flagTaggedNotableItems) { + try { + List caseDisplayNamesList = dbManager.getListCasesHavingArtifactInstancesKnownBad(filesType, md5); + if (!caseDisplayNamesList.isEmpty()) { + postCorrelatedBadFileToBlackboard(abstractFile, caseDisplayNamesList); } + } catch (EamDbException ex) { + logger.log(Level.SEVERE, "Error searching database for content.", ex); // NON-NLS + return ProcessResult.ERROR; } } From 10ad0a004db2b7048f22e4394a24da1ad96fd687 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Tue, 6 Mar 2018 22:28:11 -0500 Subject: [PATCH 12/20] Fixed typo. --- .../autopsy/centralrepository/ingestmodule/IngestModule.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java index 865a5d7086..8c4bd236a2 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java @@ -129,7 +129,7 @@ final class IngestModule implements FileIngestModule { postCorrelatedBadFileToBlackboard(abstractFile, caseDisplayNamesList); } } catch (EamDbException ex) { - logger.log(Level.SEVERE, "Error searching database for content.", ex); // NON-NLS + logger.log(Level.SEVERE, "Error searching database for artifact.", ex); // NON-NLS return ProcessResult.ERROR; } } From 8522b322a4acddd51f1ae3419622242334f0532b Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Tue, 6 Mar 2018 22:43:55 -0500 Subject: [PATCH 13/20] Removed unnecessary initializations. --- .../eventlisteners/IngestEventsListener.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java index 9e6d488069..bc404e2606 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java @@ -56,8 +56,8 @@ public class IngestEventsListener { private static final Logger LOGGER = Logger.getLogger(CorrelationAttribute.class.getName()); final Collection recentlyAddedCeArtifacts = new LinkedHashSet<>(); - private static int correlationModuleInstanceCount = 0; - private static int correlationModulesFlaggingNotableCount = 0; + private static int correlationModuleInstanceCount; + private static int correlationModulesFlaggingNotableCount; private final ExecutorService jobProcessingExecutor; private static final String INGEST_EVENT_THREAD_NAME = "Ingest-Event-Listener-%d"; private final PropertyChangeListener pcl1 = new IngestModuleEventListener(); From 2000a48a1bb5f7d19f0b3be8761a023f4771d124 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Thu, 8 Mar 2018 11:31:36 -0500 Subject: [PATCH 14/20] Module version changed to match application version. --- .../centralrepository/ingestmodule/IngestModuleFactory.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModuleFactory.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModuleFactory.java index e9c9a5a88f..d6fe88a51d 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModuleFactory.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModuleFactory.java @@ -25,6 +25,7 @@ import org.sleuthkit.autopsy.ingest.IngestModuleFactoryAdapter; import org.sleuthkit.autopsy.ingest.IngestModuleGlobalSettingsPanel; import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettings; import org.sleuthkit.autopsy.centralrepository.optionspanel.GlobalSettingsPanel; +import org.sleuthkit.autopsy.coreutils.Version; import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettingsPanel; /** @@ -35,8 +36,6 @@ import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettingsPanel; "IngestModuleFactory.ingestmodule.desc=Saves properties to the central repository for later correlation"}) public class IngestModuleFactory extends IngestModuleFactoryAdapter { - private static final String VERSION_NUMBER = "0.9.0"; - /** * Get the name of the module. * @@ -58,7 +57,7 @@ public class IngestModuleFactory extends IngestModuleFactoryAdapter { @Override public String getModuleVersionNumber() { - return VERSION_NUMBER; + return Version.getVersion(); } @Override From 6397c0e9368393f95e4f120231646fb20b12f555 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Tue, 13 Mar 2018 10:31:17 -0400 Subject: [PATCH 15/20] Fixed merge issue regarding LOGGER variable. --- .../autopsy/centralrepository/ingestmodule/IngestModule.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java index 2d4fd7c798..6acd46f175 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java @@ -96,7 +96,7 @@ final class IngestModule implements FileIngestModule { try { blackboard = Case.getOpenCase().getServices().getBlackboard(); } catch (NoCurrentCaseException ex) { - LOGGER.log(Level.SEVERE, "Exception while getting open case.", ex); + logger.log(Level.SEVERE, "Exception while getting open case.", ex); return ProcessResult.ERROR; } @@ -226,7 +226,7 @@ final class IngestModule implements FileIngestModule { try { autopsyCase = Case.getOpenCase(); } catch (NoCurrentCaseException ex) { - LOGGER.log(Level.SEVERE, "Exception while getting open case.", ex); + logger.log(Level.SEVERE, "Exception while getting open case.", ex); throw new IngestModuleException("Exception while getting open case.", ex); } From 43187f2e1cd039f6b043910479cd79dd66c1376b Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Tue, 13 Mar 2018 14:05:21 -0400 Subject: [PATCH 16/20] Updated IngestEventsListener to track last CE module setting. --- .../eventlisteners/CaseEventListener.java | 2 +- .../eventlisteners/IngestEventsListener.java | 47 ++++++------------- .../ingestmodule/IngestModule.java | 29 ++++++------ 3 files changed, 30 insertions(+), 48 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/CaseEventListener.java b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/CaseEventListener.java index 0f65dd8130..da11671a08 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/CaseEventListener.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/CaseEventListener.java @@ -42,6 +42,7 @@ import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationCase; import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationDataSource; import org.sleuthkit.autopsy.centralrepository.datamodel.EamDb; import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbException; +import org.sleuthkit.autopsy.centralrepository.datamodel.EamOrganization; import org.sleuthkit.autopsy.coreutils.ThreadUtils; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.BlackboardArtifact; @@ -480,7 +481,6 @@ final class CaseEventListener implements PropertyChangeListener { if ((null == event.getOldValue()) && (event.getNewValue() instanceof Case)) { Case curCase = (Case) event.getNewValue(); IngestEventsListener.resetCeModuleInstanceCount(); - IngestEventsListener.resetCorrelationModulesFlaggingNotableCount(); if (!EamDb.isEnabled()) { return; diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java index 10bc4f61c3..87f675daa0 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java @@ -58,7 +58,7 @@ public class IngestEventsListener { final Collection recentlyAddedCeArtifacts = new LinkedHashSet<>(); private static int correlationModuleInstanceCount; - private static int correlationModulesFlaggingNotableCount; + private static boolean flagNotableItems; private final ExecutorService jobProcessingExecutor; private static final String INGEST_EVENT_THREAD_NAME = "Ingest-Event-Listener-%d"; private final PropertyChangeListener pcl1 = new IngestModuleEventListener(); @@ -123,42 +123,23 @@ public class IngestEventsListener { private synchronized static int getCeModuleInstanceCount() { return correlationModuleInstanceCount; } - + /** - * Increase the number of IngestEventsListeners adding contents to the - * Correlation Engine with notable item flagging enabled. + * Are notable items being flagged? + * + * @return True if flagging notable items; otherwise false. */ - public synchronized static void incrementCorrelationModulesFlaggingNotableCount() { - correlationModulesFlaggingNotableCount++; + private synchronized static boolean isFlagNotableItems() { + return flagNotableItems; } - + /** - * Decrease the number of IngestEventsListeners adding contents to the - * Correlation Engine with notable item flagging enabled. + * Configure the listener to flag notable items or not. + * + * @param value True to flag notable items; otherwise false. */ - public synchronized static void decrementCorrelationModulesFlaggingNotableCount() { - if (correlationModulesFlaggingNotableCount > 0) { - correlationModulesFlaggingNotableCount--; - } - } - - /** - * Reset the counter which keeps track of if the Correlation Engine Module - * is being run during injest and flagging notable items to 0. - */ - synchronized static void resetCorrelationModulesFlaggingNotableCount() { - correlationModulesFlaggingNotableCount = 0; - } - - /** - * Whether or not the Correlation Engine Module is enabled for any of the - * currently running ingest jobs and flagging notable items. - * - * @return boolean True for Correlation Engine flagging enabled, False for - * disabled - */ - private synchronized static int getCorrelationModulesFlaggingNotableCount() { - return correlationModulesFlaggingNotableCount; + public synchronized static void setFlagNotableItems(boolean value) { + flagNotableItems = value; } @NbBundle.Messages({"IngestEventsListener.prevTaggedSet.text=Previously Tagged As Notable (Central Repository)", @@ -278,7 +259,7 @@ public class IngestEventsListener { // query db for artifact instances having this TYPE/VALUE and knownStatus = "Bad". // if gettKnownStatus() is "Unknown" and this artifact instance was marked bad in a previous case, // create TSK_INTERESTING_ARTIFACT_HIT artifact on BB. - if (getCorrelationModulesFlaggingNotableCount() > 0) { + if (isFlagNotableItems()) { List caseDisplayNames = dbManager.getListCasesHavingArtifactInstancesKnownBad(eamArtifact.getCorrelationType(), eamArtifact.getCorrelationValue()); if (!caseDisplayNames.isEmpty()) { postCorrelatedBadArtifactToBlackboard(bbArtifact, diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java index 6acd46f175..252b996e17 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java @@ -74,7 +74,7 @@ final class IngestModule implements FileIngestModule { /** * Instantiate the Correlation Engine ingest module. - * + * * @param settings The ingest settings for the module instance. */ IngestModule(IngestSettings settings) { @@ -127,7 +127,6 @@ final class IngestModule implements FileIngestModule { * Search the central repo to see if this file was previously marked as * being bad. Create artifact if it was. */ - if (abstractFile.getKnown() != TskData.FileKnown.KNOWN && flagTaggedNotableItems) { try { List caseDisplayNamesList = dbManager.getListCasesHavingArtifactInstancesKnownBad(filesType, md5); @@ -163,11 +162,7 @@ final class IngestModule implements FileIngestModule { @Override public void shutDown() { IngestEventsListener.decrementCorrelationEngineModuleCount(); - - if (flagTaggedNotableItems) { - IngestEventsListener.decrementCorrelationModulesFlaggingNotableCount(); - } - + if ((EamDb.isEnabled() == false) || (eamCase == null) || (eamDataSource == null)) { return; } @@ -202,11 +197,17 @@ final class IngestModule implements FileIngestModule { @Override public void startUp(IngestJobContext context) throws IngestModuleException { IngestEventsListener.incrementCorrelationEngineModuleCount(); - - if (flagTaggedNotableItems) { - IngestEventsListener.incrementCorrelationModulesFlaggingNotableCount(); - } - + + /* + * Tell the IngestEventsListener to flag notable items based on the + * current module's configuration. This is a work around for the lack of + * an artifacts pipeline. Note that this can be changed by another + * module instance. All modules are affected by the value. While not + * ideal, this will be good enough until a better solution can be + * posited. + */ + IngestEventsListener.setFlagNotableItems(flagTaggedNotableItems); + if (EamDb.isEnabled() == false) { /* * Not throwing the customary exception for now. This is a @@ -227,9 +228,9 @@ final class IngestModule implements FileIngestModule { autopsyCase = Case.getOpenCase(); } catch (NoCurrentCaseException ex) { logger.log(Level.SEVERE, "Exception while getting open case.", ex); - throw new IngestModuleException("Exception while getting open case.", ex); + throw new IngestModuleException("Exception while getting open case.", ex); } - + // Don't allow sqlite central repo databases to be used for multi user cases if ((autopsyCase.getCaseType() == Case.CaseType.MULTI_USER_CASE) && (EamDbPlatformEnum.getSelectedPlatform() == EamDbPlatformEnum.SQLITE)) { From b01b197556c7075c7de432e74204a459fd1447dd Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Tue, 13 Mar 2018 14:36:43 -0400 Subject: [PATCH 17/20] Simplified getIngestJobSettingsPanel(). --- .../centralrepository/ingestmodule/IngestModuleFactory.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModuleFactory.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModuleFactory.java index d6fe88a51d..26be4930e1 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModuleFactory.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModuleFactory.java @@ -94,9 +94,6 @@ public class IngestModuleFactory extends IngestModuleFactoryAdapter { @Override public IngestModuleIngestJobSettingsPanel getIngestJobSettingsPanel(IngestModuleIngestJobSettings settings) { - if (!(settings instanceof IngestSettings)) { - throw new IllegalArgumentException("Expected settings argument to be an instance of IngestSettings"); - } return new IngestSettingsPanel((IngestSettings) settings); } From 9522e7930f52c55ea3859f2144ea00ff4612a2b7 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Thu, 15 Mar 2018 12:31:55 -0400 Subject: [PATCH 18/20] Added restrictions on when flagging can be disabled. --- .../eventlisteners/IngestEventsListener.java | 4 ++-- .../ingestmodule/IngestModule.java | 4 +++- .../ingestmodule/IngestModuleFactory.java | 13 ++++++++++++- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java index 87f675daa0..44d45e08bc 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java @@ -120,7 +120,7 @@ public class IngestEventsListener { * * @return boolean True for Correlation Engine enabled, False for disabled */ - private synchronized static int getCeModuleInstanceCount() { + public synchronized static int getCeModuleInstanceCount() { return correlationModuleInstanceCount; } @@ -129,7 +129,7 @@ public class IngestEventsListener { * * @return True if flagging notable items; otherwise false. */ - private synchronized static boolean isFlagNotableItems() { + public synchronized static boolean isFlagNotableItems() { return flagNotableItems; } diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java index 252b996e17..fbe85ccbc5 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java @@ -206,7 +206,9 @@ final class IngestModule implements FileIngestModule { * ideal, this will be good enough until a better solution can be * posited. */ - IngestEventsListener.setFlagNotableItems(flagTaggedNotableItems); + if (IngestEventsListener.getCeModuleInstanceCount() == 1 || !IngestEventsListener.isFlagNotableItems()) { + IngestEventsListener.setFlagNotableItems(flagTaggedNotableItems); + } if (EamDb.isEnabled() == false) { /* diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModuleFactory.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModuleFactory.java index 26be4930e1..6ef03ae00d 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModuleFactory.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModuleFactory.java @@ -27,6 +27,7 @@ import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettings; import org.sleuthkit.autopsy.centralrepository.optionspanel.GlobalSettingsPanel; import org.sleuthkit.autopsy.coreutils.Version; import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettingsPanel; +import org.sleuthkit.autopsy.ingest.NoIngestModuleIngestJobSettings; /** * Factory for Central Repository ingest modules @@ -94,7 +95,17 @@ public class IngestModuleFactory extends IngestModuleFactoryAdapter { @Override public IngestModuleIngestJobSettingsPanel getIngestJobSettingsPanel(IngestModuleIngestJobSettings settings) { - return new IngestSettingsPanel((IngestSettings) settings); + if (settings instanceof IngestSettings) { + return new IngestSettingsPanel((IngestSettings) settings); + } + /* + * Compatibility check for older versions. + */ + if (settings instanceof NoIngestModuleIngestJobSettings) { + return new IngestSettingsPanel(new IngestSettings()); + } + + throw new IllegalArgumentException("Expected settings argument to be an instance of IngestSettings"); } } From 6f75e3d953d353d53b52138c27baf55496c28387 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Fri, 16 Mar 2018 00:23:35 -0400 Subject: [PATCH 19/20] Store flagging value in DataAddedTask. --- .../eventlisteners/IngestEventsListener.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java index 44d45e08bc..508798f02b 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java @@ -192,7 +192,7 @@ public class IngestEventsListener { } switch (IngestManager.IngestModuleEvent.valueOf(evt.getPropertyName())) { case DATA_ADDED: { - jobProcessingExecutor.submit(new DataAddedTask(dbManager, evt)); + jobProcessingExecutor.submit(new DataAddedTask(dbManager, evt, isFlagNotableItems())); break; } } @@ -230,10 +230,12 @@ public class IngestEventsListener { private final EamDb dbManager; private final PropertyChangeEvent event; + private final boolean flagNotableItemsEnabled; - private DataAddedTask(EamDb db, PropertyChangeEvent evt) { + private DataAddedTask(EamDb db, PropertyChangeEvent evt, boolean flagNotableItemsEnabled) { dbManager = db; event = evt; + this.flagNotableItemsEnabled = flagNotableItemsEnabled; } @Override @@ -259,7 +261,7 @@ public class IngestEventsListener { // query db for artifact instances having this TYPE/VALUE and knownStatus = "Bad". // if gettKnownStatus() is "Unknown" and this artifact instance was marked bad in a previous case, // create TSK_INTERESTING_ARTIFACT_HIT artifact on BB. - if (isFlagNotableItems()) { + if (flagNotableItemsEnabled) { List caseDisplayNames = dbManager.getListCasesHavingArtifactInstancesKnownBad(eamArtifact.getCorrelationType(), eamArtifact.getCorrelationValue()); if (!caseDisplayNames.isEmpty()) { postCorrelatedBadArtifactToBlackboard(bbArtifact, From 7f503055e13617a72330657600103fc3c1d32d6e Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Fri, 16 Mar 2018 13:01:24 -0400 Subject: [PATCH 20/20] Added additional commentary for setFlagNotableItems() call. --- .../centralrepository/ingestmodule/IngestModule.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java index fbe85ccbc5..991da1ad58 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java @@ -205,6 +205,12 @@ final class IngestModule implements FileIngestModule { * module instance. All modules are affected by the value. While not * ideal, this will be good enough until a better solution can be * posited. + * + * Note: Flagging cannot be disabled if any other instances of the + * Correlation Engine module are running. This restriction is to prevent + * missing results in the case where the first module is flagging + * notable items, and the proceeding module (with flagging disabled) + * causes the first to stop flagging. */ if (IngestEventsListener.getCeModuleInstanceCount() == 1 || !IngestEventsListener.isFlagNotableItems()) { IngestEventsListener.setFlagNotableItems(flagTaggedNotableItems);