Improvements and bug fixes

Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/CaseEventListener.java

@@ -724,6 +724,21 @@ public final class CaseEventListener implements PropertyChangeListener {
                                     Blackboard blackboard = tskCase.getBlackboard();
 
                                     List<String> caseDisplayNames = dbManager.getListCasesHavingArtifactInstances(osAcctType, correlationAttributeInstance.getCorrelationValue());
+
+                                    // calculate score
+                                    Score score;
+                                    int numCases = caseDisplayNames.size();
+                                    if (numCases <= IngestEventsListener.MAX_NUM_PREVIOUS_CASES_FOR_LIKELY_NOTABLE_SCORE) {
+                                        score = Score.SCORE_LIKELY_NOTABLE;
+                                    } else if (numCases > IngestEventsListener.MAX_NUM_PREVIOUS_CASES_FOR_LIKELY_NOTABLE_SCORE && numCases <= IngestEventsListener.MAX_NUM_PREVIOUS_CASES_FOR_PREV_SEEN_ARTIFACT_CREATION) {
+                                        score = Score.SCORE_NONE;
+                                    } else {
+                                        // don't make an Analysis Result, the artifact is too common.
+                                        continue;
+                                    }
+
+                                    String prevCases = caseDisplayNames.stream().distinct().collect(Collectors.joining(","));
+                                    String justification = "Previously seen in cases " + prevCases;
                                     Collection<BlackboardAttribute> attributesForNewArtifact = Arrays.asList(
                                             new BlackboardAttribute(
                                                     TSK_SET_NAME, MODULE_NAME,
@@ -736,10 +751,10 @@ public final class CaseEventListener implements PropertyChangeListener {
                                                     correlationAttributeInstance.getCorrelationValue()),
                                             new BlackboardAttribute(
                                                     TSK_OTHER_CASES, MODULE_NAME,
-                                                    caseDisplayNames.stream().distinct().collect(Collectors.joining(","))));
+                                                    prevCases));
                                     BlackboardArtifact newAnalysisResult = osAccount.newAnalysisResult(
-                                            BlackboardArtifact.Type.TSK_PREVIOUSLY_SEEN, Score.SCORE_LIKELY_NOTABLE,
+                                            BlackboardArtifact.Type.TSK_PREVIOUSLY_SEEN, score,
-                                            null, Bundle.CaseEventsListener_prevExists_text(), null, attributesForNewArtifact, osAccountInstance.getDataSource().getId()).getAnalysisResult();
+                                            null, Bundle.CaseEventsListener_prevExists_text(), justification, attributesForNewArtifact, osAccountInstance.getDataSource().getId()).getAnalysisResult();
                                     try {
                                         // index the artifact for keyword search
                                         blackboard.postArtifact(newAnalysisResult, MODULE_NAME);

Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java

@@ -83,6 +83,9 @@ public class IngestEventsListener {
     private final PropertyChangeListener pcl1 = new IngestModuleEventListener();
     private final PropertyChangeListener pcl2 = new IngestJobEventListener();
     final Collection<String> recentlyAddedCeArtifacts = new LinkedHashSet<>();
+    
+    static final int MAX_NUM_PREVIOUS_CASES_FOR_LIKELY_NOTABLE_SCORE = 10;
+    static final int MAX_NUM_PREVIOUS_CASES_FOR_PREV_SEEN_ARTIFACT_CREATION = 20;
 
     public IngestEventsListener() {
         jobProcessingExecutor = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setNameFormat(INGEST_EVENT_THREAD_NAME).build());
@@ -211,7 +214,7 @@ public class IngestEventsListener {
     static private void makeAndPostPreviousNotableArtifact(BlackboardArtifact originalArtifact, List<String> caseDisplayNames,
             CorrelationAttributeInstance.Type aType, String value) {
         String prevCases = caseDisplayNames.stream().distinct().collect(Collectors.joining(","));
-        String justification = "Previously marked as notable in " + prevCases;
+        String justification = "Previously marked as notable in cases " + prevCases;
         Collection<BlackboardAttribute> attributesForNewArtifact = Arrays.asList(new BlackboardAttribute(
                 TSK_SET_NAME, MODULE_NAME,
                 Bundle.IngestEventsListener_prevTaggedSet_text()),
@@ -223,14 +226,14 @@ public class IngestEventsListener {
                         value),
                 new BlackboardAttribute(
                         TSK_OTHER_CASES, MODULE_NAME,
-                        Bundle.IngestEventsListener_prevCaseComment_text() + prevCases));
+                        prevCases));
         makeAndPostArtifact(BlackboardArtifact.Type.TSK_PREVIOUSLY_NOTABLE, originalArtifact, attributesForNewArtifact, Bundle.IngestEventsListener_prevTaggedSet_text(),
                 Score.SCORE_NOTABLE, justification);
     }
 
     /**
      * Create a "previously seen" hit for a device which was previously seen
-     * in the central repository.
+     * in the central repository. NOTE: Artifacts that are too common will be skipped.
      *
      * @param originalArtifact the artifact to create the "previously seen" item for
      * @param caseDisplayNames the case names the artifact was previously seen
@@ -242,8 +245,21 @@ public class IngestEventsListener {
         "IngestEventsListener.prevCount.text=Number of previous {0}: {1}"})
     static private void makeAndPostPreviousSeenArtifact(BlackboardArtifact originalArtifact, List<String> caseDisplayNames,
             CorrelationAttributeInstance.Type aType, String value) {
+        
+        // calculate score
+        Score score;
+        int numCases = caseDisplayNames.size();
+        if (numCases <= MAX_NUM_PREVIOUS_CASES_FOR_LIKELY_NOTABLE_SCORE) {
+            score = Score.SCORE_LIKELY_NOTABLE;
+        } else if (numCases > MAX_NUM_PREVIOUS_CASES_FOR_LIKELY_NOTABLE_SCORE && numCases <= MAX_NUM_PREVIOUS_CASES_FOR_PREV_SEEN_ARTIFACT_CREATION) {
+            score = Score.SCORE_NONE; 
+        } else {
+            // don't make an Analysis Result, the artifact is too common.
+            return;
+        }
+        
         String prevCases = caseDisplayNames.stream().distinct().collect(Collectors.joining(","));
-        String justification = "Previously seen in " + prevCases;
+        String justification = "Previously seen in cases " + prevCases;
         Collection<BlackboardAttribute> attributesForNewArtifact = Arrays.asList(new BlackboardAttribute(
                 TSK_SET_NAME, MODULE_NAME,
                 Bundle.IngestEventsListener_prevExists_text()),
@@ -255,10 +271,9 @@ public class IngestEventsListener {
                         value),
                 new BlackboardAttribute(
                         TSK_OTHER_CASES, MODULE_NAME,
-                        Bundle.IngestEventsListener_prevCaseComment_text() + prevCases));
+                        prevCases));     
-        // ELTODO calculate score        
         makeAndPostArtifact(BlackboardArtifact.Type.TSK_PREVIOUSLY_SEEN, originalArtifact, attributesForNewArtifact, Bundle.IngestEventsListener_prevExists_text(),
-                Score.SCORE_LIKELY_NOTABLE, justification);
+                score, justification);
     }
     
     /**
@@ -517,6 +532,9 @@ public class IngestEventsListener {
                                     if (!caseDisplayNames.isEmpty()) {
                                         makeAndPostPreviousNotableArtifact(bbArtifact,
                                                 caseDisplayNames, eamArtifact.getCorrelationType(), eamArtifact.getCorrelationValue());
+                                        
+                                        // if we have marked this artifact as notable, then skip the analysis of whether it was previously seen
+                                        continue;
                                     }
                                 } catch (CorrelationAttributeNormalizationException ex) {
                                     LOGGER.log(Level.INFO, String.format("Unable to flag notable item: %s.", eamArtifact.toString()), ex);

Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/CentralRepoIngestModule.java

@@ -48,7 +48,7 @@ import org.sleuthkit.autopsy.ingest.IngestServices;
 import org.sleuthkit.datamodel.AbstractFile;
 import org.sleuthkit.datamodel.Blackboard;
 import org.sleuthkit.datamodel.BlackboardArtifact;
-import static org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE.TSK_PREVIOUSLY_SEEN;
+import static org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE.TSK_PREVIOUSLY_NOTABLE;
 import org.sleuthkit.datamodel.BlackboardAttribute;
 import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME;
 import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_CORRELATION_TYPE;
@@ -335,6 +335,8 @@ final class CentralRepoIngestModule implements FileIngestModule {
      * @param caseDisplayNames Case names to be added to a TSK_COMMON attribute.
      */
     private void postCorrelatedBadFileToBlackboard(AbstractFile abstractFile, List<String> caseDisplayNames, CorrelationAttributeInstance.Type aType, String value) {
+        String prevCases = caseDisplayNames.stream().distinct().collect(Collectors.joining(","));
+        String justification = "Previously marked as notable in cases " + prevCases;
         Collection<BlackboardAttribute> attributes = Arrays.asList(
                 new BlackboardAttribute(
                         TSK_SET_NAME, MODULE_NAME,
@@ -347,14 +349,13 @@ final class CentralRepoIngestModule implements FileIngestModule {
                         value),
                 new BlackboardAttribute(
                         TSK_OTHER_CASES, MODULE_NAME,
-                        caseDisplayNames.stream().distinct().collect(Collectors.joining(","))));
+                        prevCases));
         try {
-
             // Create artifact if it doesn't already exist.
-            if (!blackboard.artifactExists(abstractFile, TSK_PREVIOUSLY_SEEN, attributes)) {
+            if (!blackboard.artifactExists(abstractFile, TSK_PREVIOUSLY_NOTABLE, attributes)) {
                 BlackboardArtifact tifArtifact = abstractFile.newAnalysisResult(
-                        BlackboardArtifact.Type.TSK_PREVIOUSLY_SEEN, Score.SCORE_LIKELY_NOTABLE, 
+                        BlackboardArtifact.Type.TSK_PREVIOUSLY_NOTABLE, Score.SCORE_NOTABLE, 
-                        null, Bundle.CentralRepoIngestModule_prevTaggedSet_text(), null, attributes)
+                        null, Bundle.CentralRepoIngestModule_prevTaggedSet_text(), justification, attributes)
                         .getAnalysisResult();
                 try {
                     // index the artifact for keyword search

Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/PastCasesSummary.java

@@ -37,6 +37,7 @@ import org.sleuthkit.datamodel.BlackboardArtifact;
 import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE;
 import org.sleuthkit.datamodel.BlackboardAttribute;
 import org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE;
+import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_OTHER_CASES;
 import org.sleuthkit.datamodel.Content;
 import org.sleuthkit.datamodel.DataSource;
 import org.sleuthkit.datamodel.SleuthkitCase;
@@ -49,7 +50,7 @@ import org.sleuthkit.datamodel.TskCoreException;
  * ingest process, this code could break. This code expects that the central
  * repository ingest module:
  *
- * a) Creates a TSK_PREVIOUSLY_SEEN artifact for a file whose hash is in
+ * a) Creates a TSK_PREVIOUSLY_NOTABLE artifact for a file whose hash is in
  * the central repository as a notable file.
  *
  * b) Creates a TSK_PREVIOUSLY_SEEN artifact for a matching id in the
@@ -101,11 +102,12 @@ public class PastCasesSummary implements DefaultArtifactUpdateGovernor {
     }
 
     private static final Set<Integer> ARTIFACT_UPDATE_TYPE_IDS = new HashSet<>(Arrays.asList(
-            ARTIFACT_TYPE.TSK_PREVIOUSLY_SEEN.getTypeID()
+            ARTIFACT_TYPE.TSK_PREVIOUSLY_SEEN.getTypeID(), 
+            ARTIFACT_TYPE.TSK_PREVIOUSLY_NOTABLE.getTypeID()
     ));
 
     private static final String CENTRAL_REPO_INGEST_NAME = CentralRepoIngestModuleFactory.getModuleName().toUpperCase().trim();
-    private static final BlackboardAttribute.Type TYPE_COMMENT = new BlackboardAttribute.Type(ATTRIBUTE_TYPE.TSK_COMMENT);
+    private static final BlackboardAttribute.Type TYPE_COMMENT = new BlackboardAttribute.Type(ATTRIBUTE_TYPE.TSK_OTHER_CASES);
 
     private static final Set<Integer> CR_DEVICE_TYPE_IDS = new HashSet<>(Arrays.asList(
             ARTIFACT_TYPE.TSK_DEVICE_ATTACHED.getTypeID(),
@@ -115,7 +117,6 @@ public class PastCasesSummary implements DefaultArtifactUpdateGovernor {
     ));
 
     private static final String CASE_SEPARATOR = ",";
-    private static final String PREFIX_END = ":";
 
     private final SleuthkitCaseProvider caseProvider;
     private final java.util.logging.Logger logger;
@@ -172,9 +173,8 @@ public class PastCasesSummary implements DefaultArtifactUpdateGovernor {
     }
 
     /**
-     * Gets a list of cases from the TSK_COMMENT of an artifact. The cases
+     * Gets a list of cases from the TSK_OTHER_CASES of an artifact. The cases
-     * string is expected to be of a form of "Previous Case:
+     * string is expected to be of a form of "case1,case2...caseN".
-     * case1,case2...caseN".
      *
      * @param artifact The artifact.
      *
@@ -200,14 +200,7 @@ public class PastCasesSummary implements DefaultArtifactUpdateGovernor {
             return Collections.emptyList();
         }
 
-        String commentStr = commentAttr.getValueString();
+        String justCasesStr = commentAttr.getValueString().trim();
-
-        int prefixCharIdx = commentStr.indexOf(PREFIX_END);
-        if (prefixCharIdx < 0 || prefixCharIdx >= commentStr.length() - 1) {
-            return Collections.emptyList();
-        }
-
-        String justCasesStr = commentStr.substring(prefixCharIdx + 1).trim();
         return Stream.of(justCasesStr.split(CASE_SEPARATOR))
                 .map(String::trim)
                 .collect(Collectors.toList());
@@ -242,9 +235,9 @@ public class PastCasesSummary implements DefaultArtifactUpdateGovernor {
     }
 
     /**
-     * Given a TSK_PREVIOUSLY_SEEN artifact, retrieves it's parent artifact.
+     * Given a TSK_PREVIOUSLY_SEEN or TSK_PREVIOUSLY_NOTABLE artifact, retrieves it's parent artifact.
      *
-     * @param artifact The input TSK_PREVIOUSLY_SEEN artifact.
+     * @param artifact The input artifact.
      *
      * @return The artifact if found or null if not.
      *
@@ -316,6 +309,19 @@ public class PastCasesSummary implements DefaultArtifactUpdateGovernor {
             }
         }
         
+        for (BlackboardArtifact artifact : skCase.getBlackboard().getArtifacts(ARTIFACT_TYPE.TSK_PREVIOUSLY_NOTABLE.getTypeID(), dataSource.getId())) {
+            List<String> cases = getCasesFromArtifact(artifact);
+            if (cases == null || cases.isEmpty()) {
+                continue;
+            }
+
+            if (hasDeviceAssociatedArtifact(artifact)) {
+                deviceArtifactCases.addAll(cases);
+            } else {
+                nonDeviceArtifactCases.addAll(cases);
+            }
+        }        
+        
         return new PastCasesResult(
                 getCaseCounts(deviceArtifactCases.stream()),
                 getCaseCounts(nonDeviceArtifactCases.stream())