mstoeck3/autopsy-flatpak
1
0
Fork 0
mirror of https://github.com/overcuriousity/autopsy-flatpak.git synced 2025-07-14 17:06:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch '6714-displayDomainGroups-with6713' of https://github.com/wschaeferB/autopsy into 6715-DomainSummaryDisplay-Integrated

Browse Source
This commit is contained in:
William Schaefer 2020-09-16 08:57:30 -04:00
commit 9b105b10c8
5 changed files with 49 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
Core/src/org/sleuthkit/autopsy/discovery/search/DiscoveryAttributes.java
View File

@ -222,7 +222,7 @@ public class DiscoveryAttributes {
static class FrequencyAttribute extends AttributeType { static class FrequencyAttribute extends AttributeType {
static final int BATCH_SIZE = 50; // Number of hashes to look up at one time static final int BATCH_SIZE = 50; // Number of hashes to look up at one time
static final int DOMAIN_BATCH_SIZE = 500; // Number of domains to look up at one time static final int DOMAIN_BATCH_SIZE = 500; // Number of domains to look up at one time
@Override @Override
@ -255,13 +255,13 @@ public class DiscoveryAttributes {
private void processResultFilesForCR(List<Result> results, private void processResultFilesForCR(List<Result> results,
CentralRepository centralRepoDb) throws DiscoveryException { CentralRepository centralRepoDb) throws DiscoveryException {
List<ResultFile> currentFiles = new ArrayList<>(); List<ResultFile> currentFiles = new ArrayList<>();
Set<String> hashesToLookUp = new HashSet<>(); Set<String> hashesToLookUp = new HashSet<>();
List<ResultDomain> domainsToQuery = new ArrayList<>(); List<ResultDomain> domainsToQuery = new ArrayList<>();
for (Result result : results) { for (Result result : results) {
if (result.getKnown() == TskData.FileKnown.KNOWN) { if (result.getKnown() == TskData.FileKnown.KNOWN) {
result.setFrequency(SearchData.Frequency.KNOWN); result.setFrequency(SearchData.Frequency.KNOWN);
} }
if (result.getType() != SearchData.Type.DOMAIN) { if (result.getType() != SearchData.Type.DOMAIN) {
ResultFile file = (ResultFile) result; ResultFile file = (ResultFile) result;
if (file.getFrequency() == SearchData.Frequency.UNKNOWN if (file.getFrequency() == SearchData.Frequency.UNKNOWN
@ -270,56 +270,41 @@ public class DiscoveryAttributes {
hashesToLookUp.add(file.getFirstInstance().getMd5Hash()); hashesToLookUp.add(file.getFirstInstance().getMd5Hash());
currentFiles.add(file); currentFiles.add(file);
} }
if (hashesToLookUp.size() >= BATCH_SIZE) {
computeFrequency(hashesToLookUp, currentFiles, centralRepoDb);
hashesToLookUp.clear(); if (hashesToLookUp.size() >= BATCH_SIZE) {
currentFiles.clear(); computeFrequency(hashesToLookUp, currentFiles, centralRepoDb);
hashesToLookUp.clear();
currentFiles.clear();
} }
} else { } else {
ResultDomain domain = (ResultDomain) result;
try {
CorrelationAttributeInstance.Type domainAttributeType
= centralRepoDb.getCorrelationTypeById(CorrelationAttributeInstance.DOMAIN_TYPE_ID);
Long count = centralRepoDb.getCountArtifactInstancesByTypeValue(domainAttributeType, domain.getDomain());
domain.setFrequency(SearchData.Frequency.fromCount(count));
} catch (CentralRepoException ex) {
throw new DiscoveryException("Error encountered querying the central repository.", ex);
} catch (CorrelationAttributeNormalizationException ex) {
logger.log(Level.INFO, "Domain [%s] could not be normalized for central repository querying, skipping...", domain.getDomain());
}
}
if (hashesToLookUp.size() >= BATCH_SIZE) {
computeFrequency(hashesToLookUp, currentFiles, centralRepoDb);
ResultDomain domainInstance = (ResultDomain) result; ResultDomain domainInstance = (ResultDomain) result;
domainsToQuery.add(domainInstance); domainsToQuery.add(domainInstance);
if (domainsToQuery.size() == DOMAIN_BATCH_SIZE) { if (domainsToQuery.size() == DOMAIN_BATCH_SIZE) {
queryDomainFrequency(domainsToQuery, centralRepoDb); queryDomainFrequency(domainsToQuery, centralRepoDb);
domainsToQuery.clear(); domainsToQuery.clear();
} }
} }
} }
queryDomainFrequency(domainsToQuery, centralRepoDb); queryDomainFrequency(domainsToQuery, centralRepoDb);
computeFrequency(hashesToLookUp, currentFiles, centralRepoDb); computeFrequency(hashesToLookUp, currentFiles, centralRepoDb);
} }
} }
private static void queryDomainFrequency(List<ResultDomain> domainsToQuery, CentralRepository centralRepository) throws DiscoveryException { private static void queryDomainFrequency(List<ResultDomain> domainsToQuery, CentralRepository centralRepository) throws DiscoveryException {
if (domainsToQuery.isEmpty()) { if (domainsToQuery.isEmpty()) {
return; return;
} }
try { try {
final Map<String, List<ResultDomain>> resultDomainTable = new HashMap<>(); final Map<String, List<ResultDomain>> resultDomainTable = new HashMap<>();
final StringJoiner joiner = new StringJoiner(", "); final StringJoiner joiner = new StringJoiner(", ");
final CorrelationAttributeInstance.Type attributeType = centralRepository.getCorrelationTypeById(CorrelationAttributeInstance.DOMAIN_TYPE_ID); final CorrelationAttributeInstance.Type attributeType = centralRepository.getCorrelationTypeById(CorrelationAttributeInstance.DOMAIN_TYPE_ID);
for(ResultDomain domainInstance : domainsToQuery) { for (ResultDomain domainInstance : domainsToQuery) {
try { try {
final String domainValue = domainInstance.getDomain(); final String domainValue = domainInstance.getDomain();
final String normalizedDomain = CorrelationAttributeNormalizer.normalize(attributeType, domainValue); final String normalizedDomain = CorrelationAttributeNormalizer.normalize(attributeType, domainValue);
@ -333,10 +318,10 @@ public class DiscoveryAttributes {
} }
final String tableName = CentralRepoDbUtil.correlationTypeToInstanceTableName(attributeType); final String tableName = CentralRepoDbUtil.correlationTypeToInstanceTableName(attributeType);
final String domainFrequencyQuery = " value AS domain_name, COUNT(*) AS frequency " + final String domainFrequencyQuery = " value AS domain_name, COUNT(*) AS frequency "
"FROM " + tableName + " " + + "FROM " + tableName + " "
"WHERE value IN (" + joiner + ") " + + "WHERE value IN (" + joiner + ") "
"GROUP BY value"; + "GROUP BY value";
final DomainFrequencyCallback frequencyCallback = new DomainFrequencyCallback(resultDomainTable); final DomainFrequencyCallback frequencyCallback = new DomainFrequencyCallback(resultDomainTable);
centralRepository.processSelectClause(domainFrequencyQuery, frequencyCallback); centralRepository.processSelectClause(domainFrequencyQuery, frequencyCallback);
@ -348,15 +333,15 @@ public class DiscoveryAttributes {
throw new DiscoveryException("Fatal exception encountered querying the CR.", ex); throw new DiscoveryException("Fatal exception encountered querying the CR.", ex);
} }
} }
private static class DomainFrequencyCallback implements InstanceTableCallback { private static class DomainFrequencyCallback implements InstanceTableCallback {
private final Map<String, List<ResultDomain>> domainLookup; private final Map<String, List<ResultDomain>> domainLookup;
private SQLException sqlCause; private SQLException sqlCause;
private DomainFrequencyCallback(Map<String, List<ResultDomain>> domainLookup) { private DomainFrequencyCallback(Map<String, List<ResultDomain>> domainLookup) {
this.domainLookup = domainLookup; this.domainLookup = domainLookup;
} }
@Override @Override
public void process(ResultSet resultSet) { public void process(ResultSet resultSet) {
@ -364,9 +349,9 @@ public class DiscoveryAttributes {
while (resultSet.next()) { while (resultSet.next()) {
String domain = resultSet.getString("domain_name"); String domain = resultSet.getString("domain_name");
Long frequency = resultSet.getLong("frequency"); Long frequency = resultSet.getLong("frequency");
List<ResultDomain> domainInstances = domainLookup.get(domain); List<ResultDomain> domainInstances = domainLookup.get(domain);
for(ResultDomain domainInstance : domainInstances) { for (ResultDomain domainInstance : domainInstances) {
domainInstance.setFrequency(SearchData.Frequency.fromCount(frequency)); domainInstance.setFrequency(SearchData.Frequency.fromCount(frequency));
} }
} }
@ -374,7 +359,7 @@ public class DiscoveryAttributes {
this.sqlCause = ex; this.sqlCause = ex;
} }
} }
SQLException getCause() { SQLException getCause() {
return this.sqlCause; return this.sqlCause;
} }
@ -733,7 +718,7 @@ public class DiscoveryAttributes {
FILE_TAG(new FileTagAttribute(), Bundle.DiscoveryAttributes_GroupingAttributeType_tag_displayName()), FILE_TAG(new FileTagAttribute(), Bundle.DiscoveryAttributes_GroupingAttributeType_tag_displayName()),
OBJECT_DETECTED(new ObjectDetectedAttribute(), Bundle.DiscoveryAttributes_GroupingAttributeType_object_displayName()), OBJECT_DETECTED(new ObjectDetectedAttribute(), Bundle.DiscoveryAttributes_GroupingAttributeType_object_displayName()),
MOST_RECENT_DATE(new MostRecentActivityDateAttribute(), Bundle.DiscoveryAttributes_GroupingAttributeType_mostRecentDate_displayName()), MOST_RECENT_DATE(new MostRecentActivityDateAttribute(), Bundle.DiscoveryAttributes_GroupingAttributeType_mostRecentDate_displayName()),
FIRST_DATE(new MostRecentActivityDateAttribute(), Bundle.DiscoveryAttributes_GroupingAttributeType_firstDate_displayName()), FIRST_DATE(new FirstActivityDateAttribute(), Bundle.DiscoveryAttributes_GroupingAttributeType_firstDate_displayName()),
NO_GROUPING(new NoGroupingAttribute(), Bundle.DiscoveryAttributes_GroupingAttributeType_none_displayName()); NO_GROUPING(new NoGroupingAttribute(), Bundle.DiscoveryAttributes_GroupingAttributeType_none_displayName());
private final AttributeType attributeType; private final AttributeType attributeType;
@ -771,7 +756,7 @@ public class DiscoveryAttributes {
return Arrays.asList(FREQUENCY, MOST_RECENT_DATE, FIRST_DATE); return Arrays.asList(FREQUENCY, MOST_RECENT_DATE, FIRST_DATE);
} }
} }
/** /**
* Computes the CR frequency of all the given hashes and updates the list of * Computes the CR frequency of all the given hashes and updates the list of
* files. * files.

12
Core/src/org/sleuthkit/autopsy/discovery/search/DiscoveryKeyUtils.java
View File

@ -22,6 +22,7 @@ import java.text.SimpleDateFormat;
import java.util.Collections; import java.util.Collections;
import java.util.Date; import java.util.Date;
import java.util.List; import java.util.List;
import java.util.Locale;
import java.util.Objects; import java.util.Objects;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.logging.Level; import java.util.logging.Level;
@ -166,7 +167,8 @@ public class DiscoveryKeyUtils {
/** /**
* Get the fileSorting * Get the fileSorting
* @return *
* @return
*/ */
ResultsSorter.SortingMethod getFileSortingMethod() { ResultsSorter.SortingMethod getFileSortingMethod() {
return fileSortingMethod; return fileSortingMethod;
@ -938,6 +940,9 @@ public class DiscoveryKeyUtils {
} }
} }
/**
* Key representing a date of most recent activity.
*/
static class MostRecentActivityDateGroupKey extends GroupKey { static class MostRecentActivityDateGroupKey extends GroupKey {
private final Long epochDate; private final Long epochDate;
@ -1020,6 +1025,9 @@ public class DiscoveryKeyUtils {
} }
} }
/**
* Key representing a date of first activity.
*/
static class FirstActivityDateGroupKey extends GroupKey { static class FirstActivityDateGroupKey extends GroupKey {
private final Long epochDate; private final Long epochDate;
@ -1030,7 +1038,7 @@ public class DiscoveryKeyUtils {
FirstActivityDateGroupKey(Result result) { FirstActivityDateGroupKey(Result result) {
if (result instanceof ResultDomain) { if (result instanceof ResultDomain) {
epochDate = ((ResultDomain) result).getActivityStart(); epochDate = ((ResultDomain) result).getActivityStart();
dateNameString = new SimpleDateFormat("yyyy/MM/dd").format(new Date(TimeUnit.SECONDS.toMillis(epochDate))); dateNameString = new SimpleDateFormat("yyyy/MM/dd", Locale.getDefault()).format(new Date(TimeUnit.SECONDS.toMillis(epochDate)));
} else { } else {
epochDate = Long.MAX_VALUE; epochDate = Long.MAX_VALUE;
dateNameString = Bundle.DiscoveryKeyUtils_FirstActivityDateGroupKey_noDate(); dateNameString = Bundle.DiscoveryKeyUtils_FirstActivityDateGroupKey_noDate();

10
Core/src/org/sleuthkit/autopsy/discovery/search/DomainSearchCache.java
View File

@ -36,10 +36,10 @@ import org.sleuthkit.datamodel.SleuthkitCase;
class DomainSearchCache { class DomainSearchCache {
private static final int MAXIMUM_CACHE_SIZE = 10; private static final int MAXIMUM_CACHE_SIZE = 10;
private static final LoadingCache<SearchKey, Map<GroupKey, List<Result>>> cache = private static final LoadingCache<SearchKey, Map<GroupKey, List<Result>>> cache
CacheBuilder.newBuilder() = CacheBuilder.newBuilder()
.maximumSize(MAXIMUM_CACHE_SIZE) .maximumSize(MAXIMUM_CACHE_SIZE)
.build(new DomainSearchCacheLoader()); .build(new DomainSearchCacheLoader());
/** /**
* Get domain search results matching the given parameters. If no results * Get domain search results matching the given parameters. If no results
@ -57,7 +57,7 @@ class DomainSearchCache {
groupSortingType, domainSortingMethod, caseDb, centralRepoDb); groupSortingType, domainSortingMethod, caseDb, centralRepoDb);
return cache.get(searchKey); return cache.get(searchKey);
} catch (Throwable ex) { } catch (ExecutionException ex) {
throw new DiscoveryException("Error fetching results from cache", ex.getCause()); throw new DiscoveryException("Error fetching results from cache", ex.getCause());
} }
} }

4
Core/src/org/sleuthkit/autopsy/discovery/ui/ArtifactTypeFilterPanel.java
View File

@ -25,6 +25,7 @@ import javax.swing.DefaultListModel;
import javax.swing.JCheckBox; import javax.swing.JCheckBox;
import javax.swing.JLabel; import javax.swing.JLabel;
import javax.swing.JList; import javax.swing.JList;
import org.openide.util.NbBundle;
import org.sleuthkit.autopsy.discovery.search.SearchData; import org.sleuthkit.autopsy.discovery.search.SearchData;
import org.sleuthkit.autopsy.discovery.search.SearchFiltering.ArtifactTypeFilter; import org.sleuthkit.autopsy.discovery.search.SearchFiltering.ArtifactTypeFilter;
import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardArtifact;
@ -133,10 +134,11 @@ class ArtifactTypeFilterPanel extends AbstractDiscoveryFilterPanel {
return null; return null;
} }
@NbBundle.Messages({"ArtifactTypeFilterPanel.selectionNeeded.text=At least one Result type must be selected."})
@Override @Override
String checkForError() { String checkForError() {
if (artifactTypeCheckbox.isSelected() && artifactList.getSelectedValuesList().isEmpty()) { if (artifactTypeCheckbox.isSelected() && artifactList.getSelectedValuesList().isEmpty()) {
return "At least one Result type must be selected."; return Bundle.ArtifactTypeFilterPanel_selectionNeeded_text();
} }
return ""; return "";
} }

7
Core/src/org/sleuthkit/autopsy/discovery/ui/DateFilterPanel.java
View File

@ -278,13 +278,15 @@ class DateFilterPanel extends AbstractDiscoveryFilterPanel {
} }
} }
@NbBundle.Messages({"DateFilterPanel.invalidRange.text=Range or Only Last must be selected",
"DateFilterPanel.startOrEndNeeded.text=A start or end date must be specified to use the range filter"})
@Override @Override
String checkForError() { String checkForError() {
if (dateFilterCheckBox.isSelected()) { if (dateFilterCheckBox.isSelected()) {
if (!(rangeRadioButton.isSelected() || mostRecentRadioButton.isSelected())) { if (!(rangeRadioButton.isSelected() || mostRecentRadioButton.isSelected())) {
return "Range or Only Last must be selected"; return Bundle.DateFilterPanel_invalidRange_text();
} else if (rangeRadioButton.isSelected() && !(startCheckBox.isSelected() || endCheckBox.isSelected())) { } else if (rangeRadioButton.isSelected() && !(startCheckBox.isSelected() || endCheckBox.isSelected())) {
return "A start or end date must be specified to use the range filter"; return Bundle.DateFilterPanel_startOrEndNeeded_text();
} }
} }
return ""; return "";
@ -292,7 +294,6 @@ class DateFilterPanel extends AbstractDiscoveryFilterPanel {
@Override @Override
AbstractFilter getFilter() { AbstractFilter getFilter() {
if (dateFilterCheckBox.isSelected()) { if (dateFilterCheckBox.isSelected()) {
LocalDate startDate = LocalDate.MIN; LocalDate startDate = LocalDate.MIN;
LocalDate endDate = LocalDate.MAX; LocalDate endDate = LocalDate.MAX;