Merge branch '7217-errorHandling' of github.com:gdicristofaro/autopsy into 7212-xleappSources

This commit is contained in:
Greg DiCristofaro 2021-01-14 14:47:23 -05:00
commit 8bf0233bc8
5 changed files with 146 additions and 68 deletions

View File

@ -63,7 +63,7 @@ class DomainSearchCacheLoader extends CacheLoader<SearchKey, Map<GroupKey, List<
// Grouping by CR Frequency, for example, will require further processing
// in order to make the correct decision. The attribute types that require
// more information implement their logic by overriding `addAttributeToResults`.
List<AttributeType> searchAttributes = new ArrayList<>();
Set<AttributeType> searchAttributes = new HashSet<>();
searchAttributes.add(key.getGroupAttributeType());
searchAttributes.addAll(key.getFileSortingMethod().getRequiredAttributes());
for (AttributeType attr : searchAttributes) {
@ -304,7 +304,7 @@ class DomainSearchCacheLoader extends CacheLoader<SearchKey, Map<GroupKey, List<
Content dataSource = skc.getContentById(dataSourceID);
resultDomains.add(new ResultDomain(domain, activityStart,
activityEnd, totalPageViews, pageViewsInLast60, filesDownloaded,
activityEnd, totalPageViews, pageViewsInLast60, filesDownloaded,
countOfKnownAccountTypes, dataSource));
}
} catch (SQLException ex) {

View File

@ -257,60 +257,60 @@ public class ResultsSorter implements Comparator<Result> {
return compareStrings(first.getDomain().toLowerCase(), second.getDomain().toLowerCase());
};
}
/**
* Sorts domains by page view count.
*
*
* This comparator sorts results in descending order (largest -> smallest).
*/
private static Comparator<Result> getPageViewComparator() {
return (Result domain1, Result domain2) -> {
if (domain1.getType() != SearchData.Type.DOMAIN ||
domain2.getType() != SearchData.Type.DOMAIN) {
if (domain1.getType() != SearchData.Type.DOMAIN
|| domain2.getType() != SearchData.Type.DOMAIN) {
return 0;
}
ResultDomain first = (ResultDomain) domain1;
ResultDomain second = (ResultDomain) domain2;
long firstPageViews = first.getTotalPageViews();
long secondPageViews = second.getTotalPageViews();
return Long.compare(secondPageViews, firstPageViews);
};
}
/**
* Sorts result domains by last activity date time. The results will be in
* Sorts result domains by last activity date time. The results will be in
* descending order.
*/
private static Comparator<Result> getLastActivityDateTimeComparator() {
return (Result domain1, Result domain2) -> {
if (domain1.getType() != SearchData.Type.DOMAIN ||
domain2.getType() != SearchData.Type.DOMAIN) {
if (domain1.getType() != SearchData.Type.DOMAIN
|| domain2.getType() != SearchData.Type.DOMAIN) {
return 0;
}
ResultDomain first = (ResultDomain) domain1;
ResultDomain second = (ResultDomain) domain2;
long firstActivityEnd = first.getActivityEnd();
long secondActivityEnd = second.getActivityEnd();
return Long.compare(secondActivityEnd, firstActivityEnd);
};
}
/**
* Sorts result domains by most file downloads. The results will be in
* Sorts result domains by most file downloads. The results will be in
* descending order.
*/
private static Comparator<Result> getWebDownloadsComparator() {
return (Result domain1, Result domain2) -> {
if (domain1.getType() != SearchData.Type.DOMAIN ||
domain2.getType() != SearchData.Type.DOMAIN) {
if (domain1.getType() != SearchData.Type.DOMAIN
|| domain2.getType() != SearchData.Type.DOMAIN) {
return 0;
}
ResultDomain first = (ResultDomain) domain1;
ResultDomain second = (ResultDomain) domain2;
long firstFilesDownloaded = first.getFilesDownloaded();
long secondFilesDownloaded = second.getFilesDownloaded();
return Long.compare(secondFilesDownloaded, firstFilesDownloaded);
@ -388,10 +388,10 @@ public class ResultsSorter implements Comparator<Result> {
Bundle.FileSorter_SortingMethod_keywordlist_displayName()), // Sort alphabetically by list of keyword list names found
BY_FULL_PATH(new ArrayList<>(),
Bundle.FileSorter_SortingMethod_fullPath_displayName()), // Sort alphabetically by path
BY_DOMAIN_NAME(new ArrayList<>(),Bundle.FileSorter_SortingMethod_domain_displayName()),
BY_PAGE_VIEWS(new ArrayList<>(), Bundle.FileSorter_SortingMethod_pageViews_displayName()),
BY_DOWNLOADS(new ArrayList<>(), Bundle.FileSorter_SortingMethod_downloads_displayName()),
BY_LAST_ACTIVITY(new ArrayList<>(), Bundle.FileSorter_SortingMethod_activity_displayName());
BY_DOMAIN_NAME(Arrays.asList(new DiscoveryAttributes.DomainCategoryAttribute()), Bundle.FileSorter_SortingMethod_domain_displayName()),
BY_PAGE_VIEWS(Arrays.asList(new DiscoveryAttributes.DomainCategoryAttribute()), Bundle.FileSorter_SortingMethod_pageViews_displayName()),
BY_DOWNLOADS(Arrays.asList(new DiscoveryAttributes.DomainCategoryAttribute()), Bundle.FileSorter_SortingMethod_downloads_displayName()),
BY_LAST_ACTIVITY(Arrays.asList(new DiscoveryAttributes.DomainCategoryAttribute()), Bundle.FileSorter_SortingMethod_activity_displayName());
private final String displayName;
private final List<DiscoveryAttributes.AttributeType> requiredAttributes;

View File

@ -287,10 +287,10 @@ public final class LeappFileProcessor {
TskCoreException {
if (LeappFile == null || !LeappFile.exists() || fileName == null) {
logger.log(Level.SEVERE, String.format("Leap file: %s is null or does not exist", LeappFile == null ? LeappFile.toString() : "<null>"));
logger.log(Level.WARNING, String.format("Leap file: %s is null or does not exist", LeappFile == null ? LeappFile.toString() : "<null>"));
return;
} else if (attrList == null || artifactType == null || dataSource == null) {
logger.log(Level.SEVERE, String.format("attribute list, artifact type or dataSource not provided for %s", LeappFile == null ? LeappFile.toString() : "<null>"));
logger.log(Level.WARNING, String.format("attribute list, artifact type or dataSource not provided for %s", LeappFile == null ? LeappFile.toString() : "<null>"));
return;
}
@ -330,16 +330,18 @@ public final class LeappFileProcessor {
if (MapUtils.isEmpty(columnNumberToProcess)) {
return Collections.emptyList();
} else if (line == null) {
logger.log(Level.SEVERE, "Line is null. Returning empty list for attributes.");
logger.log(Level.WARNING, "Line is null. Returning empty list for attributes.");
return Collections.emptyList();
}
String[] columnValues;
// Check to see if the 2 values are equal, they may not be equal if there is no corresponding data in the line.
// or if the size of the line to split is not equal to the column numbers we are looking to process. This
// can happen when the last value of the tsv line has no data in it.
// If this happens then adding an empty value(s) for each columnValue where data does not exist
Integer maxColumnNumber = Collections.max(columnNumberToProcess.keySet());
if (maxColumnNumber > line.split("\\t").length) {
if ((maxColumnNumber > line.split("\\t").length) || (columnNumberToProcess.size() > line.split("\\t").length)) {
columnValues = Arrays.copyOf(line.split("\\t"), maxColumnNumber + 1);
} else {
columnValues = line.split("\\t");
@ -351,15 +353,17 @@ public final class LeappFileProcessor {
Integer columnNumber = columnToProcess.getKey();
String attributeName = columnToProcess.getValue();
try {
BlackboardAttribute.Type attributeType = Case.getCurrentCase().getSleuthkitCase().getAttributeType(attributeName.toUpperCase());
if (attributeType == null) {
continue;
if (columnValues[columnNumber] != null) {
try {
BlackboardAttribute.Type attributeType = Case.getCurrentCase().getSleuthkitCase().getAttributeType(attributeName.toUpperCase());
if (attributeType == null) {
continue;
}
String attrType = attributeType.getValueType().getLabel().toUpperCase();
checkAttributeType(bbattributes, attrType, columnValues, columnNumber, attributeType, fileName);
} catch (TskCoreException ex) {
throw new IngestModuleException(String.format("Error getting Attribute type for Attribute Name %s", attributeName), ex); //NON-NLS
}
String attrType = attributeType.getValueType().getLabel().toUpperCase();
checkAttributeType(bbattributes, attrType, columnValues, columnNumber, attributeType, fileName);
} catch (TskCoreException ex) {
throw new IngestModuleException(String.format("Error getting Attribute type for Attribute Name %s", attributeName), ex); //NON-NLS
}
}
@ -375,7 +379,7 @@ public final class LeappFileProcessor {
String fileName) {
if (columnValues == null || columnNumber < 0 || columnNumber > columnValues.length || columnValues[columnNumber] == null) {
logger.log(Level.SEVERE, String.format("Unable to determine column value at index %d in columnValues: %s",
logger.log(Level.WARNING, String.format("Unable to determine column value at index %d in columnValues: %s",
columnNumber,
columnValues == null ? "<null>" : "[" + String.join(", ", columnValues) + "]"));
return;
@ -469,7 +473,7 @@ public final class LeappFileProcessor {
.mapToObj((idx) -> String.format("'%s'", attrList.get(idx).getColumnName() == null ? "<null>" : attrList.get(idx).getColumnName()))
.collect(Collectors.joining(", "));
logger.log(Level.SEVERE, String.format("Columns size expected not found in file %s based on xml from %s. Column Keys Missing = [%s]; Header Line = '%s'.",
logger.log(Level.WARNING, String.format("Columns size expected not found in file %s based on xml from %s. Column Keys Missing = [%s]; Header Line = '%s'.",
this.xmlFile == null ? "<null>" : this.xmlFile,
fileName,
missingColumns,
@ -540,7 +544,7 @@ public final class LeappFileProcessor {
} catch (TskCoreException ex) {
logger.log(Level.SEVERE, String.format("There was an issue that arose while trying to fetch artifact type for %s.", artifactName), ex);
}
if (foundArtifactType == null) {
logger.log(Level.SEVERE, String.format("No known artifact mapping found for [artifact: %s, %s]",
artifactName, getXmlFileIdentifier(parentName)));

View File

@ -31,8 +31,8 @@
<FileName filename="accounts ce 0.tsv" description="Accounts_ce">
<ArtifactName artifactname="TSK_SERVICE_ACCOUNT" comment="accounts ce 0">
<AttributeName attributename="TSK_USER_ID" columnName="Name" required="yes" />
<AttributeName attributename="TSK_PROG_NAME" columnName=" Type" required="yes" />
<AttributeName attributename="TSK_PASSWORD" columnName=" Password" required="yes" />
<AttributeName attributename="TSK_PROG_NAME" columnName="Type" required="yes" />
<AttributeName attributename="TSK_PASSWORD" columnName="Password" required="yes" />
</ArtifactName>
</FileName>
@ -40,9 +40,9 @@
<ArtifactName artifactname="TSK_SERVICE_ACCOUNT" comment="Authtokens">
<AttributeName attributename="null" columnName="ID" required="no" />
<AttributeName attributename="TSK_USER_ID" columnName=" Name" required="yes" />
<AttributeName attributename="TSK_PROG_NAME" columnName=" Account Type" required="yes" />
<AttributeName attributename="TSK_PROG_NAME" columnName="Account Type" required="yes" />
<AttributeName attributename="null" columnName="Authtoken Type" required="no" />
<AttributeName attributename="TSK_PASSWORD" columnName=" Authtoken" required="yes" />
<AttributeName attributename="TSK_PASSWORD" columnName="Authtoken" required="yes" />
</ArtifactName>
</FileName>
@ -56,17 +56,17 @@
<FileName filename="Browser Bookmarks.tsv" description="Browser Bookmarks">
<ArtifactName artifactname="TSK_WEB_BOOKMARK" comment="Browser Bookmarks">
<AttributeName attributename="TSK_DATETIME_CREATED " columnName="Added Date" required="yes" />
<AttributeName attributename="TSK_URL" columnName=" URL" required="yes" />
<AttributeName attributename="TSK_TITLE" columnName=" Name" required="yes" />
<AttributeName attributename="null" columnName=" Parent" required="no" />
<AttributeName attributename="null" columnName=" Type" required="no" />
<AttributeName attributename="TSK_DATETIME_CREATED" columnName="Added Date" required="yes" />
<AttributeName attributename="TSK_URL" columnName="URL" required="yes" />
<AttributeName attributename="TSK_TITLE" columnName="Name" required="yes" />
<AttributeName attributename="null" columnName="Parent" required="no" />
<AttributeName attributename="null" columnName="Type" required="no" />
</ArtifactName>
</FileName>
<FileName filename="Browser cookies.tsv" description="Browser Cookies">
<ArtifactName artifactname="TSK_WEB_COOKIE" comment="Browser Cookies">
<AttributeName attributename="TSK_DATETIME_ACCESS" columnName="Last Access Date" required="yes" />
<AttributeName attributename="TSK_DATETIME_ACCESSED" columnName="Last Access Date" required="yes" />
<AttributeName attributename="TSK_DOMAIN" columnName="Host" required="yes" />
<AttributeName attributename="TSK_NAME" columnName="Name" required="yes" />
<AttributeName attributename="TSK_VALUE" columnName="Value" required="yes" />
@ -108,11 +108,11 @@
<ArtifactName artifactname="TSK_WEB_HISTORY" comment="Browser Offline Pages">
<AttributeName attributename="TSK_DATETIME_CREATED" columnName="Creation Time" required="yes" />
<AttributeName attributename="TSK_DATETIME_ACCESSED" columnName="Last Access Time" required="yes" />
<AttributeName attributename="TSK_URL" columnName=" Online URL" required="yes" />
<AttributeName attributename="null" columnName=" File Path" required="no" />
<AttributeName attributename="TSK_TITLE" columnName=" Title" required="no" />
<AttributeName attributename="null" columnName=" Access Count" required="no" />
<AttributeName attributename="null" columnName=" File Size" required="no" />
<AttributeName attributename="TSK_URL" columnName="Online URL" required="yes" />
<AttributeName attributename="null" columnName="File Path" required="no" />
<AttributeName attributename="TSK_TITLE" columnName="Title" required="no" />
<AttributeName attributename="null" columnName="Access Count" required="no" />
<AttributeName attributename="null" columnName="File Size" required="no" />
</ArtifactName>
</FileName>
@ -153,17 +153,17 @@
<FileName filename="Chrome Bookmarks.tsv" description="Chrome Bookmarks">
<ArtifactName artifactname="TSK_WEB_BOOKMARK" comment="Chrome Bookmarks">
<AttributeName attributename="TSK_DATETIME_CREATED " columnName="Added Date" required="yes" />
<AttributeName attributename="TSK_URL" columnName=" URL" required="yes" />
<AttributeName attributename="TSK_TITLE" columnName=" Name" required="yes" />
<AttributeName attributename="null" columnName=" Parent" required="no" />
<AttributeName attributename="null" columnName=" Type" required="no" />
<AttributeName attributename="TSK_DATETIME_CREATED" columnName="Added Date" required="yes" />
<AttributeName attributename="TSK_URL" columnName="URL" required="yes" />
<AttributeName attributename="TSK_TITLE" columnName="Name" required="yes" />
<AttributeName attributename="null" columnName="Parent" required="no" />
<AttributeName attributename="null" columnName="Type" required="no" />
</ArtifactName>
</FileName>
<FileName filename="Chrome cookies.tsv" description="Chrome Cookies">
<ArtifactName artifactname="TSK_WEB_COOKIE" comment="Chrome Cookies">
<AttributeName attributename="TSK_DATETIME_ACCESS" columnName="Last Access Date" required="yes" />
<AttributeName attributename="TSK_DATETIME_ACCESSED" columnName="Last Access Date" required="yes" />
<AttributeName attributename="TSK_DOMAIN" columnName="Host" required="yes" />
<AttributeName attributename="TSK_NAME" columnName="Name" required="yes" />
<AttributeName attributename="TSK_VALUE" columnName="Value" required="yes" />
@ -197,11 +197,11 @@
<ArtifactName artifactname="TSK_WEB_HISTORY" comment="Chrome Offline Pages">
<AttributeName attributename="TSK_DATETIME_CREATED" columnName="Creation Time" required="yes" />
<AttributeName attributename="TSK_DATETIME_ACCESSED" columnName="Last Access Time" required="yes" />
<AttributeName attributename="TSK_URL" columnName=" Online URL" required="yes" />
<AttributeName attributename="null" columnName=" File Path" required="no" />
<AttributeName attributename="TSK_TITLE" columnName=" Title" required="no" />
<AttributeName attributename="null" columnName=" Access Count" required="no" />
<AttributeName attributename="null" columnName=" File Size" required="no" />
<AttributeName attributename="TSK_URL" columnName="Online URL" required="yes" />
<AttributeName attributename="null" columnName="File Path" required="no" />
<AttributeName attributename="TSK_TITLE" columnName="Title" required="no" />
<AttributeName attributename="null" columnName="Access Count" required="no" />
<AttributeName attributename="null" columnName="File Size" required="no" />
</ArtifactName>
</FileName>
@ -224,6 +224,79 @@
</ArtifactName>
</FileName>
<FileName filename="Edge Bookmarks.tsv" description="Edge Bookmarks">
<ArtifactName artifactname="TSK_WEB_BOOKMARK" comment="Chrome Bookmarks">
<AttributeName attributename="TSK_DATETIME_CREATED" columnName="Added Date" required="yes" />
<AttributeName attributename="TSK_URL" columnName="URL" required="yes" />
<AttributeName attributename="TSK_TITLE" columnName="Name" required="yes" />
<AttributeName attributename="null" columnName="Parent" required="no" />
<AttributeName attributename="null" columnName="Type" required="no" />
</ArtifactName>
</FileName>
<FileName filename="Edge cookies.tsv" description="Edge Cookies">
<ArtifactName artifactname="TSK_WEB_COOKIE" comment="Edge Cookies">
<AttributeName attributename="TSK_DATETIME_ACCESSED" columnName="Last Access Date" required="yes" />
<AttributeName attributename="TSK_DOMAIN" columnName="Host" required="yes" />
<AttributeName attributename="TSK_NAME" columnName="Name" required="yes" />
<AttributeName attributename="TSK_VALUE" columnName="Value" required="yes" />
<AttributeName attributename="TSK_DATETIME_CREATED" columnName="Created Date" required="yes" />
<AttributeName attributename="TSK_DATETIME_END" columnName="Expiration Date" required="yes" />
<AttributeName attributename="TSK_PATH" columnName="Path" required="yes" />
</ArtifactName>
</FileName>
<FileName filename="Edge History.tsv" description="Edge History">
<ArtifactName artifactname="TSK_WEB_HISTORY" comment="Edge History">
<AttributeName attributename="TSK_DATETIME_ACCESSED" columnName="Last Visit Time" required="yes"/>
<AttributeName attributename="TSK_URL" columnName="URL" required="yes"/>
<AttributeName attributename="TSK_TITLE" columnName="Title" required="yes"/>
<AttributeName attributename="null" columnName="Visit Count" required="no"/>
<AttributeName attributename="null" columnName="Hidden" required="no"/>
</ArtifactName>
</FileName>
<FileName filename="Edge login data.tsv" description="Edge Login Data">
<ArtifactName artifactname="TSK_SERVICE_ACCOUNT" comment="Edge Login">
<AttributeName attributename="TSK_DATETIME_CREATED" columnName="Created Time" required="yes" />
<AttributeName attributename="TSK_USER_NAME" columnName="Username" required="yes" />
<AttributeName attributename="TSK_PASSWORD" columnName="Password" required="yes" />
<AttributeName attributename="TSK_URL" columnName="Origin URL" required="no" />
<AttributeName attributename="null" columnName="Blacklisted by User" required="no" />
</ArtifactName>
</FileName>
<FileName filename="Edge offline pages.tsv" description="Edge Offline Pages">
<ArtifactName artifactname="TSK_WEB_HISTORY" comment="Edge Offline Pages">
<AttributeName attributename="TSK_DATETIME_CREATED" columnName="Creation Time" required="yes" />
<AttributeName attributename="TSK_DATETIME_ACCESSED" columnName="Last Access Time" required="yes" />
<AttributeName attributename="TSK_URL" columnName="Online URL" required="yes" />
<AttributeName attributename="null" columnName="File Path" required="no" />
<AttributeName attributename="TSK_TITLE" columnName="Title" required="no" />
<AttributeName attributename="null" columnName="Access Count" required="no" />
<AttributeName attributename="null" columnName="File Size" required="no" />
</ArtifactName>
</FileName>
<FileName filename="Edge search terms.tsv" description="Edge Search Terms">
<ArtifactName artifactname="TSK_WEB_SEARCH_QUERY" comment="Chrome Search Terms">
<AttributeName attributename="TSK_DATETIME_ACCESSED" columnName="Last Visit Time" required="yes"/>
<AttributeName attributename="TSK_TEXT" columnName="Search Term" required="yes"/>
<AttributeName attributename="TSK_URL" columnName="URL" required="yes"/>
<AttributeName attributename="null" columnName="Title" required="no"/>
<AttributeName attributename="null" columnName="Visit Count" required="no"/>
</ArtifactName>
</FileName>
<FileName filename="Edge top sites.tsv" description="Edge Top Sites">
<ArtifactName artifactname="TSK_WEB_HISTORY" comment="Edge Top Sites">
<AttributeName attributename="TSK_URL" columnName="URL" required="yes" />
<AttributeName attributename="null" columnName="Rank" required="no" />
<AttributeName attributename="TSK_TITLE" columnName="Title" required="no" />
<AttributeName attributename="null" columnName="Redirects" required="no" />
</ArtifactName>
</FileName>
<FileName filename="google play searches.tsv" description="Google Play Searches">
<ArtifactName artifactname="TSK_WEB_SEARCH_QUERY" comment="Google Play Search">
<AttributeName attributename="TSK_DATETIME_ACCESSED" columnName="Timestamp" required="yes" />
@ -233,10 +306,11 @@
</FileName>
<FileName filename="google quick search box.tsv" description="Google quick search box">
<ArtifactName artifactname="TSK_WEB_SEARCH_QUERY" comment="Google Quick Search Search">
<AttributeName attributename="TSK_DATETIME" columnName="File Timestamp" required="yes" />
<ArtifactName artifactname="TSK_WEB_SEARCH_QUERY" comment="Google Quick Search">
<AttributeName attributename="TSK_DATETIME_ACCESSED" columnName="File Timestamp" required="yes" />
<AttributeName attributename="null" columnName="Type" required="no" />
<AttributeName attributename="TSK_TEXT" columnName="Queries Response" required="yes" />
<AttributeName attributename="TSK_TEXT" columnName="Queries" required="yes" />
<AttributeName attributename="null" columnName="Response" required="no" />
<AttributeName attributename="null" columnName="Source File" required="no" />
</ArtifactName>
</FileName>
@ -256,12 +330,12 @@
</FileName>
<FileName filename="installed apps vending.tsv" description="Installed Apps (Vending)">
<ArtifactName artifactname="TSK_INSTALLED_PROG" comment="Installed Apps (VEnding)">
<ArtifactName artifactname="TSK_INSTALLED_PROG" comment="Installed Apps (Vending)">
<AttributeName attributename="TSK_DATETIME" columnName="First Download" required="yes" />
<AttributeName attributename="TSK_PROG_NAME" columnName="Package Name" required="yes" />
<AttributeName attributename="TSK_TITLE" columnName=" Title" required="yes" />
<AttributeName attributename="TSK_TITLE" columnName="Title" required="yes" />
<AttributeName attributename="null" columnName="Install Reason" required="no" />
<AttributeName attributename="null" columnName=" Auto Update?" required="no" />
<AttributeName attributename="null" columnName="Auto Update?" required="no" />
</ArtifactName>
</FileName>

Binary file not shown.