diff --git a/Core/build.xml b/Core/build.xml index bbf612c3d9..e54c4eda97 100644 --- a/Core/build.xml +++ b/Core/build.xml @@ -162,7 +162,7 @@ - + @@ -195,5 +195,47 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Core/src/org/sleuthkit/autopsy/contentviewers/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/contentviewers/Bundle.properties-MERGED index ec1b7be47e..6df3399a0a 100644 --- a/Core/src/org/sleuthkit/autopsy/contentviewers/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/contentviewers/Bundle.properties-MERGED @@ -96,7 +96,7 @@ Metadata.tableRowTitle.mimeType=MIME Type Metadata.tableRowTitle.name=Name Metadata.tableRowTitle.sectorSize=Sector Size Metadata.tableRowTitle.sha1=SHA1 -Metadata.tableRowTitle.sha256=SHA256 +Metadata.tableRowTitle.sha256=SHA-256 Metadata.tableRowTitle.size=Size Metadata.tableRowTitle.fileNameAlloc=File Name Allocation Metadata.tableRowTitle.metadataAlloc=Metadata Allocation diff --git a/Core/src/org/sleuthkit/autopsy/contentviewers/Metadata.java b/Core/src/org/sleuthkit/autopsy/contentviewers/Metadata.java index 796386a5b3..45dae8e2e0 100644 --- a/Core/src/org/sleuthkit/autopsy/contentviewers/Metadata.java +++ b/Core/src/org/sleuthkit/autopsy/contentviewers/Metadata.java @@ -137,7 +137,7 @@ public class Metadata extends javax.swing.JPanel implements DataContentViewer { "Metadata.tableRowTitle.mimeType=MIME Type", "Metadata.nodeText.truncated=(results truncated)", "Metadata.tableRowTitle.sha1=SHA1", - "Metadata.tableRowTitle.sha256=SHA256", + "Metadata.tableRowTitle.sha256=SHA-256", "Metadata.tableRowTitle.imageType=Type", "Metadata.tableRowTitle.sectorSize=Sector Size", "Metadata.tableRowTitle.timezone=Time Zone", @@ -182,6 +182,11 @@ public class Metadata extends javax.swing.JPanel implements DataContentViewer { md5 = NbBundle.getMessage(this.getClass(), "Metadata.tableRowContent.md5notCalc"); } addRow(sb, NbBundle.getMessage(this.getClass(), "Metadata.tableRowTitle.md5"), md5); + String sha256 = file.getSha256Hash(); + if (sha256 == null) { + sha256 = NbBundle.getMessage(this.getClass(), "Metadata.tableRowContent.md5notCalc"); + } + addRow(sb, NbBundle.getMessage(this.getClass(), "Metadata.tableRowTitle.sha256"), sha256); addRow(sb, NbBundle.getMessage(this.getClass(), "Metadata.tableRowTitle.hashLookupResults"), file.getKnown().toString()); addAcquisitionDetails(sb, dataSource); diff --git a/Core/src/org/sleuthkit/autopsy/contentviewers/PDFViewer.java b/Core/src/org/sleuthkit/autopsy/contentviewers/PDFViewer.java index b21a48a193..59e276ce43 100755 --- a/Core/src/org/sleuthkit/autopsy/contentviewers/PDFViewer.java +++ b/Core/src/org/sleuthkit/autopsy/contentviewers/PDFViewer.java @@ -20,6 +20,7 @@ package org.sleuthkit.autopsy.contentviewers; import java.awt.BorderLayout; import java.awt.Component; +import java.awt.Container; import java.io.IOException; import java.lang.reflect.InvocationTargetException; import java.util.Arrays; @@ -97,6 +98,9 @@ final class PDFViewer implements FileTypeViewer { // Add the IcePDF view to the center of our container. this.container.add(icePdfPanel, BorderLayout.CENTER); + + // Disable all components until the document is ready to view. + enableComponents(container, false); // Document is the 'M' in IcePDFs MVC set up. Read the data needed to // populate the model in the background. @@ -122,12 +126,13 @@ final class PDFViewer implements FileTypeViewer { // will cause UI widgets to be updated. try { Document doc = get(); - controller.openDocument(doc, null); + controller.openDocument(doc, file.getName()); // This makes the PDF viewer appear as one continuous // document, which is the default for most popular PDF viewers. controller.setPageViewMode(DocumentViewControllerImpl.ONE_COLUMN_VIEW, true); // This makes it possible to select text by left clicking and dragging. controller.setDisplayTool(DocumentViewModelImpl.DISPLAY_TOOL_TEXT_SELECTION); + enableComponents(container, true); } catch (InterruptedException ex) { // Do nothing. } catch (ExecutionException ex) { @@ -140,10 +145,28 @@ final class PDFViewer implements FileTypeViewer { file.getId(), file.getName()), ex); showErrorDialog(); } + } catch (Throwable ex) { + logger.log(Level.WARNING, String.format("PDF content viewer " + + "was unable to open document with id %d and name %s", + file.getId(), file.getName()), ex); } } }.execute(); } + + /** + * Recursively enable/disable all components in this content viewer. + * This will disable/enable all internal IcePDF Swing components too. + */ + private void enableComponents(Container container, boolean enabled) { + Component[] components = container.getComponents(); + for(Component component : components) { + component.setEnabled(enabled); + if (component instanceof Container) { + enableComponents((Container)component, enabled); + } + } + } @Override public Component getComponent() { diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java index 74ac603e0b..ee862eee96 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java @@ -278,6 +278,7 @@ public abstract class AbstractAbstractFileNode extends A "AbstractAbstractFileNode.typeMetaColLbl=Type(Meta)", "AbstractAbstractFileNode.knownColLbl=Known", "AbstractAbstractFileNode.md5HashColLbl=MD5 Hash", + "AbstractAbstractFileNode.sha256HashColLbl=SHA-256 Hash", "AbstractAbstractFileNode.objectId=Object ID", "AbstractAbstractFileNode.mimeType=MIME Type", "AbstractAbstractFileNode.extensionColLbl=Extension"}) @@ -305,6 +306,7 @@ public abstract class AbstractAbstractFileNode extends A TYPE_META(AbstractAbstractFileNode_typeMetaColLbl()), KNOWN(AbstractAbstractFileNode_knownColLbl()), MD5HASH(AbstractAbstractFileNode_md5HashColLbl()), + SHA256HASH(AbstractAbstractFileNode_sha256HashColLbl()), ObjectID(AbstractAbstractFileNode_objectId()), MIMETYPE(AbstractAbstractFileNode_mimeType()), EXTENSION(AbstractAbstractFileNode_extensionColLbl()); @@ -358,6 +360,7 @@ public abstract class AbstractAbstractFileNode extends A properties.add(new NodeProperty<>(KNOWN.toString(), KNOWN.toString(), NO_DESCR, content.getKnown().getName())); properties.add(new NodeProperty<>(LOCATION.toString(), LOCATION.toString(), NO_DESCR, getContentPath(content))); properties.add(new NodeProperty<>(MD5HASH.toString(), MD5HASH.toString(), NO_DESCR, StringUtils.defaultString(content.getMd5Hash()))); + properties.add(new NodeProperty<>(SHA256HASH.toString(), SHA256HASH.toString(), NO_DESCR, StringUtils.defaultString(content.getSha256Hash()))); properties.add(new NodeProperty<>(MIMETYPE.toString(), MIMETYPE.toString(), NO_DESCR, StringUtils.defaultString(content.getMIMEType()))); properties.add(new NodeProperty<>(EXTENSION.toString(), EXTENSION.toString(), NO_DESCR, content.getNameExtension())); @@ -577,6 +580,7 @@ public abstract class AbstractAbstractFileNode extends A map.put(FLAGS_META.toString(), content.getMetaFlagsAsString()); map.put(KNOWN.toString(), content.getKnown().getName()); map.put(MD5HASH.toString(), StringUtils.defaultString(content.getMd5Hash())); + map.put(SHA256HASH.toString(), StringUtils.defaultString(content.getSha256Hash())); map.put(MIMETYPE.toString(), StringUtils.defaultString(content.getMIMEType())); map.put(EXTENSION.toString(), content.getNameExtension()); } diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/datamodel/Bundle.properties-MERGED index e7310882a3..4e87eab218 100755 --- a/Core/src/org/sleuthkit/autopsy/datamodel/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/datamodel/Bundle.properties-MERGED @@ -30,6 +30,7 @@ AbstractAbstractFileNode.modifiedTimeColLbl=Modified Time AbstractAbstractFileNode.nameColLbl=Name AbstractAbstractFileNode.objectId=Object ID AbstractAbstractFileNode.originalName=Original Name +AbstractAbstractFileNode.sha256HashColLbl=SHA-256 Hash AbstractAbstractFileNode.sizeColLbl=Size AbstractAbstractFileNode.tagsProperty.displayName=Tags AbstractAbstractFileNode.typeDirColLbl=Type(Dir) diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/accounts/Accounts.java b/Core/src/org/sleuthkit/autopsy/datamodel/accounts/Accounts.java index 8e9a17931e..3c5f94f34c 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/accounts/Accounts.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/accounts/Accounts.java @@ -529,9 +529,11 @@ final public class Accounts implements AutopsyVisitableItem { + getRejectedArtifactFilterClause(); //NON-NLS try (SleuthkitCase.CaseDbQuery results = skCase.executeQuery(query); ResultSet rs = results.getResultSet();) { + List tempList = new ArrayList<>(); while (rs.next()) { - list.add(rs.getLong("artifact_id")); //NON-NLS + tempList.add(rs.getLong("artifact_id")); // NON-NLS } + list.addAll(tempList); } catch (TskCoreException | SQLException ex) { LOGGER.log(Level.SEVERE, "Error querying for account artifacts.", ex); //NON-NLS } diff --git a/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/DataSourceInfoUtilities.java b/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/DataSourceInfoUtilities.java index 649c15940a..c711f3c9a0 100644 --- a/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/DataSourceInfoUtilities.java +++ b/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/DataSourceInfoUtilities.java @@ -398,6 +398,20 @@ final class DataSourceInfoUtilities { BlackboardAttribute attr = getAttributeOrNull(artifact, attributeType); return (attr == null) ? null : attr.getValueLong(); } + + /** + * Retrieves the int value of a certain attribute type from an artifact. + * + * @param artifact The artifact. + * @param attributeType The attribute type. + * + * @return The 'getValueInt()' value or null if the attribute could not be + * retrieved. + */ + static Integer getIntOrNull(BlackboardArtifact artifact, Type attributeType) { + BlackboardAttribute attr = getAttributeOrNull(artifact, attributeType); + return (attr == null) ? null : attr.getValueInt(); + } /** * Retrieves the long value of a certain attribute type from an artifact and diff --git a/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/RecentFilesSummary.java b/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/RecentFilesSummary.java index 0a47e0ea6c..6c4601af88 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/RecentFilesSummary.java +++ b/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/RecentFilesSummary.java @@ -118,7 +118,7 @@ public class RecentFilesSummary implements DefaultArtifactUpdateGovernor { dataSource, DATETIME_ATT, DataSourceInfoUtilities.SortOrder.DESCENDING, - 10); + maxCount); List fileDetails = new ArrayList<>(); for (BlackboardArtifact artifact : artifactList) { @@ -134,12 +134,11 @@ public class RecentFilesSummary implements DefaultArtifactUpdateGovernor { } else if (attribute.getAttributeType().equals(PATH_ATT)) { path = attribute.getValueString(); } - - if (accessedTime != null) { - fileDetails.add(new RecentFileDetails(path, accessedTime)); - } } + if (accessedTime != null && accessedTime != 0) { + fileDetails.add(new RecentFileDetails(path, accessedTime)); + } } return fileDetails; @@ -190,7 +189,7 @@ public class RecentFilesSummary implements DefaultArtifactUpdateGovernor { path = attribute.getValueString(); } } - if (accessedTime != null) { + if (accessedTime != null && accessedTime != 0L) { fileDetails.add(new RecentDownloadDetails(path, accessedTime, domain)); } } @@ -215,6 +214,10 @@ public class RecentFilesSummary implements DefaultArtifactUpdateGovernor { return Collections.emptyList(); } + if (maxCount < 0) { + throw new IllegalArgumentException("Invalid maxCount passed to getRecentAttachments, value must be equal to or greater than 0"); + } + return createListFromMap(buildAttachmentMap(dataSource), maxCount); } @@ -241,7 +244,7 @@ public class RecentFilesSummary implements DefaultArtifactUpdateGovernor { } BlackboardArtifact messageArtifact = skCase.getBlackboardArtifact(attribute.getValueLong()); - if (isMessageArtifact(messageArtifact)) { + if (messageArtifact != null && isMessageArtifact(messageArtifact)) { Content content = artifact.getParent(); if (content instanceof AbstractFile) { String sender; diff --git a/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/TopProgramsSummary.java b/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/TopProgramsSummary.java deleted file mode 100644 index e984b9efca..0000000000 --- a/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/TopProgramsSummary.java +++ /dev/null @@ -1,370 +0,0 @@ -/* - * Autopsy Forensic Browser - * - * Copyright 2020 Basis Technology Corp. - * Contact: carrier sleuthkit org - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.sleuthkit.autopsy.datasourcesummary.datamodel; - -import org.sleuthkit.autopsy.datasourcesummary.uiutils.DefaultArtifactUpdateGovernor; -import java.io.File; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.Date; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.function.Function; -import java.util.stream.Collectors; -import org.apache.commons.lang.StringUtils; -import org.sleuthkit.autopsy.datasourcesummary.datamodel.SleuthkitCaseProvider.SleuthkitCaseProviderException; -import org.sleuthkit.datamodel.BlackboardArtifact; -import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE; -import org.sleuthkit.datamodel.BlackboardAttribute; -import org.sleuthkit.datamodel.DataSource; -import org.sleuthkit.datamodel.SleuthkitCase; -import org.sleuthkit.datamodel.TskCoreException; - -/** - * Provides information to populate Top Programs Summary queries. - */ -public class TopProgramsSummary implements DefaultArtifactUpdateGovernor { - - private static final Set ARTIFACT_UPDATE_TYPE_IDS = new HashSet<>(Arrays.asList( - ARTIFACT_TYPE.TSK_PROG_RUN.getTypeID() - )); - - /** - * A SQL join type. - */ - private enum JoinType { - LEFT, - RIGHT, - INNER, - OUTER - } - - /** - * A blackboard attribute value column. - */ - private enum AttributeColumn { - value_text, - value_int32, - value_int64 - } - - /** - * The suffix joined to a key name for use as an identifier of a query. - */ - private static final String QUERY_SUFFIX = "_query"; - - /** - * Functions that determine the folder name of a list of path elements. If - * not matched, function returns null. - */ - private static final List, String>> SHORT_FOLDER_MATCHERS = Arrays.asList( - // handle Program Files and Program Files (x86) - if true, return the next folder - (pathList) -> { - if (pathList.size() < 2) { - return null; - } - - String rootParent = pathList.get(0).toUpperCase(); - if ("PROGRAM FILES".equals(rootParent) || "PROGRAM FILES (X86)".equals(rootParent)) { - return pathList.get(1); - } else { - return null; - } - }, - // if there is a folder named "APPLICATION DATA" or "APPDATA" - (pathList) -> { - for (String pathEl : pathList) { - String uppered = pathEl.toUpperCase(); - if ("APPLICATION DATA".equals(uppered) || "APPDATA".equals(uppered)) { - return "AppData"; - } - } - return null; - } - ); - - /** - * Creates a sql statement querying the blackboard attributes table for a - * particular attribute type and returning a specified value. That query - * also joins with the blackboard artifact table. - * - * @param joinType The type of join statement to create. - * @param attributeColumn The blackboard attribute column that should be - * returned. - * @param attrType The attribute type to query for. - * @param keyName The aliased name of the attribute to return. This - * is also used to calculate the alias of the query - * same as getFullKey. - * @param bbaName The blackboard artifact table alias. - * - * @return The generated sql statement. - */ - private static String getAttributeJoin(JoinType joinType, AttributeColumn attributeColumn, BlackboardAttribute.ATTRIBUTE_TYPE attrType, String keyName, String bbaName) { - String queryName = keyName + QUERY_SUFFIX; - String innerQueryName = "inner_attribute_" + queryName; - - return "\n" + joinType + " JOIN (\n" - + " SELECT \n" - + " " + innerQueryName + ".artifact_id,\n" - + " " + innerQueryName + "." + attributeColumn + " AS " + keyName + "\n" - + " FROM blackboard_attributes " + innerQueryName + "\n" - + " WHERE " + innerQueryName + ".attribute_type_id = " + attrType.getTypeID() + " -- " + attrType.name() + "\n" - + ") " + queryName + " ON " + queryName + ".artifact_id = " + bbaName + ".artifact_id\n"; - } - - /** - * Given a column key, creates the full name for the column key. - * - * @param key The column key. - * - * @return The full identifier for the column key. - */ - private static String getFullKey(String key) { - return key + QUERY_SUFFIX + "." + key; - } - - /** - * Constructs a SQL 'where' statement from a list of clauses and puts - * parenthesis around each clause. - * - * @param clauses The clauses - * - * @return The generated 'where' statement. - */ - private static String getWhereString(List clauses) { - if (clauses.isEmpty()) { - return ""; - } - - List parenthesized = clauses.stream() - .map(c -> "(" + c + ")") - .collect(Collectors.toList()); - - return "\nWHERE " + String.join("\n AND ", parenthesized) + "\n"; - } - - /** - * Generates a [column] LIKE sql clause. - * - * @param column The column identifier. - * @param likeString The string that will be used as column comparison. - * @param isLike if false, the statement becomes NOT LIKE. - * - * @return The generated statement. - */ - private static String getLikeClause(String column, String likeString, boolean isLike) { - return column + (isLike ? "" : " NOT") + " LIKE '" + likeString + "'"; - } - - private final SleuthkitCaseProvider provider; - - public TopProgramsSummary() { - this(SleuthkitCaseProvider.DEFAULT); - } - - public TopProgramsSummary(SleuthkitCaseProvider provider) { - this.provider = provider; - } - - @Override - public Set getArtifactTypeIdsForRefresh() { - return ARTIFACT_UPDATE_TYPE_IDS; - } - - /** - * Retrieves a list of the top programs used on the data source. Currently - * determines this based off of which prefetch results return the highest - * count. - * - * @param dataSource The data source. - * @param count The number of programs to return. - * - * @return The top results objects found. - * - * @throws SleuthkitCaseProviderException - * @throws TskCoreException - * @throws SQLException - */ - public List getTopPrograms(DataSource dataSource, int count) - throws SleuthkitCaseProviderException, TskCoreException, SQLException { - if (dataSource == null || count <= 0) { - return Collections.emptyList(); - } - - // ntosboot should be ignored - final String ntosBootIdentifier = "NTOSBOOT"; - // programs in windows directory to be ignored - final String windowsDir = "/WINDOWS%"; - - final String nameParam = "name"; - final String pathParam = "path"; - final String runCountParam = "run_count"; - final String lastRunParam = "last_run"; - - String bbaQuery = "bba"; - - final String query = "SELECT\n" - + " " + getFullKey(nameParam) + " AS " + nameParam + ",\n" - + " " + getFullKey(pathParam) + " AS " + pathParam + ",\n" - + " MAX(" + getFullKey(runCountParam) + ") AS " + runCountParam + ",\n" - + " MAX(" + getFullKey(lastRunParam) + ") AS " + lastRunParam + "\n" - + "FROM blackboard_artifacts " + bbaQuery + "\n" - + getAttributeJoin(JoinType.INNER, AttributeColumn.value_text, BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PROG_NAME, nameParam, bbaQuery) - + getAttributeJoin(JoinType.LEFT, AttributeColumn.value_text, BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PATH, pathParam, bbaQuery) - + getAttributeJoin(JoinType.LEFT, AttributeColumn.value_int32, BlackboardAttribute.ATTRIBUTE_TYPE.TSK_COUNT, runCountParam, bbaQuery) - + getAttributeJoin(JoinType.LEFT, AttributeColumn.value_int64, BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME, lastRunParam, bbaQuery) - + getWhereString(Arrays.asList( - bbaQuery + ".artifact_type_id = " + BlackboardArtifact.ARTIFACT_TYPE.TSK_PROG_RUN.getTypeID(), - bbaQuery + ".data_source_obj_id = " + dataSource.getId(), - // exclude ntosBootIdentifier from results - getLikeClause(getFullKey(nameParam), ntosBootIdentifier, false), - // exclude windows directory items from results - getFullKey(pathParam) + " IS NULL OR " + getLikeClause(getFullKey(pathParam), windowsDir, false) - )) - + "GROUP BY " + getFullKey(nameParam) + ", " + getFullKey(pathParam) + "\n" - + "ORDER BY \n" - + " MAX(" + getFullKey(runCountParam) + ") DESC,\n" - + " MAX(" + getFullKey(lastRunParam) + ") DESC,\n" - + " " + getFullKey(nameParam) + " ASC"; - - DataSourceInfoUtilities.ResultSetHandler> handler = (resultSet) -> { - List progResults = new ArrayList<>(); - - boolean quitAtCount = false; - - while (resultSet.next() && (!quitAtCount || progResults.size() < count)) { - long lastRunEpoch = resultSet.getLong(lastRunParam); - Date lastRun = (resultSet.wasNull()) ? null : new Date(lastRunEpoch * 1000); - - Long runCount = resultSet.getLong(runCountParam); - if (resultSet.wasNull()) { - runCount = null; - } - - if (lastRun != null || runCount != null) { - quitAtCount = true; - } - - progResults.add(new TopProgramsResult( - resultSet.getString(nameParam), - resultSet.getString(pathParam), - runCount, - lastRun)); - } - - return progResults; - }; - - try (SleuthkitCase.CaseDbQuery dbQuery = provider.get().executeQuery(query); - ResultSet resultSet = dbQuery.getResultSet()) { - - return handler.process(resultSet); - } - } - - /** - * Determines a short folder name if any. Otherwise, returns empty string. - * - * @param strPath The string path. - * @param applicationName The application name. - * - * @return The short folder name or empty string if not found. - */ - public String getShortFolderName(String strPath, String applicationName) { - if (strPath == null) { - return ""; - } - - List pathEls = new ArrayList<>(Arrays.asList(applicationName)); - - File file = new File(strPath); - while (file != null && StringUtils.isNotBlank(file.getName())) { - pathEls.add(file.getName()); - file = file.getParentFile(); - } - - Collections.reverse(pathEls); - - for (Function, String> matchEntry : SHORT_FOLDER_MATCHERS) { - String result = matchEntry.apply(pathEls); - if (StringUtils.isNotBlank(result)) { - return result; - } - } - - return ""; - } - - /** - * Describes a result of a program run on a datasource. - */ - public static class TopProgramsResult { - - private final String programName; - private final String programPath; - private final Long runTimes; - private final Date lastRun; - - /** - * Main constructor. - * - * @param programName The name of the program. - * @param programPath The path of the program. - * @param runTimes The number of runs. - */ - TopProgramsResult(String programName, String programPath, Long runTimes, Date lastRun) { - this.programName = programName; - this.programPath = programPath; - this.runTimes = runTimes; - this.lastRun = lastRun; - } - - /** - * @return The name of the program - */ - public String getProgramName() { - return programName; - } - - /** - * @return The path of the program. - */ - public String getProgramPath() { - return programPath; - } - - /** - * @return The number of run times or null if not present. - */ - public Long getRunTimes() { - return runTimes; - } - - /** - * @return The last time the program was run or null if not present. - */ - public Date getLastRun() { - return lastRun; - } - } -} diff --git a/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/UserActivitySummary.java b/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/UserActivitySummary.java index 481840776f..16da6f5c4b 100644 --- a/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/UserActivitySummary.java +++ b/Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/UserActivitySummary.java @@ -18,6 +18,7 @@ */ package org.sleuthkit.autopsy.datasourcesummary.datamodel; +import java.io.File; import org.sleuthkit.autopsy.datasourcesummary.uiutils.DefaultArtifactUpdateGovernor; import java.util.ArrayList; import java.util.Arrays; @@ -30,6 +31,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.Function; import java.util.logging.Level; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -54,6 +56,36 @@ import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE; */ public class UserActivitySummary implements DefaultArtifactUpdateGovernor { + /** + * Functions that determine the folder name of a list of path elements. If + * not matched, function returns null. + */ + private static final List, String>> SHORT_FOLDER_MATCHERS = Arrays.asList( + // handle Program Files and Program Files (x86) - if true, return the next folder + (pathList) -> { + if (pathList.size() < 2) { + return null; + } + + String rootParent = pathList.get(0).toUpperCase(); + if ("PROGRAM FILES".equals(rootParent) || "PROGRAM FILES (X86)".equals(rootParent)) { + return pathList.get(1); + } else { + return null; + } + }, + // if there is a folder named "APPLICATION DATA" or "APPDATA" + (pathList) -> { + for (String pathEl : pathList) { + String uppered = pathEl.toUpperCase(); + if ("APPLICATION DATA".equals(uppered) || "APPDATA".equals(uppered)) { + return "AppData"; + } + } + return null; + } + ); + private static final BlackboardArtifact.Type TYPE_DEVICE_ATTACHED = new BlackboardArtifact.Type(ARTIFACT_TYPE.TSK_DEVICE_ATTACHED); private static final BlackboardArtifact.Type TYPE_WEB_HISTORY = new BlackboardArtifact.Type(ARTIFACT_TYPE.TSK_WEB_HISTORY); @@ -69,17 +101,51 @@ public class UserActivitySummary implements DefaultArtifactUpdateGovernor { private static final BlackboardAttribute.Type TYPE_DATETIME_START = new BlackboardAttribute.Type(ATTRIBUTE_TYPE.TSK_DATETIME_START); private static final BlackboardAttribute.Type TYPE_DATETIME_END = new BlackboardAttribute.Type(ATTRIBUTE_TYPE.TSK_DATETIME_END); private static final BlackboardAttribute.Type TYPE_DOMAIN = new BlackboardAttribute.Type(ATTRIBUTE_TYPE.TSK_DOMAIN); + private static final BlackboardAttribute.Type TYPE_PROG_NAME = new BlackboardAttribute.Type(ATTRIBUTE_TYPE.TSK_PROG_NAME); + private static final BlackboardAttribute.Type TYPE_PATH = new BlackboardAttribute.Type(ATTRIBUTE_TYPE.TSK_PATH); + private static final BlackboardAttribute.Type TYPE_COUNT = new BlackboardAttribute.Type(ATTRIBUTE_TYPE.TSK_COUNT); + + private static final String NTOS_BOOT_IDENTIFIER = "NTOSBOOT"; + private static final String WINDOWS_PREFIX = "/WINDOWS"; private static final Comparator TOP_ACCOUNT_RESULT_DATE_COMPARE = (a, b) -> a.getLastAccess().compareTo(b.getLastAccess()); private static final Comparator TOP_WEBSEARCH_RESULT_DATE_COMPARE = (a, b) -> a.getDateAccessed().compareTo(b.getDateAccessed()); + /** + * Sorts TopProgramsResults pushing highest run time count then most recent + * run and then the program name that comes earliest in the alphabet. + */ + private static final Comparator TOP_PROGRAMS_RESULT_COMPARE = (a, b) -> { + // first priority for sorting is the run times + // if non-0, this is the return value for the comparator + int runTimesCompare = nullableCompare(a.getRunTimes(), b.getRunTimes()); + if (runTimesCompare != 0) { + return -runTimesCompare; + } + + // second priority for sorting is the last run date + // if non-0, this is the return value for the comparator + int lastRunCompare = nullableCompare( + a.getLastRun() == null ? null : a.getLastRun().getTime(), + b.getLastRun() == null ? null : b.getLastRun().getTime()); + + if (lastRunCompare != 0) { + return -lastRunCompare; + } + + // otherwise sort alphabetically + return (a.getProgramName() == null ? "" : a.getProgramName()) + .compareToIgnoreCase((b.getProgramName() == null ? "" : b.getProgramName())); + }; + private static final Set ARTIFACT_UPDATE_TYPE_IDS = new HashSet<>(Arrays.asList( ARTIFACT_TYPE.TSK_WEB_SEARCH_QUERY.getTypeID(), ARTIFACT_TYPE.TSK_MESSAGE.getTypeID(), ARTIFACT_TYPE.TSK_EMAIL_MSG.getTypeID(), ARTIFACT_TYPE.TSK_CALLLOG.getTypeID(), ARTIFACT_TYPE.TSK_DEVICE_ATTACHED.getTypeID(), - ARTIFACT_TYPE.TSK_WEB_HISTORY.getTypeID() + ARTIFACT_TYPE.TSK_WEB_HISTORY.getTypeID(), + ARTIFACT_TYPE.TSK_PROG_RUN.getTypeID() )); private static final Set DEVICE_EXCLUDE_LIST = new HashSet<>(Arrays.asList("ROOT_HUB", "ROOT_HUB20")); @@ -539,6 +605,189 @@ public class UserActivitySummary implements DefaultArtifactUpdateGovernor { .collect(Collectors.toList()); } + /** + * Determines a short folder name if any. Otherwise, returns empty string. + * + * @param strPath The string path. + * @param applicationName The application name. + * + * @return The short folder name or empty string if not found. + */ + public String getShortFolderName(String strPath, String applicationName) { + if (strPath == null) { + return ""; + } + + List pathEls = new ArrayList<>(Arrays.asList(applicationName)); + + File file = new File(strPath); + while (file != null && org.apache.commons.lang.StringUtils.isNotBlank(file.getName())) { + pathEls.add(file.getName()); + file = file.getParentFile(); + } + + Collections.reverse(pathEls); + + for (Function, String> matchEntry : SHORT_FOLDER_MATCHERS) { + String result = matchEntry.apply(pathEls); + if (org.apache.commons.lang.StringUtils.isNotBlank(result)) { + return result; + } + } + + return ""; + } + + /** + * Creates a TopProgramsResult from a TSK_PROG_RUN blackboard artifact. + * + * @param artifact The TSK_PROG_RUN blackboard artifact. + * + * @return The generated TopProgramsResult. + */ + private TopProgramsResult getTopProgramsResult(BlackboardArtifact artifact) { + String programName = DataSourceInfoUtilities.getStringOrNull(artifact, TYPE_PROG_NAME); + + // ignore items with no name or a ntos boot identifier + if (StringUtils.isBlank(programName) || NTOS_BOOT_IDENTIFIER.equalsIgnoreCase(programName)) { + return null; + } + + String path = DataSourceInfoUtilities.getStringOrNull(artifact, TYPE_PATH); + + // ignore windows directory + if (StringUtils.startsWithIgnoreCase(path, WINDOWS_PREFIX)) { + return null; + } + + Integer count = DataSourceInfoUtilities.getIntOrNull(artifact, TYPE_COUNT); + Long longCount = (count == null) ? null : (long) count; + + return new TopProgramsResult( + programName, + path, + longCount, + DataSourceInfoUtilities.getDateOrNull(artifact, TYPE_DATETIME) + ); + } + + /** + * Retrieves the maximum date given two (possibly null) dates. + * + * @param date1 First date. + * @param date2 Second date. + * + * @return The maximum non-null date or null if both items are null. + */ + private static Date getMax(Date date1, Date date2) { + if (date1 == null) { + return date2; + } else if (date2 == null) { + return date1; + } else { + return date1.compareTo(date2) > 0 ? date1 : date2; + } + } + + /** + * Returns the compare value favoring the higher non-null number. + * + * @param long1 First possibly null long. + * @param long2 Second possibly null long. + * + * @return Returns the compare value: 1,0,-1 favoring the higher non-null + * value. + */ + private static int nullableCompare(Long long1, Long long2) { + if (long1 == null && long2 == null) { + return 0; + } else if (long1 != null && long2 == null) { + return 1; + } else if (long1 == null && long2 != null) { + return -1; + } + + return Long.compare(long1, long2); + } + + /** + * Returns true if number is non-null and higher than 0. + * + * @param longNum The number. + * + * @return True if non-null and higher than 0. + */ + private static boolean isPositiveNum(Long longNum) { + return longNum != null && longNum > 0; + } + + + /** + * Retrieves the top programs results for the given data source limited to + * the count provided as a parameter. The highest run times are at the top + * of the list. If that information isn't available the last run date is + * used. If both, the last run date and the number of run times are + * unavailable, the programs will be sorted alphabetically, the count will + * be ignored and all items will be returned. + * + * @param dataSource The datasource. If the datasource is null, an empty + * list will be returned. + * @param count The number of results to return. This value must be > 0 + * or an IllegalArgumentException will be thrown. + * + * @return The sorted list and limited to the count if last run or run count + * information is available on any item. + * + * @throws SleuthkitCaseProviderException + * @throws TskCoreException + */ + public List getTopPrograms(DataSource dataSource, int count) throws SleuthkitCaseProviderException, TskCoreException { + assertValidCount(count); + + if (dataSource == null) { + return Collections.emptyList(); + } + + // Get TopProgramsResults for each TSK_PROG_RUN artifact + Collection results = caseProvider.get().getBlackboard().getArtifacts(ARTIFACT_TYPE.TSK_PROG_RUN.getTypeID(), dataSource.getId()) + .stream() + // convert to a TopProgramsResult object or null if missing critical information + .map((art) -> getTopProgramsResult(art)) + // remove any null items + .filter((res) -> res != null) + // group by the program name and program path + // The value will be a TopProgramsResult with the max run times + // and most recent last run date for each program name / program path pair. + .collect(Collectors.toMap( + res -> Pair.of(res.getProgramName(), res.getProgramPath()), + res -> res, + (res1, res2) -> { + return new TopProgramsResult( + res1.getProgramName(), + res1.getProgramPath(), + getMax(res1.getRunTimes(), res2.getRunTimes()), + getMax(res1.getLastRun(), res2.getLastRun())); + })).values(); + + List orderedResults = results.stream() + .sorted(TOP_PROGRAMS_RESULT_COMPARE) + .collect(Collectors.toList()); + + // only limit the list to count if there is no last run date and no run times. + if (!orderedResults.isEmpty()) { + TopProgramsResult topResult = orderedResults.get(0); + // if run times / last run information is available, the first item should have some value, + // and then the items should be limited accordingly. + if (isPositiveNum(topResult.getRunTimes()) + || (topResult.getLastRun() != null && isPositiveNum(topResult.getLastRun().getTime()))) { + return orderedResults.stream().limit(count).collect(Collectors.toList()); + } + } + + // otherwise return the alphabetized list with no limit applied. + return orderedResults; + } + /** * Object containing information about a web search artifact. */ @@ -722,4 +971,57 @@ public class UserActivitySummary implements DefaultArtifactUpdateGovernor { return lastVisit; } } + + /** + * Describes a result of a program run on a datasource. + */ + public static class TopProgramsResult { + + private final String programName; + private final String programPath; + private final Long runTimes; + private final Date lastRun; + + /** + * Main constructor. + * + * @param programName The name of the program. + * @param programPath The path of the program. + * @param runTimes The number of runs. + */ + TopProgramsResult(String programName, String programPath, Long runTimes, Date lastRun) { + this.programName = programName; + this.programPath = programPath; + this.runTimes = runTimes; + this.lastRun = lastRun; + } + + /** + * @return The name of the program + */ + public String getProgramName() { + return programName; + } + + /** + * @return The path of the program. + */ + public String getProgramPath() { + return programPath; + } + + /** + * @return The number of run times or null if not present. + */ + public Long getRunTimes() { + return runTimes; + } + + /** + * @return The last time the program was run or null if not present. + */ + public Date getLastRun() { + return lastRun; + } + } } diff --git a/Core/src/org/sleuthkit/autopsy/datasourcesummary/ui/UserActivityPanel.java b/Core/src/org/sleuthkit/autopsy/datasourcesummary/ui/UserActivityPanel.java index 476d8c5e6f..baab54217c 100644 --- a/Core/src/org/sleuthkit/autopsy/datasourcesummary/ui/UserActivityPanel.java +++ b/Core/src/org/sleuthkit/autopsy/datasourcesummary/ui/UserActivityPanel.java @@ -29,12 +29,11 @@ import org.apache.commons.lang.StringUtils; import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.datasourcesummary.datamodel.IngestModuleCheckUtil; import org.sleuthkit.autopsy.datasourcesummary.datamodel.UserActivitySummary; -import org.sleuthkit.autopsy.datasourcesummary.datamodel.TopProgramsSummary; import org.sleuthkit.autopsy.datasourcesummary.datamodel.UserActivitySummary.TopAccountResult; import org.sleuthkit.autopsy.datasourcesummary.datamodel.UserActivitySummary.TopDeviceAttachedResult; import org.sleuthkit.autopsy.datasourcesummary.datamodel.UserActivitySummary.TopWebSearchResult; -import org.sleuthkit.autopsy.datasourcesummary.datamodel.TopProgramsSummary.TopProgramsResult; import org.sleuthkit.autopsy.datasourcesummary.datamodel.UserActivitySummary.TopDomainsResult; +import org.sleuthkit.autopsy.datasourcesummary.datamodel.UserActivitySummary.TopProgramsResult; import org.sleuthkit.autopsy.datasourcesummary.uiutils.CellModelTableCellRenderer.DefaultCellModel; import org.sleuthkit.autopsy.datasourcesummary.uiutils.DataFetchWorker.DataFetchComponents; import org.sleuthkit.autopsy.datasourcesummary.uiutils.IngestRunningLabel; @@ -227,35 +226,30 @@ public class UserActivityPanel extends BaseDataSourceSummaryPanel { private final IngestRunningLabel ingestRunningLabel = new IngestRunningLabel(); private final List> dataFetchComponents; - private final TopProgramsSummary topProgramsData; - + private final UserActivitySummary userActivityData; + /** * Creates a new UserActivityPanel. */ public UserActivityPanel() { - this(new TopProgramsSummary(), new UserActivitySummary()); + this(new UserActivitySummary()); } /** * Creates a new UserActivityPanel. * - * @param topProgramsData Class from which to obtain top programs data. * @param userActivityData Class from which to obtain remaining user * activity data. */ - public UserActivityPanel( - TopProgramsSummary topProgramsData, - UserActivitySummary userActivityData) { - - super(topProgramsData, userActivityData); - - this.topProgramsData = topProgramsData; + public UserActivityPanel(UserActivitySummary userActivityData) { + super(userActivityData); + this.userActivityData = userActivityData; // set up data acquisition methods this.dataFetchComponents = Arrays.asList( // top programs query new DataFetchComponents>( - (dataSource) -> topProgramsData.getTopPrograms(dataSource, TOP_PROGS_COUNT), + (dataSource) -> userActivityData.getTopPrograms(dataSource, TOP_PROGS_COUNT), (result) -> { showResultWithModuleCheck(topProgramsTable, result, IngestModuleCheckUtil.RECENT_ACTIVITY_FACTORY, @@ -307,7 +301,7 @@ public class UserActivityPanel extends BaseDataSourceSummaryPanel { * @return The underlying short folder name if one exists. */ private String getShortFolderName(String path, String appName) { - return this.topProgramsData.getShortFolderName(path, appName); + return this.userActivityData.getShortFolderName(path, appName); } @Override diff --git a/Core/src/org/sleuthkit/autopsy/filesearch/Bundle.properties b/Core/src/org/sleuthkit/autopsy/filesearch/Bundle.properties index 8960448da4..034149f5ec 100644 --- a/Core/src/org/sleuthkit/autopsy/filesearch/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/filesearch/Bundle.properties @@ -49,6 +49,7 @@ SizeSearchPanel.sizeCompareComboBox.lessThan=less than FileSearchPanel.searchButton.text=Search MimeTypePanel.mimeTypeCheckBox.text=MIME Type: HashSearchPanel.md5CheckBox.text=MD5: +Sha256HashSearchPanel.sha256CheckBox.text=SHA-256: HashSearchPanel.emptyHashMsg.text=Must enter something for hash search. FileSearchPanel.errorLabel.text=\ DataSourcePanel.dataSourceCheckBox.label=Data Source: @@ -58,3 +59,5 @@ DataSourcePanel.dataSourceNoteLabel.text=*Note: Multiple data sources can be sel DateSearchPanel.noLimitLabel.text=*Empty fields mean "No Limit" DateSearchPanel.dateFormatLabel.text=*The date format is mm/dd/yyyy MimeTypePanel.noteLabel.text=*Note: Multiple MIME types can be selected +HashSearchPanel.sha256CheckBox.text=SHA-256: +HashSearchPanel.sha256TextField.text= diff --git a/Core/src/org/sleuthkit/autopsy/filesearch/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/filesearch/Bundle.properties-MERGED index b304bc1342..93c5ed7987 100755 --- a/Core/src/org/sleuthkit/autopsy/filesearch/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/filesearch/Bundle.properties-MERGED @@ -5,7 +5,9 @@ FileSearchPanel.emptyNode.display.text=No results found. HashSearchFilter.errorMessage.emptyHash=Hash data is empty. HashSearchFilter.errorMessage.wrongCharacter=MD5 contains invalid hex characters. # {0} - hash data length -HashSearchFilter.errorMessage.wrongLength=Input length({0}), doesn''t match the MD5 length(32). +HashSearchFilter.errorMessage.wrongLengthMd5=Input length({0}), doesn''t match the MD5 length(32). +# {0} - hash data length +HashSearchFilter.errorMessage.wrongLengthSha256=Input length({0}), doesn''t match the SHA-256 length(64). KnownStatusSearchFilter.errorMessage.noKnownStatusCheckboxSelected=At least one known status checkbox must be selected. MimeTypeFilter.errorMessage.emptyMimeType=At least one MIME type must be selected. NameSearchFilter.errorMessage.emtpyName=Please input a name to search. @@ -63,6 +65,7 @@ SizeSearchPanel.sizeCompareComboBox.lessThan=less than FileSearchPanel.searchButton.text=Search MimeTypePanel.mimeTypeCheckBox.text=MIME Type: HashSearchPanel.md5CheckBox.text=MD5: +Sha256HashSearchPanel.sha256CheckBox.text=SHA-256: HashSearchPanel.emptyHashMsg.text=Must enter something for hash search. FileSearchPanel.errorLabel.text=\ DataSourcePanel.dataSourceCheckBox.label=Data Source: @@ -72,3 +75,5 @@ DataSourcePanel.dataSourceNoteLabel.text=*Note: Multiple data sources can be sel DateSearchPanel.noLimitLabel.text=*Empty fields mean "No Limit" DateSearchPanel.dateFormatLabel.text=*The date format is mm/dd/yyyy MimeTypePanel.noteLabel.text=*Note: Multiple MIME types can be selected +HashSearchPanel.sha256CheckBox.text=SHA-256: +HashSearchPanel.sha256TextField.text= diff --git a/Core/src/org/sleuthkit/autopsy/filesearch/HashSearchFilter.java b/Core/src/org/sleuthkit/autopsy/filesearch/HashSearchFilter.java index d77fe5826e..1585f1051e 100644 --- a/Core/src/org/sleuthkit/autopsy/filesearch/HashSearchFilter.java +++ b/Core/src/org/sleuthkit/autopsy/filesearch/HashSearchFilter.java @@ -41,18 +41,36 @@ class HashSearchFilter extends AbstractFileSearchFilter { @Override public boolean isEnabled() { - return this.getComponent().getHashCheckBox().isSelected(); + return (this.getComponent().getMd5HashCheckBox().isSelected() + || this.getComponent().getSha256HashCheckBox().isSelected()); } @Override public String getPredicate() throws FilterValidationException { - String md5Hash = this.getComponent().getSearchTextField().getText(); + String predicate = ""; + if (this.getComponent().getMd5HashCheckBox().isSelected()) { + String md5Hash = this.getComponent().getMd5TextField().getText(); - if (md5Hash.isEmpty()) { - throw new FilterValidationException(EMPTY_HASH_MESSAGE); + if (md5Hash.isEmpty()) { + throw new FilterValidationException(EMPTY_HASH_MESSAGE); + } + predicate = "md5 = '" + md5Hash.toLowerCase() + "'"; //NON-NLS + } + + if (this.getComponent().getSha256HashCheckBox().isSelected()) { + String sha256Hash = this.getComponent().getSha256TextField().getText(); + + if (sha256Hash.isEmpty()) { + throw new FilterValidationException(EMPTY_HASH_MESSAGE); + } + if (predicate.isEmpty()) { + predicate = "sha256 = '" + sha256Hash.toLowerCase() + "'"; //NON-NLS + } else { + predicate = "( " + predicate + " AND sha256 = '" + sha256Hash.toLowerCase() + "')"; //NON-NLS + } } - return "md5 = '" + md5Hash.toLowerCase() + "'"; //NON-NLS + return predicate; } @Override @@ -63,23 +81,45 @@ class HashSearchFilter extends AbstractFileSearchFilter { @Override @Messages({ "HashSearchFilter.errorMessage.emptyHash=Hash data is empty.", - "# {0} - hash data length", "HashSearchFilter.errorMessage.wrongLength=Input length({0}), doesn''t match the MD5 length(32).", + "# {0} - hash data length", + "HashSearchFilter.errorMessage.wrongLengthMd5=Input length({0}), doesn''t match the MD5 length(32).", + "# {0} - hash data length", + "HashSearchFilter.errorMessage.wrongLengthSha256=Input length({0}), doesn''t match the SHA-256 length(64).", "HashSearchFilter.errorMessage.wrongCharacter=MD5 contains invalid hex characters." }) public boolean isValid() { - String inputHashData = this.getComponent().getSearchTextField().getText(); - if (inputHashData.isEmpty()) { - setLastError(Bundle.HashSearchFilter_errorMessage_emptyHash()); - return false; + if (this.getComponent().getMd5HashCheckBox().isSelected()) { + String inputHashData = this.getComponent().getMd5TextField().getText(); + if (inputHashData.isEmpty()) { + setLastError(Bundle.HashSearchFilter_errorMessage_emptyHash()); + return false; + } + if (inputHashData.length() != 32) { + setLastError(Bundle.HashSearchFilter_errorMessage_wrongLengthMd5(inputHashData.length())); + return false; + } + if (!inputHashData.matches("[0-9a-fA-F]+")) { + setLastError(Bundle.HashSearchFilter_errorMessage_wrongCharacter()); + return false; + } } - if (inputHashData.length() != 32) { - setLastError(Bundle.HashSearchFilter_errorMessage_wrongLength(inputHashData.length())); - return false; - } - if (!inputHashData.matches("[0-9a-fA-F]+")) { - setLastError(Bundle.HashSearchFilter_errorMessage_wrongCharacter()); - return false; + + if (this.getComponent().getSha256HashCheckBox().isSelected()) { + String inputHashData = this.getComponent().getSha256TextField().getText(); + if (inputHashData.isEmpty()) { + setLastError(Bundle.HashSearchFilter_errorMessage_emptyHash()); + return false; + } + if (inputHashData.length() != 64) { + setLastError(Bundle.HashSearchFilter_errorMessage_wrongLengthSha256(inputHashData.length())); + return false; + } + if (!inputHashData.matches("[0-9a-fA-F]+")) { + setLastError(Bundle.HashSearchFilter_errorMessage_wrongCharacter()); + return false; + } } + return true; } } diff --git a/Core/src/org/sleuthkit/autopsy/filesearch/HashSearchPanel.form b/Core/src/org/sleuthkit/autopsy/filesearch/HashSearchPanel.form index 0ad53c9653..88fd9cabfc 100644 --- a/Core/src/org/sleuthkit/autopsy/filesearch/HashSearchPanel.form +++ b/Core/src/org/sleuthkit/autopsy/filesearch/HashSearchPanel.form @@ -56,34 +56,67 @@ - + + + + + + + + + + + + - - - - - + + + + + + + + + + - + - + - + + + + + + + + + + + + + + + + + + diff --git a/Core/src/org/sleuthkit/autopsy/filesearch/HashSearchPanel.java b/Core/src/org/sleuthkit/autopsy/filesearch/HashSearchPanel.java index f166d01264..66d5396420 100644 --- a/Core/src/org/sleuthkit/autopsy/filesearch/HashSearchPanel.java +++ b/Core/src/org/sleuthkit/autopsy/filesearch/HashSearchPanel.java @@ -45,19 +45,19 @@ class HashSearchPanel extends javax.swing.JPanel { private void customizeComponents() { - searchTextField.setComponentPopupMenu(rightClickMenu); + md5TextField.setComponentPopupMenu(rightClickMenu); ActionListener actList = new ActionListener() { @Override public void actionPerformed(ActionEvent e) { JMenuItem jmi = (JMenuItem) e.getSource(); if (jmi.equals(cutMenuItem)) { - searchTextField.cut(); + md5TextField.cut(); } else if (jmi.equals(copyMenuItem)) { - searchTextField.copy(); + md5TextField.copy(); } else if (jmi.equals(pasteMenuItem)) { - searchTextField.paste(); + md5TextField.paste(); } else if (jmi.equals(selectAllMenuItem)) { - searchTextField.selectAll(); + md5TextField.selectAll(); } } }; @@ -65,7 +65,24 @@ class HashSearchPanel extends javax.swing.JPanel { copyMenuItem.addActionListener(actList); pasteMenuItem.addActionListener(actList); selectAllMenuItem.addActionListener(actList); - this.searchTextField.getDocument().addDocumentListener(new DocumentListener() { + this.md5TextField.getDocument().addDocumentListener(new DocumentListener() { + @Override + public void insertUpdate(DocumentEvent e) { + firePropertyChange(FileSearchPanel.EVENT.CHECKED.toString(), null, null); + } + + @Override + public void removeUpdate(DocumentEvent e) { + firePropertyChange(FileSearchPanel.EVENT.CHECKED.toString(), null, null); + } + + @Override + public void changedUpdate(DocumentEvent e) { + firePropertyChange(FileSearchPanel.EVENT.CHECKED.toString(), null, null); + } + }); + + this.sha256TextField.getDocument().addDocumentListener(new DocumentListener() { @Override public void insertUpdate(DocumentEvent e) { firePropertyChange(FileSearchPanel.EVENT.CHECKED.toString(), null, null); @@ -84,17 +101,27 @@ class HashSearchPanel extends javax.swing.JPanel { } - JCheckBox getHashCheckBox() { - return hashCheckBox; + JCheckBox getMd5HashCheckBox() { + return md5CheckBox; } - JTextField getSearchTextField() { - return searchTextField; + JTextField getMd5TextField() { + return md5TextField; + } + + JCheckBox getSha256HashCheckBox() { + return sha256CheckBox; + } + + JTextField getSha256TextField() { + return sha256TextField; } void setComponentsEnabled() { - boolean enabled = hashCheckBox.isSelected(); - this.searchTextField.setEnabled(enabled); + boolean md5Enabled = md5CheckBox.isSelected(); + this.md5TextField.setEnabled(md5Enabled); + boolean sha256Enabled = sha256CheckBox.isSelected(); + this.sha256TextField.setEnabled(sha256Enabled); } /** @@ -111,8 +138,10 @@ class HashSearchPanel extends javax.swing.JPanel { copyMenuItem = new javax.swing.JMenuItem(); pasteMenuItem = new javax.swing.JMenuItem(); selectAllMenuItem = new javax.swing.JMenuItem(); - hashCheckBox = new javax.swing.JCheckBox(); - searchTextField = new javax.swing.JTextField(); + md5CheckBox = new javax.swing.JCheckBox(); + md5TextField = new javax.swing.JTextField(); + sha256CheckBox = new javax.swing.JCheckBox(); + sha256TextField = new javax.swing.JTextField(); cutMenuItem.setText(org.openide.util.NbBundle.getMessage(HashSearchPanel.class, "NameSearchPanel.cutMenuItem.text")); // NOI18N rightClickMenu.add(cutMenuItem); @@ -126,48 +155,75 @@ class HashSearchPanel extends javax.swing.JPanel { selectAllMenuItem.setText(org.openide.util.NbBundle.getMessage(HashSearchPanel.class, "NameSearchPanel.selectAllMenuItem.text")); // NOI18N rightClickMenu.add(selectAllMenuItem); - hashCheckBox.setText(org.openide.util.NbBundle.getMessage(HashSearchPanel.class, "HashSearchPanel.md5CheckBox.text")); // NOI18N - hashCheckBox.addActionListener(new java.awt.event.ActionListener() { + md5CheckBox.setText(org.openide.util.NbBundle.getMessage(HashSearchPanel.class, "HashSearchPanel.md5CheckBox.text")); // NOI18N + md5CheckBox.addActionListener(new java.awt.event.ActionListener() { public void actionPerformed(java.awt.event.ActionEvent evt) { - hashCheckBoxActionPerformed(evt); + md5CheckBoxActionPerformed(evt); } }); + sha256CheckBox.setText(org.openide.util.NbBundle.getMessage(HashSearchPanel.class, "HashSearchPanel.sha256CheckBox.text")); // NOI18N + sha256CheckBox.addActionListener(new java.awt.event.ActionListener() { + public void actionPerformed(java.awt.event.ActionEvent evt) { + sha256CheckBoxActionPerformed(evt); + } + }); + + sha256TextField.setText(org.openide.util.NbBundle.getMessage(HashSearchPanel.class, "HashSearchPanel.sha256TextField.text")); // NOI18N + javax.swing.GroupLayout layout = new javax.swing.GroupLayout(this); this.setLayout(layout); layout.setHorizontalGroup( layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addGroup(layout.createSequentialGroup() .addGap(0, 0, 0) - .addComponent(hashCheckBox) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) - .addComponent(searchTextField, javax.swing.GroupLayout.PREFERRED_SIZE, 247, javax.swing.GroupLayout.PREFERRED_SIZE) - .addGap(0, 0, 0)) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addGroup(layout.createSequentialGroup() + .addComponent(sha256CheckBox) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addComponent(sha256TextField, javax.swing.GroupLayout.DEFAULT_SIZE, 254, Short.MAX_VALUE)) + .addGroup(layout.createSequentialGroup() + .addComponent(md5CheckBox) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addComponent(md5TextField))) + .addContainerGap()) ); layout.setVerticalGroup( layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) - .addComponent(hashCheckBox) - .addComponent(searchTextField, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)) + .addGroup(layout.createSequentialGroup() + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) + .addComponent(md5CheckBox) + .addComponent(md5TextField, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) + .addComponent(sha256CheckBox) + .addComponent(sha256TextField, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE))) ); }// //GEN-END:initComponents - private void hashCheckBoxActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_hashCheckBoxActionPerformed + private void md5CheckBoxActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_md5CheckBoxActionPerformed setComponentsEnabled(); firePropertyChange(FileSearchPanel.EVENT.CHECKED.toString(), null, null); - }//GEN-LAST:event_hashCheckBoxActionPerformed + }//GEN-LAST:event_md5CheckBoxActionPerformed + + private void sha256CheckBoxActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_sha256CheckBoxActionPerformed + setComponentsEnabled(); + firePropertyChange(FileSearchPanel.EVENT.CHECKED.toString(), null, null); + }//GEN-LAST:event_sha256CheckBoxActionPerformed // Variables declaration - do not modify//GEN-BEGIN:variables private javax.swing.JMenuItem copyMenuItem; private javax.swing.JMenuItem cutMenuItem; - private javax.swing.JCheckBox hashCheckBox; + private javax.swing.JCheckBox md5CheckBox; + private javax.swing.JTextField md5TextField; private javax.swing.JMenuItem pasteMenuItem; private javax.swing.JPopupMenu rightClickMenu; - private javax.swing.JTextField searchTextField; private javax.swing.JMenuItem selectAllMenuItem; + private javax.swing.JCheckBox sha256CheckBox; + private javax.swing.JTextField sha256TextField; // End of variables declaration//GEN-END:variables void addActionListener(ActionListener l) { - searchTextField.addActionListener(l); + md5TextField.addActionListener(l); } } diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java index 5921071e65..8e2f7cf86d 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobPipeline.java @@ -1274,6 +1274,9 @@ final class IngestJobPipeline { } } } + + // If a data source had no tasks in progress it may now be complete. + checkForStageCompleted(); } /** diff --git a/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/HashDbIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/HashDbIngestModule.java index 65bf047fa1..19b9ff28b7 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/HashDbIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/HashDbIngestModule.java @@ -18,8 +18,8 @@ */ package org.sleuthkit.autopsy.modules.hashdatabase; -import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.List; @@ -208,10 +208,17 @@ public class HashDbIngestModule implements FileIngestModule { // Safely get a reference to the totalsForIngestJobs object IngestJobTotals totals = getTotalsForIngestJobs(jobId); - // calc hash value - String md5Hash = getHash(file, totals); - if (md5Hash == null) { - return ProcessResult.ERROR; + // calc hash values + try { + calculateHashes(file, totals); + } catch (TskCoreException ex) { + logger.log(Level.WARNING, String.format("Error calculating hash of file '%s' (id=%d).", file.getName(), file.getId()), ex); //NON-NLS + services.postMessage(IngestMessage.createErrorMessage( + HashLookupModuleFactory.getModuleName(), + NbBundle.getMessage(this.getClass(), "HashDbIngestModule.fileReadErrorMsg", file.getName()), + NbBundle.getMessage(this.getClass(), "HashDbIngestModule.calcHashValueErr", + file.getParentPath() + file.getName(), + file.isMetaFlagSet(TskData.TSK_FS_META_FLAG_ENUM.ALLOC) ? "Allocated File" : "Deleted File"))); } // the processing result of handling this file @@ -451,50 +458,46 @@ public class HashDbIngestModule implements FileIngestModule { } /** - * Retrieves the md5 hash for a file or generates one if no one exists on - * the file. + * Generates hashes for the given file if they haven't already been set. + * Hashes are saved to the AbstractFile object. * * @param file The file in order to determine the hash. * @param totals The timing metrics for this process. - * - * @return The found or determined md5 hash or null if none could be - * determined. */ - private String getHash(AbstractFile file, IngestJobTotals totals) { + private void calculateHashes(AbstractFile file, IngestJobTotals totals) throws TskCoreException { + + // First check if we've already calculated the hashes. String md5Hash = file.getMd5Hash(); - if (md5Hash != null && md5Hash.isEmpty()) { - return md5Hash; + String sha256Hash = file.getSha256Hash(); + if ((md5Hash != null && ! md5Hash.isEmpty()) + && (sha256Hash != null && ! sha256Hash.isEmpty())) { + return; } - try { - TimingMetric metric = HealthMonitor.getTimingMetric("Disk Reads: Hash calculation"); - long calcstart = System.currentTimeMillis(); - md5Hash = HashUtility.calculateMd5Hash(file); - if (file.getSize() > 0) { - // Surprisingly, the hash calculation does not seem to be correlated that - // strongly with file size until the files get large. - // Only normalize if the file size is greater than ~1MB. - if (file.getSize() < 1000000) { - HealthMonitor.submitTimingMetric(metric); - } else { - // In testing, this normalization gave reasonable resuls - HealthMonitor.submitNormalizedTimingMetric(metric, file.getSize() / 500000); - } + TimingMetric metric = HealthMonitor.getTimingMetric("Disk Reads: Hash calculation"); + long calcstart = System.currentTimeMillis(); + List newHashResults = + HashUtility.calculateHashes(file, Arrays.asList(HashUtility.HashType.MD5,HashUtility.HashType.SHA256 )); + if (file.getSize() > 0) { + // Surprisingly, the hash calculation does not seem to be correlated that + // strongly with file size until the files get large. + // Only normalize if the file size is greater than ~1MB. + if (file.getSize() < 1000000) { + HealthMonitor.submitTimingMetric(metric); + } else { + // In testing, this normalization gave reasonable resuls + HealthMonitor.submitNormalizedTimingMetric(metric, file.getSize() / 500000); } - file.setMd5Hash(md5Hash); - long delta = (System.currentTimeMillis() - calcstart); - totals.totalCalctime.addAndGet(delta); - return md5Hash; - } catch (IOException ex) { - logger.log(Level.WARNING, String.format("Error calculating hash of file '%s' (id=%d).", file.getName(), file.getId()), ex); //NON-NLS - services.postMessage(IngestMessage.createErrorMessage( - HashLookupModuleFactory.getModuleName(), - NbBundle.getMessage(this.getClass(), "HashDbIngestModule.fileReadErrorMsg", file.getName()), - NbBundle.getMessage(this.getClass(), "HashDbIngestModule.calcHashValueErr", - file.getParentPath() + file.getName(), - file.isMetaFlagSet(TskData.TSK_FS_META_FLAG_ENUM.ALLOC) ? "Allocated File" : "Deleted File"))); - return null; } + for (HashUtility.HashResult hash : newHashResults) { + if (hash.getType().equals(HashUtility.HashType.MD5)) { + file.setMd5Hash(hash.getValue()); + } else if (hash.getType().equals(HashUtility.HashType.SHA256)) { + file.setSha256Hash(hash.getValue()); + } + } + long delta = (System.currentTimeMillis() - calcstart); + totals.totalCalctime.addAndGet(delta); } /** diff --git a/Core/src/org/sleuthkit/autopsy/modules/vmextractor/VMExtractorIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/vmextractor/VMExtractorIngestModule.java index 843b07a3d6..56e6b3fbdb 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/vmextractor/VMExtractorIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/vmextractor/VMExtractorIngestModule.java @@ -198,7 +198,6 @@ final class VMExtractorIngestModule extends DataSourceIngestModuleAdapter { // for extracted virtual machines there is no manifest XML file to read data source ID from so use parent data source ID. // ingest the data sources ingestVirtualMachineImage(Paths.get(folder, file)); - logger.log(Level.INFO, "Ingest complete for virtual machine file {0} in folder {1}", new Object[]{file, folder}); //NON-NLS } catch (InterruptedException ex) { logger.log(Level.INFO, "Interrupted while ingesting virtual machine file " + file + " in folder " + folder, ex); //NON-NLS } catch (IOException ex) { @@ -287,8 +286,8 @@ final class VMExtractorIngestModule extends DataSourceIngestModuleAdapter { } /* - * If the image was added, analyze it with the ingest modules for this - * ingest context. + * If the image was added, start analysis on it with the ingest modules for this + * ingest context. Note that this does not wait for ingest to complete. */ if (!dspCallback.vmDataSources.isEmpty()) { Case.getCurrentCaseThrows().notifyDataSourceAdded(dspCallback.vmDataSources.get(0), taskId); @@ -300,7 +299,7 @@ final class VMExtractorIngestModule extends DataSourceIngestModuleAdapter { IngestServices.getInstance().postMessage(IngestMessage.createMessage(IngestMessage.MessageType.INFO, VMExtractorIngestModuleFactory.getModuleName(), NbBundle.getMessage(this.getClass(), "VMExtractorIngestModule.addedVirtualMachineImage.message", vmFile.toString()))); - IngestManager.getInstance().queueIngestJob(dataSourceContent, ingestJobSettings); + IngestManager.getInstance().beginIngestJob(dataSourceContent, ingestJobSettings); } else { Case.getCurrentCaseThrows().notifyFailedAddingDataSource(taskId); } diff --git a/Core/src/org/sleuthkit/autopsy/report/modules/portablecase/PortableCaseReportModule.java b/Core/src/org/sleuthkit/autopsy/report/modules/portablecase/PortableCaseReportModule.java index 83bfb2fdae..67b253a1fa 100644 --- a/Core/src/org/sleuthkit/autopsy/report/modules/portablecase/PortableCaseReportModule.java +++ b/Core/src/org/sleuthkit/autopsy/report/modules/portablecase/PortableCaseReportModule.java @@ -1117,7 +1117,7 @@ public class PortableCaseReportModule implements ReportModule { newContent = portableSkCase.addLocalFile(abstractFile.getName(), relativePath, abstractFile.getSize(), abstractFile.getCtime(), abstractFile.getCrtime(), abstractFile.getAtime(), abstractFile.getMtime(), - abstractFile.getMd5Hash(), abstractFile.getKnown(), abstractFile.getMIMEType(), + abstractFile.getMd5Hash(), abstractFile.getSha256Hash(), abstractFile.getKnown(), abstractFile.getMIMEType(), true, TskData.EncodingType.NONE, newParent, trans); } catch (IOException ex) { diff --git a/Core/test/qa-functional/src/org/sleuthkit/autopsy/README.txt b/Core/test/qa-functional/src/org/sleuthkit/autopsy/README.txt index 139b74c67f..592bb50d72 100644 --- a/Core/test/qa-functional/src/org/sleuthkit/autopsy/README.txt +++ b/Core/test/qa-functional/src/org/sleuthkit/autopsy/README.txt @@ -1 +1,3 @@ -Netbeans platform does not properly scope classloaders while running qa-functional test code. The result is that NoClassDefError's occur in instances where an external jar (i.e. importing a class from common-io) is referenced in test code and the same external jar is referenced in multiple NBM's. Importing from external jars in qa-functional should be avoided. See jira issue 6954 for more information. \ No newline at end of file +Netbeans platform does not properly scope classloaders while running qa-functional test code. The result is that NoClassDefError's occur in instances where an external jar (i.e. importing a class from common-io) is referenced in test code and the same external jar is referenced in multiple NBM's. Importing from external jars in qa-functional should be avoided. See jira issue 6954 for more information. + +Many of the functional tests require external data sources. The ant target 'getTestDataFiles' must be run successfully to download the files. This should occur as a part of the 'test-init' ant target in build.xml. \ No newline at end of file diff --git a/Core/test/qa-functional/src/org/sleuthkit/autopsy/centralrepository/datamodel/CentralRepoDatamodelTest.java b/Core/test/qa-functional/src/org/sleuthkit/autopsy/centralrepository/datamodel/CentralRepoDatamodelTest.java index 357d5e0641..65907e8e87 100755 --- a/Core/test/qa-functional/src/org/sleuthkit/autopsy/centralrepository/datamodel/CentralRepoDatamodelTest.java +++ b/Core/test/qa-functional/src/org/sleuthkit/autopsy/centralrepository/datamodel/CentralRepoDatamodelTest.java @@ -1256,7 +1256,8 @@ public class CentralRepoDatamodelTest extends TestCase { List types = CentralRepository.getInstance().getDefinedCorrelationTypes(); // We expect 11 total - 10 default and the custom one made earlier - assertTrue("getDefinedCorrelationTypes returned " + types.size() + " entries - expected 11", types.size() == 11); + // Note: this test will need to be updated based on the current default items defined in the correlation_types table + assertTrue("getDefinedCorrelationTypes returned " + types.size() + " entries - expected 28", types.size() == 28); } catch (CentralRepoException ex) { Exceptions.printStackTrace(ex); Assert.fail(ex.getMessage()); @@ -1267,7 +1268,8 @@ public class CentralRepoDatamodelTest extends TestCase { List types = CentralRepository.getInstance().getEnabledCorrelationTypes(); // We expect 10 - the custom type is disabled - assertTrue("getDefinedCorrelationTypes returned " + types.size() + " enabled entries - expected 10", types.size() == 10); + // Note: this test will need to be updated based on the current default items defined in the correlation_types table + assertTrue("getDefinedCorrelationTypes returned " + types.size() + " enabled entries - expected 27", types.size() == 27); } catch (CentralRepoException ex) { Exceptions.printStackTrace(ex); Assert.fail(ex.getMessage()); @@ -1278,7 +1280,8 @@ public class CentralRepoDatamodelTest extends TestCase { List types = CentralRepository.getInstance().getSupportedCorrelationTypes(); // We expect 10 - the custom type is not supported - assertTrue("getDefinedCorrelationTypes returned " + types.size() + " supported entries - expected 10", types.size() == 10); + // Note: this test will need to be updated based on the current default items defined in the correlation_types table + assertTrue("getDefinedCorrelationTypes returned " + types.size() + " supported entries - expected 27", types.size() == 27); } catch (CentralRepoException ex) { Exceptions.printStackTrace(ex); Assert.fail(ex.getMessage()); diff --git a/Core/test/qa-functional/src/org/sleuthkit/autopsy/commonpropertiessearch/InterCaseTestUtils.java b/Core/test/qa-functional/src/org/sleuthkit/autopsy/commonpropertiessearch/InterCaseTestUtils.java index 014b08fe84..f76676cd25 100644 --- a/Core/test/qa-functional/src/org/sleuthkit/autopsy/commonpropertiessearch/InterCaseTestUtils.java +++ b/Core/test/qa-functional/src/org/sleuthkit/autopsy/commonpropertiessearch/InterCaseTestUtils.java @@ -218,7 +218,9 @@ class InterCaseTestUtils { // kitchenSink.add(keywordSearchTemplate); this.kitchenShink = new IngestJobSettings(InterCaseTestUtils.class.getCanonicalName(), IngestType.ALL_MODULES, kitchenSink); + } + void setupCorrelationTypes() { try { Collection types = CentralRepository.getInstance().getDefinedCorrelationTypes(); @@ -254,16 +256,16 @@ class InterCaseTestUtils { } } } - + Map getDataSourceMap() throws NoCurrentCaseException, TskCoreException, SQLException { return DataSourceLoader.getAllDataSources(); - } + } Map getCaseMap() throws CentralRepoException { if (CentralRepository.isEnabled()) { Map mapOfCaseIdsToCase = new HashMap<>(); - + for (CorrelationCase correlationCase : CentralRepository.getInstance().getCases()) { mapOfCaseIdsToCase.put(correlationCase.getDisplayName(), correlationCase.getID()); } @@ -300,7 +302,7 @@ class InterCaseTestUtils { RdbmsCentralRepoFactory centralRepoSchemaFactory = new RdbmsCentralRepoFactory(CentralRepoPlatforms.SQLITE, crSettings); centralRepoSchemaFactory.initializeDatabaseSchema(); centralRepoSchemaFactory.insertDefaultDatabaseContent(); - + crSettings.saveSettings(); CentralRepoDbManager.saveDbChoice(CentralRepoDbChoice.SQLITE); } @@ -313,10 +315,10 @@ class InterCaseTestUtils { * The length of caseNames and caseDataSourcePaths should be the same, and * cases should appear in the same order. * - * @param caseNames list case names - * @param caseDataSourcePaths two dimensional array listing the datasources - * in each case - * @param ingestJobSettings HashLookup FileType etc... + * @param caseNames list case names + * @param caseDataSourcePaths two dimensional array listing the datasources + * in each case + * @param ingestJobSettings HashLookup FileType etc... * @param caseReferenceToStore */ Case createCases(String[] caseNames, Path[][] caseDataSourcePaths, IngestJobSettings ingestJobSettings, String caseReferenceToStore) throws TskCoreException { diff --git a/Core/test/qa-functional/src/org/sleuthkit/autopsy/ingest/IngestFileFiltersTest.java b/Core/test/qa-functional/src/org/sleuthkit/autopsy/ingest/IngestFileFiltersTest.java index 262e54774f..ce0ffb1526 100755 --- a/Core/test/qa-functional/src/org/sleuthkit/autopsy/ingest/IngestFileFiltersTest.java +++ b/Core/test/qa-functional/src/org/sleuthkit/autopsy/ingest/IngestFileFiltersTest.java @@ -253,7 +253,7 @@ public class IngestFileFiltersTest extends NbTestCase { IngestUtils.runIngestJob(currentCase.getDataSources(), ingestJobSettings); FileManager fileManager = currentCase.getServices().getFileManager(); List results = fileManager.findFiles("%%"); - assertEquals(70, results.size()); + assertEquals(71, results.size()); int carvedJpgGifFiles = 0; for (AbstractFile file : results) { if (file.getNameExtension().equalsIgnoreCase("jpg") || file.getNameExtension().equalsIgnoreCase("gif")) { //Unalloc file and .jpg files in dir1, dir2, $CarvedFiles, root directory should have MIME type diff --git a/Core/test/unit/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/GetArtifactsTest.java b/Core/test/unit/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/DataSourceInfoUtilitiesTest.java similarity index 73% rename from Core/test/unit/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/GetArtifactsTest.java rename to Core/test/unit/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/DataSourceInfoUtilitiesTest.java index ebb7db601a..28b55155c1 100644 --- a/Core/test/unit/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/GetArtifactsTest.java +++ b/Core/test/unit/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/DataSourceInfoUtilitiesTest.java @@ -21,6 +21,7 @@ package org.sleuthkit.autopsy.datasourcesummary.datamodel; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.Date; import java.util.List; import org.junit.Assert; import org.junit.Rule; @@ -35,6 +36,7 @@ import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.autopsy.testutils.TskMockUtils; import static org.mockito.Mockito.*; +import org.sleuthkit.autopsy.testutils.RandomizationUtils; import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE; import org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE; import org.sleuthkit.datamodel.BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE; @@ -42,7 +44,7 @@ import org.sleuthkit.datamodel.BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALU /** * Unit tests for DataSourceInfoUtilities.getArtifacts */ -public class GetArtifactsTest { +public class DataSourceInfoUtilitiesTest { @Rule public ExpectedException thrown = ExpectedException.none(); @@ -145,38 +147,13 @@ public class GetArtifactsTest { List toRet = new ArrayList<>(); for (int i = 0; i < values.size(); i++) { - toRet.add(TskMockUtils.mockArtifact(new BlackboardArtifact.Type(artifactType), 1000 + i, dataSource, + toRet.add(TskMockUtils.getArtifact(new BlackboardArtifact.Type(artifactType), 1000 + i, dataSource, attrMaker.make(attrType, "TEST SOURCE", values.get(i)))); } return toRet; } - /** - * Returns list in 0, n-1, 1, n-2 ... order. Deterministic so same results - * each time, but not in original order. - * - * @return Mixed up list. - */ - private List getMixedUp(List list) { - int forward = 0; - int backward = list.size() - 1; - - List newList = new ArrayList<>(); - while (forward <= backward) { - newList.add(list.get(forward)); - - if (forward < backward) { - newList.add(list.get(backward)); - } - - forward++; - backward--; - } - - return newList; - } - /** * Does a basic test passing a list of generated artifacts in mixed up order * to DataSourceInfoUtilities.getArtifacts and expecting a sorted list to be @@ -194,11 +171,11 @@ public class GetArtifactsTest { private void testSorted(ARTIFACT_TYPE artifactType, ATTRIBUTE_TYPE attrType, List values, AttrMaker attrMaker, SortOrder sortOrder, int count) throws TskCoreException { - DataSource dataSource = TskMockUtils.mockDataSource(1); + DataSource dataSource = TskMockUtils.getDataSource(1); List sortedArtifacts = getArtifacts(artifactType, new BlackboardAttribute.Type(attrType), dataSource, values, attrMaker); - List mixedUpArtifacts = getMixedUp(sortedArtifacts); + List mixedUpArtifacts = RandomizationUtils.getMixedUp(sortedArtifacts); List expectedArtifacts = count == 0 ? sortedArtifacts @@ -250,17 +227,17 @@ public class GetArtifactsTest { } @Test - public void testSortAscending() throws TskCoreException { + public void getArtifacts_sortAscending() throws TskCoreException { testAscDesc(SortOrder.ASCENDING); } @Test - public void testSortDescending() throws TskCoreException { + public void getArtifacts_sortDescending() throws TskCoreException { testAscDesc(SortOrder.DESCENDING); } @Test - public void testLimits() throws TskCoreException { + public void getArtifacts_limits() throws TskCoreException { List integers = Arrays.asList(22, 31, 42, 50, 60); testSorted(ARTIFACT_TYPE.TSK_PROG_RUN, ATTRIBUTE_TYPE.TSK_COUNT, integers, BlackboardAttribute::new, SortOrder.ASCENDING, 3); testSorted(ARTIFACT_TYPE.TSK_PROG_RUN, ATTRIBUTE_TYPE.TSK_COUNT, integers, BlackboardAttribute::new, SortOrder.ASCENDING, 5); @@ -281,11 +258,11 @@ public class GetArtifactsTest { private void testFailOnBadAttrType(BlackboardArtifact.Type artifactType, BlackboardAttribute.Type attributeType, T val, AttrMaker attrMaker) throws TskCoreException { - DataSource dataSource = TskMockUtils.mockDataSource(1); + DataSource dataSource = TskMockUtils.getDataSource(1); List artifacts = Arrays.asList( - TskMockUtils.mockArtifact(artifactType, 2, dataSource, attrMaker.make(attributeType, "TEST SOURCE", val)), - TskMockUtils.mockArtifact(artifactType, 3, dataSource, attrMaker.make(attributeType, "TEST SOURCE", val)) + TskMockUtils.getArtifact(artifactType, 2, dataSource, attrMaker.make(attributeType, "TEST SOURCE", val)), + TskMockUtils.getArtifact(artifactType, 3, dataSource, attrMaker.make(attributeType, "TEST SOURCE", val)) ); test(artifactType, dataSource, @@ -299,7 +276,7 @@ public class GetArtifactsTest { } @Test - public void testFailOnJson() throws TskCoreException { + public void getArtifacts_failOnJson() throws TskCoreException { testFailOnBadAttrType( new BlackboardArtifact.Type(ARTIFACT_TYPE.TSK_GPS_ROUTE), new BlackboardAttribute.Type(ATTRIBUTE_TYPE.TSK_GEO_WAYPOINTS), @@ -308,7 +285,7 @@ public class GetArtifactsTest { } @Test - public void testFailOnBytes() throws TskCoreException { + public void getArtifacts_failOnBytes() throws TskCoreException { testFailOnBadAttrType( new BlackboardArtifact.Type(999, "BYTE_ARRAY_TYPE", "Byte Array Type"), new BlackboardAttribute.Type(999, "BYTE_ARR_ATTR_TYPE", "Byte Array Attribute Type", TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.BYTE), @@ -317,20 +294,20 @@ public class GetArtifactsTest { } @Test - public void testPurgeAttrNotPresent() throws TskCoreException { + public void getArtifacts_purgeAttrNotPresent() throws TskCoreException { long day = 24 * 60 * 60; - DataSource dataSource = TskMockUtils.mockDataSource(1); + DataSource dataSource = TskMockUtils.getDataSource(1); BlackboardArtifact.Type ART_TYPE = new BlackboardArtifact.Type(ARTIFACT_TYPE.TSK_PROG_RUN); - BlackboardArtifact mock1 = TskMockUtils.mockArtifact(ART_TYPE, 10, dataSource, + BlackboardArtifact mock1 = TskMockUtils.getArtifact(ART_TYPE, 10, dataSource, new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_COUNT, "TEST SOURCE", 5), new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME, "TEST SOURCE", day)); - BlackboardArtifact mock2 = TskMockUtils.mockArtifact(ART_TYPE, 20, dataSource, + BlackboardArtifact mock2 = TskMockUtils.getArtifact(ART_TYPE, 20, dataSource, new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_COUNT, "TEST SOURCE", 6)); - BlackboardArtifact mock3 = TskMockUtils.mockArtifact(ART_TYPE, 30, dataSource, + BlackboardArtifact mock3 = TskMockUtils.getArtifact(ART_TYPE, 30, dataSource, new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_COUNT, "TEST SOURCE", 7), new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME, "TEST SOURCE", 3 * day)); @@ -346,20 +323,20 @@ public class GetArtifactsTest { } @Test - public void testMultAttrsPresent() throws TskCoreException { + public void getArtifacts_multipleAttrsPresent() throws TskCoreException { long day = 24 * 60 * 60; - DataSource dataSource = TskMockUtils.mockDataSource(1); + DataSource dataSource = TskMockUtils.getDataSource(1); BlackboardArtifact.Type ART_TYPE = new BlackboardArtifact.Type(ARTIFACT_TYPE.TSK_PROG_RUN); - BlackboardArtifact mock1 = TskMockUtils.mockArtifact(ART_TYPE, 10, dataSource, + BlackboardArtifact mock1 = TskMockUtils.getArtifact(ART_TYPE, 10, dataSource, new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_COUNT, "TEST SOURCE", 7), new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME, "TEST SOURCE", day)); - BlackboardArtifact mock2 = TskMockUtils.mockArtifact(ART_TYPE, 20, dataSource, + BlackboardArtifact mock2 = TskMockUtils.getArtifact(ART_TYPE, 20, dataSource, new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_COUNT, "TEST SOURCE", 6)); - BlackboardArtifact mock3 = TskMockUtils.mockArtifact(ART_TYPE, 30, dataSource, + BlackboardArtifact mock3 = TskMockUtils.getArtifact(ART_TYPE, 30, dataSource, new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_COUNT, "TEST SOURCE", 5), new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME, "TEST SOURCE", 3 * day)); @@ -375,9 +352,9 @@ public class GetArtifactsTest { } @Test - public void testTskCoreExceptionThrown() throws TskCoreException { + public void getArtifacts_tskCoreExceptionThrown() throws TskCoreException { test(new BlackboardArtifact.Type(ARTIFACT_TYPE.TSK_ACCOUNT), - TskMockUtils.mockDataSource(1), + TskMockUtils.getDataSource(1), new BlackboardAttribute.Type(ATTRIBUTE_TYPE.TSK_ACCOUNT_TYPE), SortOrder.ASCENDING, 0, @@ -388,9 +365,9 @@ public class GetArtifactsTest { } @Test - public void testThrowOnLessThan0() throws TskCoreException { + public void getArtifacts_throwOnLessThan0() throws TskCoreException { test(new BlackboardArtifact.Type(ARTIFACT_TYPE.TSK_ACCOUNT), - TskMockUtils.mockDataSource(1), + TskMockUtils.getDataSource(1), new BlackboardAttribute.Type(ATTRIBUTE_TYPE.TSK_ACCOUNT_TYPE), SortOrder.ASCENDING, -1, @@ -401,9 +378,9 @@ public class GetArtifactsTest { } @Test - public void testEmptyListReturned() throws TskCoreException { + public void getArtifacts_emptyListReturned() throws TskCoreException { test(new BlackboardArtifact.Type(ARTIFACT_TYPE.TSK_ACCOUNT), - TskMockUtils.mockDataSource(1), + TskMockUtils.getDataSource(1), new BlackboardAttribute.Type(ATTRIBUTE_TYPE.TSK_ACCOUNT_TYPE), SortOrder.ASCENDING, 0, @@ -412,4 +389,76 @@ public class GetArtifactsTest { Collections.emptyList(), null); } + + /** + * Retrieves the value of an artifact. + */ + private interface GetAttrVal { + /** + * A method for retrieving the value of an artifact. + * @param artifact The artifact. + * @param type The type of attribute. + * @return The value. + */ + T getOrNull(BlackboardArtifact artifact, BlackboardAttribute.Type type); + } + + private void testNullAttrValue(String id, GetAttrVal getter, ARTIFACT_TYPE artifactType, + ATTRIBUTE_TYPE attributeType, T nonNullVal) + throws TskCoreException { + + BlackboardAttribute.Type attrType = new BlackboardAttribute.Type(attributeType); + BlackboardArtifact.Type artType = new BlackboardArtifact.Type(artifactType); + + BlackboardArtifact noAttribute = TskMockUtils.getArtifact(artType, 1000, + TskMockUtils.getDataSource(1), new ArrayList<>()); + + T nullValue = getter.getOrNull(noAttribute, attrType); + Assert.assertNull(String.format("Expected function %s to return null when no attribute present", id), nullValue); + + BlackboardArtifact hasAttribute = TskMockUtils.getArtifact(artType, 1000, + TskMockUtils.getDataSource(1), TskMockUtils.getAttribute(attributeType, nonNullVal)); + + T valueReceived = getter.getOrNull(hasAttribute, attrType); + + Assert.assertEquals(String.format("%s did not return the same value present in the attribute", id), nonNullVal, valueReceived); + } + + @Test + public void getStringOrNull_handlesNull() throws TskCoreException { + testNullAttrValue("getStringOrNull", DataSourceInfoUtilities::getStringOrNull, + ARTIFACT_TYPE.TSK_ACCOUNT, ATTRIBUTE_TYPE.TSK_ACCOUNT_TYPE, "Skype"); + } + + @Test + public void getIntOrNull_handlesNull() throws TskCoreException { + testNullAttrValue("getIntOrNull", DataSourceInfoUtilities::getIntOrNull, + ARTIFACT_TYPE.TSK_PROG_RUN, ATTRIBUTE_TYPE.TSK_COUNT, 16); + } + + @Test + public void getLongOrNull_handlesNull() throws TskCoreException { + testNullAttrValue("getLongOrNull", DataSourceInfoUtilities::getLongOrNull, + ARTIFACT_TYPE.TSK_ASSOCIATED_OBJECT, ATTRIBUTE_TYPE.TSK_ASSOCIATED_ARTIFACT, 1001L); + } + + @Test + public void getDateOrNull_handlesNull() throws TskCoreException { + BlackboardAttribute.Type attrType = new BlackboardAttribute.Type(ATTRIBUTE_TYPE.TSK_DATETIME); + BlackboardArtifact.Type artType = new BlackboardArtifact.Type(ARTIFACT_TYPE.TSK_BLUETOOTH_PAIRING); + + long dateTime = 24 * 60 * 60 * 42; + + BlackboardArtifact noAttribute = TskMockUtils.getArtifact(artType, 1000, + TskMockUtils.getDataSource(1), new ArrayList<>()); + + Date nullValue = DataSourceInfoUtilities.getDateOrNull(noAttribute, attrType); + Assert.assertNull(nullValue); + + BlackboardArtifact hasAttribute = TskMockUtils.getArtifact(artType, 1000, + TskMockUtils.getDataSource(1), TskMockUtils.getAttribute(ATTRIBUTE_TYPE.TSK_DATETIME, dateTime)); + + Date curVal = DataSourceInfoUtilities.getDateOrNull(hasAttribute, attrType); + Assert.assertEquals(dateTime, curVal.getTime() / 1000); + } } diff --git a/Core/test/unit/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/DataSourceSummaryMockUtils.java b/Core/test/unit/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/DataSourceSummaryMockUtils.java new file mode 100644 index 0000000000..d73c440e96 --- /dev/null +++ b/Core/test/unit/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/DataSourceSummaryMockUtils.java @@ -0,0 +1,57 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2020 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.datasourcesummary.datamodel; + +import java.util.List; +import org.apache.commons.lang3.tuple.Pair; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import org.sleuthkit.datamodel.Blackboard; +import org.sleuthkit.datamodel.BlackboardArtifact; +import org.sleuthkit.datamodel.SleuthkitCase; +import org.sleuthkit.datamodel.TskCoreException; + +/** + * Common tools for mocking in data source summary. + */ +public final class DataSourceSummaryMockUtils { + + /** + * Creates a pair of a mock SleuthkitCase and mock Blackboard. + * + * @param returnArr The return result when calling getArtifacts on the + * blackboard. + * + * @return The pair of a mock SleuthkitCase and mock Blackboard. + * + * @throws TskCoreException + */ + static Pair getArtifactsTSKMock(List returnArr) throws TskCoreException { + SleuthkitCase mockCase = mock(SleuthkitCase.class); + Blackboard mockBlackboard = mock(Blackboard.class); + when(mockCase.getBlackboard()).thenReturn(mockBlackboard); + when(mockBlackboard.getArtifacts(anyInt(), anyLong())).thenReturn(returnArr); + return Pair.of(mockCase, mockBlackboard); + } + + private DataSourceSummaryMockUtils() { + } +} diff --git a/Core/test/unit/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/RecentFilesSummaryTest.java b/Core/test/unit/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/RecentFilesSummaryTest.java new file mode 100644 index 0000000000..068f0c3904 --- /dev/null +++ b/Core/test/unit/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/RecentFilesSummaryTest.java @@ -0,0 +1,681 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2020 Basis Technology Corp. Contact: carrier sleuthkit + * org + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.sleuthkit.autopsy.datasourcesummary.datamodel; + +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import java.util.stream.Stream; +import org.apache.commons.lang3.tuple.Pair; +import org.junit.Assert; +import static org.junit.Assert.fail; +import org.junit.Test; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import org.sleuthkit.autopsy.datasourcesummary.datamodel.RecentFilesSummary.RecentAttachmentDetails; +import org.sleuthkit.autopsy.datasourcesummary.datamodel.RecentFilesSummary.RecentDownloadDetails; +import org.sleuthkit.autopsy.datasourcesummary.datamodel.RecentFilesSummary.RecentFileDetails; +import org.sleuthkit.autopsy.datasourcesummary.datamodel.SleuthkitCaseProvider.SleuthkitCaseProviderException; +import org.sleuthkit.autopsy.testutils.RandomizationUtils; +import org.sleuthkit.autopsy.testutils.TskMockUtils; +import org.sleuthkit.datamodel.Blackboard; +import org.sleuthkit.datamodel.BlackboardArtifact; +import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE; +import org.sleuthkit.datamodel.BlackboardAttribute; +import org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE; +import org.sleuthkit.datamodel.Content; +import org.sleuthkit.datamodel.DataSource; +import org.sleuthkit.datamodel.SleuthkitCase; +import org.sleuthkit.datamodel.TskCoreException; + +/** + * Tests for RecentFilesSummaryTest + */ +public class RecentFilesSummaryTest { + + /** + * An interface for calling methods in RecentFilesSummary in a uniform + * manner. + */ + private interface RecentFilesMethod { + + /** + * Means of acquiring data from a method in RecentFilesSummary. + * + * @param recentFilesSummary The RecentFilesSummary object. + * @param dataSource The datasource. + * @param count The number of items to retrieve. + * + * @return The method's return data. + * + * @throws SleuthkitCaseProviderException + * @throws TskCoreException + */ + List fetch(RecentFilesSummary recentFilesSummary, DataSource dataSource, int count) + throws SleuthkitCaseProviderException, TskCoreException; + } + + private static final RecentFilesMethod RECENT_DOCS_FUNCT + = (summary, dataSource, count) -> summary.getRecentlyOpenedDocuments(dataSource, count); + + private static final RecentFilesMethod RECENT_DOWNLOAD_FUNCT + = (summary, dataSource, count) -> summary.getRecentDownloads(dataSource, count); + + private static final RecentFilesMethod RECENT_ATTACHMENT_FUNCT + = (summary, dataSource, count) -> summary.getRecentAttachments(dataSource, count); + + /** + * If -1 count passed to method, should throw IllegalArgumentException. + * + * @param method The method to call. + * @param methodName The name of the metho + * + * @throws TskCoreException + * @throws SleuthkitCaseProviderException + */ + private void testNonPositiveCount_ThrowsError(RecentFilesMethod method, String methodName) + throws TskCoreException, SleuthkitCaseProviderException { + Pair casePair = DataSourceSummaryMockUtils.getArtifactsTSKMock(null); + DataSource dataSource = TskMockUtils.getDataSource(1); + RecentFilesSummary summary = new RecentFilesSummary(() -> casePair.getLeft()); + + try { + method.fetch(summary, dataSource, -1); + fail("Expected method " + methodName + " to fail on negative count."); + } catch (IllegalArgumentException ignored) { + verify(casePair.getRight(), + never().description("Expected negative count for " + methodName + " to not call any methods in SleuthkitCase.")) + .getArtifacts(anyInt(), anyLong()); + } + } + + @Test + public void getRecentlyOpenedDocuments_nonPositiveCount_ThrowsError() throws TskCoreException, SleuthkitCaseProviderException { + testNonPositiveCount_ThrowsError(RECENT_DOCS_FUNCT, "getRecentlyOpenedDocuments"); + } + + @Test + public void getRecentDownloads_nonPositiveCount_ThrowsError() throws TskCoreException, SleuthkitCaseProviderException { + testNonPositiveCount_ThrowsError(RECENT_DOWNLOAD_FUNCT, "getRecentDownloads"); + } + + @Test + public void getRecentAttachments_nonPositiveCount_ThrowsError() throws TskCoreException, SleuthkitCaseProviderException { + testNonPositiveCount_ThrowsError(RECENT_ATTACHMENT_FUNCT, "getRecentAttachments"); + } + + /** + * Tests that if no data source provided, an empty list is returned and + * SleuthkitCase isn't called. + * + * @param recentFilesMethod The method to call. + * @param methodName The name of the method + * + * @throws SleuthkitCaseProviderException + * @throws TskCoreException + */ + private void testNoDataSource_ReturnsEmptyList(RecentFilesMethod recentFilesMethod, String methodName) + throws SleuthkitCaseProviderException, TskCoreException { + + Pair casePair = DataSourceSummaryMockUtils.getArtifactsTSKMock(null); + RecentFilesSummary summary = new RecentFilesSummary(() -> casePair.getLeft()); + + List items = recentFilesMethod.fetch(summary, null, 10); + Assert.assertNotNull("Expected method " + methodName + " to return an empty list.", items); + Assert.assertEquals("Expected method " + methodName + " to return an empty list.", 0, items.size()); + verify(casePair.getRight(), + never().description("Expected null datasource for " + methodName + " to not call any methods in SleuthkitCase.")) + .getArtifacts(anyInt(), anyLong()); + } + + @Test + public void getRecentlyOpenedDocuments_noDataSource_ReturnsEmptyList() throws TskCoreException, SleuthkitCaseProviderException { + testNoDataSource_ReturnsEmptyList(RECENT_DOCS_FUNCT, "getRecentlyOpenedDocuments"); + } + + @Test + public void getRecentDownloads_noDataSource_ReturnsEmptyList() throws TskCoreException, SleuthkitCaseProviderException { + testNoDataSource_ReturnsEmptyList(RECENT_DOWNLOAD_FUNCT, "getRecentDownloads"); + } + + @Test + public void getRecentAttachments_noDataSource_ReturnsEmptyList() throws TskCoreException, SleuthkitCaseProviderException { + testNonPositiveCount_ThrowsError(RECENT_ATTACHMENT_FUNCT, "getRecentAttachments"); + } + + /** + * If SleuthkitCase returns no results, an empty list is returned. + * + * @param recentFilesMethod The method to call. + * @param methodName The name of the method. + * + * @throws SleuthkitCaseProviderException + * @throws TskCoreException + */ + private void testNoReturnedResults_ReturnsEmptyList(RecentFilesMethod recentFilesMethod, String methodName) + throws SleuthkitCaseProviderException, TskCoreException { + + Pair casePair = DataSourceSummaryMockUtils.getArtifactsTSKMock(Collections.emptyList()); + RecentFilesSummary summary = new RecentFilesSummary(() -> casePair.getLeft()); + DataSource dataSource = TskMockUtils.getDataSource(1); + List items = recentFilesMethod.fetch(summary, dataSource, 10); + Assert.assertNotNull("Expected method " + methodName + " to return an empty list.", items); + Assert.assertEquals("Expected method " + methodName + " to return an empty list.", 0, items.size()); + verify(casePair.getRight(), + times(1).description("Expected " + methodName + " to call Blackboard once.")) + .getArtifacts(anyInt(), anyLong()); + } + + @Test + public void getRecentlyOpenedDocuments_noReturnedResults_ReturnsEmptyList() throws TskCoreException, SleuthkitCaseProviderException { + testNoReturnedResults_ReturnsEmptyList(RECENT_DOCS_FUNCT, "getRecentlyOpenedDocuments"); + } + + @Test + public void getRecentDownloads_noReturnedResults_ReturnsEmptyList() throws TskCoreException, SleuthkitCaseProviderException { + testNoReturnedResults_ReturnsEmptyList(RECENT_DOWNLOAD_FUNCT, "getRecentDownloads"); + } + + @Test + public void getRecentAttachments_testNoDataSource_ReturnsEmptyList() throws TskCoreException, SleuthkitCaseProviderException { + testNoReturnedResults_ReturnsEmptyList(RECENT_ATTACHMENT_FUNCT, "getRecentAttachments"); + } + + private static final long DAY_SECONDS = 24 * 60 * 60; + + /** + * A means of creating a number representing seconds from epoch where the + * lower the idx, the more recent the time. + */ + private static final Function dateTimeRetriever = (idx) -> (365 - idx) * DAY_SECONDS + 1; + + /** + * Gets a mock BlackboardArtifact. + * + * @param ds The data source to which the artifact belongs. + * @param artifactId The artifact id. + * @param artType The artifact type. + * @param attributeArgs The mapping of attribute type to value for each + * attribute in the artifact. + * + * @return The mock artifact. + */ + private BlackboardArtifact getArtifact(DataSource ds, long artifactId, ARTIFACT_TYPE artType, List> attributeArgs) { + try { + List attributes = attributeArgs.stream() + .filter((arg) -> arg != null && arg.getLeft() != null && arg.getRight() != null) + .map((arg) -> { + return TskMockUtils.getAttribute(arg.getLeft(), arg.getRight()); + }) + .collect(Collectors.toList()); + + return TskMockUtils.getArtifact(new BlackboardArtifact.Type(artType), artifactId, ds, attributes); + } catch (TskCoreException ex) { + fail("There was an error mocking an artifact."); + return null; + } + } + + /** + * Returns a mock artifact for getRecentlyOpenedDocuments. + * + * @param ds The datasource for the artifact. + * @param artifactId The artifact id. + * @param dateTime The time in seconds from epoch. + * @param path The path for the document. + * + * @return The mock artifact with pertinent attributes. + */ + private BlackboardArtifact getRecentDocumentArtifact(DataSource ds, long artifactId, Long dateTime, String path) { + return getArtifact(ds, artifactId, ARTIFACT_TYPE.TSK_RECENT_OBJECT, Arrays.asList( + Pair.of(ATTRIBUTE_TYPE.TSK_DATETIME, dateTime), + Pair.of(ATTRIBUTE_TYPE.TSK_PATH, path) + )); + } + + @Test + public void getRecentlyOpenedDocuments_sortedByDateTimeAndLimited() throws SleuthkitCaseProviderException, TskCoreException { + Function pathRetriever = (idx) -> "/path/to/downloads/" + idx; + DataSource dataSource = TskMockUtils.getDataSource(1); + + int countRequest = 10; + for (int countToGenerate : new int[]{1, 9, 10, 11}) { + // generate artifacts for each artifact + List artifacts = new ArrayList<>(); + for (int idx = 0; idx < countToGenerate; idx++) { + BlackboardArtifact artifact = getRecentDocumentArtifact(dataSource, + 1000 + idx, dateTimeRetriever.apply(idx), pathRetriever.apply(idx)); + artifacts.add(artifact); + } + + // run through method + Pair casePair = DataSourceSummaryMockUtils.getArtifactsTSKMock(RandomizationUtils.getMixedUp(artifacts)); + RecentFilesSummary summary = new RecentFilesSummary(() -> casePair.getLeft()); + List results = summary.getRecentlyOpenedDocuments(dataSource, countRequest); + + // verify results + int expectedCount = Math.min(countRequest, countToGenerate); + Assert.assertNotNull(results); + Assert.assertEquals(expectedCount, results.size()); + for (int i = 0; i < expectedCount; i++) { + Assert.assertEquals(dateTimeRetriever.apply(i), results.get(i).getDateAsLong()); + Assert.assertEquals(pathRetriever.apply(i), results.get(i).getPath()); + } + } + } + + @Test + public void getRecentlyOpenedDocuments_filtersMissingData() throws SleuthkitCaseProviderException, TskCoreException { + DataSource dataSource = TskMockUtils.getDataSource(1); + + BlackboardArtifact successItem = getRecentDocumentArtifact(dataSource, 1001, DAY_SECONDS, "/a/path"); + BlackboardArtifact nullTime = getRecentDocumentArtifact(dataSource, 1002, null, "/a/path2"); + BlackboardArtifact zeroTime = getRecentDocumentArtifact(dataSource, 10021, 0L, "/a/path2a"); + List artifacts = Arrays.asList(nullTime, zeroTime, successItem); + + Pair casePair = DataSourceSummaryMockUtils.getArtifactsTSKMock(RandomizationUtils.getMixedUp(artifacts)); + RecentFilesSummary summary = new RecentFilesSummary(() -> casePair.getLeft()); + List results = summary.getRecentlyOpenedDocuments(dataSource, 10); + + // verify results (only successItem) + Assert.assertNotNull(results); + Assert.assertEquals(1, results.size()); + Assert.assertEquals((Long) DAY_SECONDS, results.get(0).getDateAsLong()); + Assert.assertTrue("/a/path".equalsIgnoreCase(results.get(0).getPath())); + } + + /** + * Creates a mock blackboard artifact for getRecentDownloads. + * + * @param ds The datasource. + * @param artifactId The artifact id. + * @param dateTime The time in seconds from epoch. + * @param domain The domain. + * @param path The path for the download. + * + * @return The mock artifact. + */ + private BlackboardArtifact getRecentDownloadArtifact(DataSource ds, long artifactId, Long dateTime, String domain, String path) { + return getArtifact(ds, artifactId, ARTIFACT_TYPE.TSK_WEB_DOWNLOAD, Arrays.asList( + Pair.of(ATTRIBUTE_TYPE.TSK_DATETIME_ACCESSED, dateTime), + Pair.of(ATTRIBUTE_TYPE.TSK_DOMAIN, domain), + Pair.of(ATTRIBUTE_TYPE.TSK_PATH, path) + )); + } + + @Test + public void getRecentDownloads_sortedByDateTimeAndLimited() throws SleuthkitCaseProviderException, TskCoreException { + Function domainRetriever = (idx) -> String.format("www.domain%d.com", idx); + Function pathRetriever = (idx) -> "/path/to/downloads/doc" + idx + ".pdf"; + + // run through method + DataSource dataSource = TskMockUtils.getDataSource(1); + + int countRequest = 10; + for (int countToGenerate : new int[]{1, 9, 10, 11}) { + // generate artifacts for each artifact + List artifacts = new ArrayList<>(); + for (int idx = 0; idx < countToGenerate; idx++) { + BlackboardArtifact artifact = getRecentDownloadArtifact(dataSource, + 1000 + idx, dateTimeRetriever.apply(idx), domainRetriever.apply(idx), + pathRetriever.apply(idx)); + + artifacts.add(artifact); + } + + // call method + Pair casePair = DataSourceSummaryMockUtils.getArtifactsTSKMock(RandomizationUtils.getMixedUp(artifacts)); + RecentFilesSummary summary = new RecentFilesSummary(() -> casePair.getLeft()); + List results = summary.getRecentDownloads(dataSource, countRequest); + + // verify results + int expectedCount = Math.min(countRequest, countToGenerate); + Assert.assertNotNull(results); + Assert.assertEquals(expectedCount, results.size()); + for (int i = 0; i < expectedCount; i++) { + Assert.assertEquals(dateTimeRetriever.apply(i), results.get(i).getDateAsLong()); + Assert.assertEquals(pathRetriever.apply(i), results.get(i).getPath()); + Assert.assertEquals(domainRetriever.apply(i), results.get(i).getWebDomain()); + } + } + } + + @Test + public void getRecentDownloads_filtersMissingData() throws SleuthkitCaseProviderException, TskCoreException { + DataSource dataSource = TskMockUtils.getDataSource(1); + + BlackboardArtifact successItem = getRecentDownloadArtifact(dataSource, 1001, DAY_SECONDS, "domain1.com", "/a/path1"); + BlackboardArtifact nullTime = getRecentDownloadArtifact(dataSource, 1002, null, "domain2.com", "/a/path2"); + BlackboardArtifact zeroTime = getRecentDownloadArtifact(dataSource, 10021, 0L, "domain2a.com", "/a/path2a"); + List artifacts = Arrays.asList(nullTime, zeroTime, successItem); + + Pair casePair = DataSourceSummaryMockUtils.getArtifactsTSKMock(RandomizationUtils.getMixedUp(artifacts)); + RecentFilesSummary summary = new RecentFilesSummary(() -> casePair.getLeft()); + + // call method + List results = summary.getRecentDownloads(dataSource, 10); + + // verify results + Assert.assertNotNull(results); + Assert.assertEquals(1, results.size()); + Assert.assertEquals((Long) DAY_SECONDS, results.get(0).getDateAsLong()); + Assert.assertTrue("/a/path1".equalsIgnoreCase(results.get(0).getPath())); + } + + /** + * getRecentAttachments method has special setup conditions. This class + * encapsulates all the SleuthkitCase/BlackboardArtifact setup for on + * possible return item. + */ + private class AttachmentArtifactItem { + + private final Integer messageArtifactTypeId; + private final boolean associatedAttrFormed; + private final String emailFrom; + private final Long messageTime; + private final boolean isParent; + private final String fileParentPath; + private final String fileName; + + /** + * Constructor with all parameters. + * + * @param messageArtifactTypeId The type id for the artifact or null if + * no message artifact to be created. + * @param emailFrom Who the message is from or null not to + * include attribute. + * @param messageTime Time in seconds from epoch or null not + * to include attribute. + * @param fileParentPath The parent AbstractFile's path value. + * @param fileName The parent AbstractFile's filename + * value. + * @param associatedAttrFormed If false, the TSK_ASSOCIATED_OBJECT + * artifact has no attribute (even though + * it is required). + * @param hasParent Whether or not the artifact has a parent + * AbstractFile. + */ + AttachmentArtifactItem(Integer messageArtifactTypeId, String emailFrom, Long messageTime, + String fileParentPath, String fileName, + boolean associatedAttrFormed, boolean hasParent) { + + this.messageArtifactTypeId = messageArtifactTypeId; + this.associatedAttrFormed = associatedAttrFormed; + this.emailFrom = emailFrom; + this.messageTime = messageTime; + this.isParent = hasParent; + this.fileParentPath = fileParentPath; + this.fileName = fileName; + } + + /** + * Convenience constructor where defaults of required attributes and + * SleuthkitCase assumed. + * + * @param messageArtifactTypeId The type id for the artifact or null if + * no message artifact to be created. + * @param emailFrom Who the message is from or null not to + * include attribute. + * @param messageTime Time in seconds from epoch or null not + * to include attribute. + * @param fileParentPath The parent AbstractFile's path value. + * @param fileName The parent AbstractFile's filename + * value. + */ + AttachmentArtifactItem(Integer messageArtifactTypeId, String emailFrom, Long messageTime, String fileParentPath, String fileName) { + this(messageArtifactTypeId, emailFrom, messageTime, fileParentPath, fileName, true, true); + } + + boolean isAssociatedAttrFormed() { + return associatedAttrFormed; + } + + String getEmailFrom() { + return emailFrom; + } + + Long getMessageTime() { + return messageTime; + } + + boolean hasParent() { + return isParent; + } + + String getFileParentPath() { + return fileParentPath; + } + + String getFileName() { + return fileName; + } + + Integer getMessageArtifactTypeId() { + return messageArtifactTypeId; + } + } + + /** + * Sets up the associated artifact message for the TSK_ASSOCIATED_OBJECT. + * + * @param artifacts The mapping of artifact id to artifact. + * @param item The record to setup. + * @param dataSource The datasource. + * @param associatedId The associated attribute id. + * @param artifactId The artifact id. + * + * @return The associated Artifact blackboard attribute. + * + * @throws TskCoreException + */ + private BlackboardAttribute setupAssociatedMessage(Map artifacts, AttachmentArtifactItem item, + DataSource dataSource, Long associatedId, Long artifactId) throws TskCoreException { + + BlackboardAttribute associatedAttr = TskMockUtils.getAttribute(ATTRIBUTE_TYPE.TSK_ASSOCIATED_ARTIFACT, associatedId); + + if (item.getMessageArtifactTypeId() == null) { + return associatedAttr; + } + + // find the artifact type or null if not found + ARTIFACT_TYPE messageType = Stream.of(ARTIFACT_TYPE.values()) + .filter((artType) -> artType.getTypeID() == item.getMessageArtifactTypeId()) + .findFirst() + .orElse(null); + + // if there is a message type, create the artifact + if (messageType != null) { + List attributes = new ArrayList<>(); + if (item.getEmailFrom() != null) { + attributes.add(TskMockUtils.getAttribute(ATTRIBUTE_TYPE.TSK_EMAIL_FROM, item.getEmailFrom())); + } + + if (item.getMessageTime() != null) { + attributes.add(TskMockUtils.getAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_SENT, item.getMessageTime())); + } + + artifacts.put(associatedId, TskMockUtils.getArtifact( + new BlackboardArtifact.Type(messageType), artifactId, dataSource, attributes)); + } + return associatedAttr; + } + + /** + * Since getRecentAttachments does not simply query one type of artifact and + * return results, this method sets up a mock SleuthkitCase and Blackboard + * to return pertinent data. + * + * @param items Each attachment item where each item could represent a + * return result if fully formed. + * + * @return The mock SleuthkitCase and Blackboard. + */ + private Pair getRecentAttachmentArtifactCase(List items) { + SleuthkitCase skCase = mock(SleuthkitCase.class); + Blackboard blackboard = mock(Blackboard.class); + when(skCase.getBlackboard()).thenReturn(blackboard); + + DataSource dataSource = TskMockUtils.getDataSource(1); + + long objIdCounter = 100; + Map artifacts = new HashMap<>(); + try { + for (AttachmentArtifactItem item : items) { + BlackboardAttribute associatedAttr = null; + // if the associated attribute is fully formed, + // create the associated attribute and related artifact + if (item.isAssociatedAttrFormed()) { + associatedAttr = setupAssociatedMessage(artifacts, item, dataSource, ++objIdCounter, ++objIdCounter); + } + + // create the content parent for the associated object if one should be present + Content parent = (item.hasParent()) + ? TskMockUtils.getAbstractFile(++objIdCounter, item.getFileParentPath(), item.getFileName()) + : null; + + Long associatedId = ++objIdCounter; + artifacts.put(associatedId, TskMockUtils.getArtifact( + new BlackboardArtifact.Type(ARTIFACT_TYPE.TSK_ASSOCIATED_OBJECT), + parent, associatedId, dataSource, associatedAttr)); + } + + // set up the blackboard to return artifacts that match the type id. + when(blackboard.getArtifacts(anyInt(), anyLong())).thenAnswer((inv) -> { + Object[] args = inv.getArguments(); + int artifactType = (Integer) args[0]; + return artifacts.values().stream() + .filter(art -> art.getArtifactTypeID() == artifactType) + .collect(Collectors.toList()); + }); + + // also set up the sleuthkitcase to return the artifact with the matching id or null. + when(skCase.getBlackboardArtifact(anyLong())).thenAnswer((inv2) -> { + Object[] args2 = inv2.getArguments(); + long id = (Long) args2[0]; + return artifacts.get(id); + }); + + return Pair.of(skCase, blackboard); + } catch (TskCoreException ex) { + fail("There was an error while creating SleuthkitCase for getRecentAttachments"); + return null; + } + } + + @Test + public void getRecentAttachments_sortedByDateTimeAndLimited() throws SleuthkitCaseProviderException, TskCoreException { + DataSource dataSource = TskMockUtils.getDataSource(1); + // a deterministic means of transforming an index into a particular attribute type so that they can be created + // and compared on return + Function emailFromRetriever = (idx) -> String.format("person%d@basistech.com", idx); + Function pathRetriever = (idx) -> "/path/to/attachment/" + idx; + Function fileNameRetriever = (idx) -> String.format("%d-filename.png", idx); + + int countRequest = 10; + for (int countToGenerate : new int[]{1, 9, 10, 11}) { + // set up the items in the sleuthkit case + List items = IntStream.range(0, countToGenerate) + .mapToObj((idx) -> new AttachmentArtifactItem(ARTIFACT_TYPE.TSK_MESSAGE.getTypeID(), + emailFromRetriever.apply(idx), dateTimeRetriever.apply(idx), + pathRetriever.apply(idx), fileNameRetriever.apply(idx))) + .collect(Collectors.toList()); + + List mixedUpItems = RandomizationUtils.getMixedUp(items); + Pair casePair = getRecentAttachmentArtifactCase(mixedUpItems); + RecentFilesSummary summary = new RecentFilesSummary(() -> casePair.getLeft()); + + // retrieve results + List results = summary.getRecentAttachments(dataSource, countRequest); + + // verify results + int expectedCount = Math.min(countRequest, countToGenerate); + Assert.assertNotNull(results); + Assert.assertEquals(expectedCount, results.size()); + + for (int i = 0; i < expectedCount; i++) { + RecentAttachmentDetails result = results.get(i); + Assert.assertEquals(dateTimeRetriever.apply(i), result.getDateAsLong()); + Assert.assertTrue(emailFromRetriever.apply(i).equalsIgnoreCase(result.getSender())); + Assert.assertTrue(Paths.get(pathRetriever.apply(i), fileNameRetriever.apply(i)).toString() + .equalsIgnoreCase(result.getPath())); + } + } + } + + @Test + public void getRecentAttachments_filterData() throws SleuthkitCaseProviderException, TskCoreException { + // setup data + DataSource dataSource = TskMockUtils.getDataSource(1); + + AttachmentArtifactItem successItem = new AttachmentArtifactItem(ARTIFACT_TYPE.TSK_EMAIL_MSG.getTypeID(), + "person@sleuthkit.com", DAY_SECONDS, "/parent/path", "msg.pdf"); + AttachmentArtifactItem successItem2 = new AttachmentArtifactItem(ARTIFACT_TYPE.TSK_MESSAGE.getTypeID(), + "person_on_skype", DAY_SECONDS + 1, "/parent/path/to/skype", "skype.png"); + AttachmentArtifactItem wrongArtType = new AttachmentArtifactItem(ARTIFACT_TYPE.TSK_CALLLOG.getTypeID(), + "5555675309", DAY_SECONDS + 2, "/path/to/callog/info", "callog.dat"); + AttachmentArtifactItem missingTimeStamp = new AttachmentArtifactItem(ARTIFACT_TYPE.TSK_EMAIL_MSG.getTypeID(), + "person2@sleuthkit.com", null, "/parent/path", "msg2.pdf"); + AttachmentArtifactItem zeroTimeStamp = new AttachmentArtifactItem(ARTIFACT_TYPE.TSK_EMAIL_MSG.getTypeID(), + "person2a@sleuthkit.com", 0L, "/parent/path", "msg2a.png"); + AttachmentArtifactItem noParentFile = new AttachmentArtifactItem(ARTIFACT_TYPE.TSK_EMAIL_MSG.getTypeID(), + "person4@sleuthkit.com", DAY_SECONDS + 4, "/parent/path", "msg4.jpg", true, false); + AttachmentArtifactItem noAssocAttr = new AttachmentArtifactItem(ARTIFACT_TYPE.TSK_EMAIL_MSG.getTypeID(), + "person3@sleuthkit.com", DAY_SECONDS + 5, "/parent/path", "msg5.gif", false, true); + AttachmentArtifactItem missingAssocArt = new AttachmentArtifactItem(null, + "person3@sleuthkit.com", DAY_SECONDS + 6, "/parent/path", "msg6.pdf"); + + List items = Arrays.asList(successItem, successItem2, + wrongArtType, missingTimeStamp, zeroTimeStamp, + noParentFile, noAssocAttr, missingAssocArt); + + Pair casePair = getRecentAttachmentArtifactCase(items); + RecentFilesSummary summary = new RecentFilesSummary(() -> casePair.getLeft()); + + // get data + List results = summary.getRecentAttachments(dataSource, 10); + + // verify results + Assert.assertNotNull(results); + Assert.assertEquals(2, results.size()); + RecentAttachmentDetails successItem2Details = results.get(0); + RecentAttachmentDetails successItemDetails = results.get(1); + + Assert.assertEquals(successItemDetails.getDateAsLong(), (Long) DAY_SECONDS); + Assert.assertTrue(Paths.get(successItem.getFileParentPath(), successItem.getFileName()) + .toString().equalsIgnoreCase(successItemDetails.getPath())); + Assert.assertTrue(successItem.getEmailFrom().equalsIgnoreCase(successItemDetails.getSender())); + + Assert.assertEquals(successItem2Details.getDateAsLong(), (Long) (DAY_SECONDS + 1)); + Assert.assertTrue(Paths.get(successItem2.getFileParentPath(), successItem2.getFileName()) + .toString().equalsIgnoreCase(successItem2Details.getPath())); + Assert.assertTrue(successItem2.getEmailFrom().equalsIgnoreCase(successItem2Details.getSender())); + } +} diff --git a/Core/test/unit/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/UserActivitySummaryTest.java b/Core/test/unit/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/UserActivitySummaryTest.java new file mode 100644 index 0000000000..0aff8a56b5 --- /dev/null +++ b/Core/test/unit/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/UserActivitySummaryTest.java @@ -0,0 +1,1243 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2020 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.datasourcesummary.datamodel; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.junit.Assert; +import static org.junit.Assert.fail; +import org.junit.Test; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.sleuthkit.autopsy.datasourcesummary.datamodel.DataSourceSummaryMockUtils.getArtifactsTSKMock; +import org.sleuthkit.autopsy.datasourcesummary.datamodel.SleuthkitCaseProvider.SleuthkitCaseProviderException; +import org.sleuthkit.autopsy.datasourcesummary.datamodel.UserActivitySummary.TopAccountResult; +import org.sleuthkit.autopsy.datasourcesummary.datamodel.UserActivitySummary.TopDeviceAttachedResult; +import org.sleuthkit.autopsy.datasourcesummary.datamodel.UserActivitySummary.TopDomainsResult; +import org.sleuthkit.autopsy.datasourcesummary.datamodel.UserActivitySummary.TopProgramsResult; +import org.sleuthkit.autopsy.datasourcesummary.datamodel.UserActivitySummary.TopWebSearchResult; +import org.sleuthkit.autopsy.testutils.TskMockUtils; +import org.sleuthkit.autopsy.texttranslation.NoServiceProviderException; +import org.sleuthkit.autopsy.texttranslation.TextTranslationService; +import org.sleuthkit.autopsy.texttranslation.TranslationException; +import org.sleuthkit.datamodel.Blackboard; +import org.sleuthkit.datamodel.BlackboardArtifact; +import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE; +import org.sleuthkit.datamodel.BlackboardAttribute; +import org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE; +import org.sleuthkit.datamodel.DataSource; +import org.sleuthkit.datamodel.SleuthkitCase; +import org.sleuthkit.datamodel.TskCoreException; + +/** + * Tests for UserActivitySummary. + */ +public class UserActivitySummaryTest { + /** + * Function to retrieve data from UserActivitySummary with the provided arguments. + */ + private interface DataFunction { + /** + * A UserActivitySummary method encapsulated in a uniform manner. + * @param userActivitySummary The UserActivitySummary class to use. + * @param datasource The data source. + * @param count The count. + * @return The list of objects to return. + * @throws SleuthkitCaseProviderException + * @throws TskCoreException + */ + List retrieve(UserActivitySummary userActivitySummary, DataSource datasource, int count) throws + SleuthkitCaseProviderException, TskCoreException; + } + + private static final DataFunction WEB_SEARCH_QUERY + = (userActivity, dataSource, count) -> userActivity.getMostRecentWebSearches(dataSource, count); + + private static final DataFunction ACCOUNT_QUERY + = (userActivity, dataSource, count) -> userActivity.getRecentAccounts(dataSource, count); + + private static final DataFunction DOMAINS_QUERY + = (userActivity, dataSource, count) -> userActivity.getRecentDomains(dataSource, count); + + private static final DataFunction DEVICE_QUERY + = (userActivity, dataSource, count) -> userActivity.getRecentDevices(dataSource, count); + + private static final DataFunction PROGRAMS_QUERY + = (userActivity, dataSource, count) -> userActivity.getTopPrograms(dataSource, count); + + private static final Map> USER_ACTIVITY_METHODS = new HashMap>() { + { + put("getMostRecentWebSearches", WEB_SEARCH_QUERY); + put("getRecentAccounts", ACCOUNT_QUERY); + put("getRecentDomains", DOMAINS_QUERY); + put("getRecentDevices", DEVICE_QUERY); + put("getTopPrograms", PROGRAMS_QUERY); + } + }; + + private static final long DAY_SECONDS = 24 * 60 * 60; + + private static void verifyCalled(Blackboard mockBlackboard, int artifactType, long datasourceId, String failureMessage) throws TskCoreException { + verify(mockBlackboard, times(1).description(failureMessage)).getArtifacts(artifactType, datasourceId); + } + + /** + * Gets a UserActivitySummary class to test. + * + * @param tskCase The SleuthkitCase. + * @param hasTranslation Whether the translation service is functional. + * @param translateFunction Function for translation. + * + * @return The UserActivitySummary class to use for testing. + * + * @throws NoServiceProviderException + * @throws TranslationException + */ + private static UserActivitySummary getTestClass(SleuthkitCase tskCase, boolean hasTranslation, Function translateFunction) + throws NoServiceProviderException, TranslationException { + + return new UserActivitySummary( + () -> tskCase, + TskMockUtils.getTextTranslationService(translateFunction, hasTranslation), + TskMockUtils.getJavaLogger("UNIT TEST LOGGER") + ); + } + + private void testMinCount(DataFunction funct, String id) + throws TskCoreException, NoServiceProviderException, TranslationException, SleuthkitCaseProviderException { + + for (int count : new int[]{0, -1}) { + Pair tskPair = getArtifactsTSKMock(null); + UserActivitySummary summary = getTestClass(tskPair.getLeft(), false, null); + + try { + funct.retrieve(summary, TskMockUtils.getDataSource(1), -1); + } catch (IllegalArgumentException ignored) { + // this exception is expected so continue if getArtifacts never called + verify(tskPair.getRight(), never().description( + String.format("Expected %s would not call getArtifacts for count %d", id, count))) + .getArtifacts(anyInt(), anyLong()); + + continue; + } + fail(String.format("Expected an Illegal argument exception to be thrown in method %s with count of %d", id, count)); + } + } + + /** + * Ensures that passing a non-positive count causes + * IllegalArgumentException. + * + * @throws TskCoreException + * @throws NoServiceProviderException + * @throws TranslationException + * @throws SleuthkitCaseProviderException + */ + @Test + public void testMinCountInvariant() + throws TskCoreException, NoServiceProviderException, TranslationException, SleuthkitCaseProviderException { + + for (Entry> query : USER_ACTIVITY_METHODS.entrySet()) { + testMinCount(query.getValue(), query.getKey()); + } + } + + private void testNullDataSource(DataFunction funct, String id) + throws TskCoreException, NoServiceProviderException, TranslationException, SleuthkitCaseProviderException { + + Pair tskPair = getArtifactsTSKMock(null); + UserActivitySummary summary = getTestClass(tskPair.getLeft(), false, null); + List retArr = funct.retrieve(summary, null, 10); + verify(tskPair.getRight(), never() + .description(String.format("Expected method %s to return empty list for null data source and not call SleuthkitCase", id))) + .getArtifacts(anyInt(), anyLong()); + + String errorMessage = String.format("Expected %s would return empty list for null data source", id); + Assert.assertTrue(errorMessage, retArr != null); + Assert.assertTrue(errorMessage, retArr.isEmpty()); + } + + /** + * If datasource is null, all methods return an empty list. + * + * @throws TskCoreException + * @throws NoServiceProviderException + * @throws TranslationException + * @throws SleuthkitCaseProviderException + */ + @Test + public void testNullDataSource() + throws TskCoreException, NoServiceProviderException, TranslationException, SleuthkitCaseProviderException { + + for (Entry> query : USER_ACTIVITY_METHODS.entrySet()) { + testNullDataSource(query.getValue(), query.getKey()); + } + } + + private void testNoResultsReturned(DataFunction funct, String id) + throws TskCoreException, NoServiceProviderException, TranslationException, SleuthkitCaseProviderException { + long dataSourceId = 1; + int count = 10; + Pair tskPair = getArtifactsTSKMock(new ArrayList<>()); + UserActivitySummary summary = getTestClass(tskPair.getLeft(), false, null); + List retArr = funct.retrieve(summary, TskMockUtils.getDataSource(dataSourceId), count); + + Assert.assertTrue(String.format("Expected non null empty list returned from %s", id), retArr != null); + Assert.assertTrue(String.format("Expected non null empty list returned from %s", id), retArr.isEmpty()); + } + + /** + * If no artifacts in SleuthkitCase, all data returning methods return an + * empty list. + * + * @throws TskCoreException + * @throws NoServiceProviderException + * @throws TranslationException + * @throws SleuthkitCaseProviderException + */ + @Test + public void testNoResultsReturned() + throws TskCoreException, NoServiceProviderException, TranslationException, SleuthkitCaseProviderException { + + for (Entry> query : USER_ACTIVITY_METHODS.entrySet()) { + testNoResultsReturned(query.getValue(), query.getKey()); + } + } + + private static List EXCLUDED_DEVICES = Arrays.asList("ROOT_HUB", "ROOT_HUB20"); + + private static BlackboardArtifact getRecentDeviceArtifact(long artifactId, DataSource dataSource, + String deviceId, String deviceMake, String deviceModel, Long date) { + + try { + return TskMockUtils.getArtifact(new BlackboardArtifact.Type(ARTIFACT_TYPE.TSK_DEVICE_ATTACHED), artifactId, dataSource, + TskMockUtils.getAttribute(ATTRIBUTE_TYPE.TSK_DEVICE_ID, deviceId), + TskMockUtils.getAttribute(ATTRIBUTE_TYPE.TSK_DATETIME, date), + TskMockUtils.getAttribute(ATTRIBUTE_TYPE.TSK_DEVICE_MAKE, deviceMake), + TskMockUtils.getAttribute(ATTRIBUTE_TYPE.TSK_DEVICE_MODEL, deviceModel) + ); + } catch (TskCoreException e) { + fail("Something went wrong while mocking"); + return null; + } + } + + /** + * Tests that UserActivitySummary.getRecentDevices removes things like + * ROOT_HUB. See EXCLUDED_DEVICES for excluded items. + * + * @throws TskCoreException + * @throws NoServiceProviderException + * @throws SleuthkitCaseProviderException + * @throws TranslationException + */ + @Test + public void getRecentDevices_appropriateFiltering() throws TskCoreException, NoServiceProviderException, + SleuthkitCaseProviderException, TranslationException { + + long dataSourceId = 1; + int count = 10; + long time = DAY_SECONDS * 42; + String acceptedDevice = "ACCEPTED DEVICE"; + + DataSource ds = TskMockUtils.getDataSource(dataSourceId); + + List allKeys = new ArrayList<>(EXCLUDED_DEVICES); + allKeys.add(acceptedDevice); + + List artifacts = IntStream.range(0, allKeys.size()) + .mapToObj((idx) -> { + String key = allKeys.get(idx); + return getRecentDeviceArtifact(1000L + idx, ds, "ID " + key, "MAKE " + key, key, time); + }) + .collect(Collectors.toList()); + + Pair tskPair = getArtifactsTSKMock(artifacts); + UserActivitySummary summary = getTestClass(tskPair.getLeft(), false, null); + + List results = summary.getRecentDevices(ds, count); + + verifyCalled(tskPair.getRight(), ARTIFACT_TYPE.TSK_DEVICE_ATTACHED.getTypeID(), dataSourceId, + "Expected getRecentDevices to call getArtifacts with correct arguments."); + Assert.assertEquals(1, results.size()); + Assert.assertEquals(acceptedDevice, results.get(0).getDeviceModel()); + Assert.assertEquals("MAKE " + acceptedDevice, results.get(0).getDeviceMake()); + Assert.assertEquals("ID " + acceptedDevice, results.get(0).getDeviceId()); + Assert.assertEquals(time, results.get(0).getDateAccessed().getTime() / 1000); + } + + /** + * Ensures that UserActivitySummary.getRecentDevices limits returned entries + * to count provided. + * + * @throws TskCoreException + * @throws NoServiceProviderException + * @throws SleuthkitCaseProviderException + * @throws TskCoreException + * @throws TranslationException + */ + @Test + public void getRecentDevices_limitedToCount() + throws TskCoreException, NoServiceProviderException, SleuthkitCaseProviderException, TskCoreException, TranslationException { + + int countRequested = 10; + for (int returnedCount : new int[]{1, 9, 10, 11}) { + long dataSourceId = 1L; + DataSource dataSource = TskMockUtils.getDataSource(dataSourceId); + + List returnedArtifacts = IntStream.range(0, returnedCount) + .mapToObj((idx) -> getRecentDeviceArtifact(1000 + idx, dataSource, "ID" + idx, "MAKE" + idx, "MODEL" + idx, DAY_SECONDS * idx)) + .collect(Collectors.toList()); + + Pair tskPair = getArtifactsTSKMock(returnedArtifacts); + UserActivitySummary summary = getTestClass(tskPair.getLeft(), false, null); + + List results = summary.getRecentDevices(dataSource, countRequested); + verifyCalled(tskPair.getRight(), ARTIFACT_TYPE.TSK_DEVICE_ATTACHED.getTypeID(), dataSourceId, + "Expected getRecentDevices to call getArtifacts with correct arguments."); + + Assert.assertEquals(Math.min(countRequested, returnedCount), results.size()); + } + } + + private static BlackboardArtifact getWebSearchArtifact(long artifactId, DataSource dataSource, String query, Long date) { + try { + return TskMockUtils.getArtifact(new BlackboardArtifact.Type(ARTIFACT_TYPE.TSK_WEB_SEARCH_QUERY), artifactId, dataSource, + TskMockUtils.getAttribute(ATTRIBUTE_TYPE.TSK_TEXT, query), + TskMockUtils.getAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_ACCESSED, date) + ); + } catch (TskCoreException e) { + fail("Something went wrong while mocking"); + return null; + } + } + + /** + * Ensures that UserActivitySummary.getMostRecentWebSearches groups + * artifacts appropriately (i.e. queries with the same name). + * + * @throws TskCoreException + * @throws NoServiceProviderException + * @throws TranslationException + * @throws SleuthkitCaseProviderException + */ + @Test + public void getMostRecentWebSearches_grouping() throws TskCoreException, NoServiceProviderException, TranslationException, SleuthkitCaseProviderException { + long dataSourceId = 1; + DataSource ds = TskMockUtils.getDataSource(dataSourceId); + + String query1 = "This is Query 1"; + String query2 = "This is Query 2"; + BlackboardArtifact art1a = getWebSearchArtifact(1001, ds, query1, DAY_SECONDS * 1); + BlackboardArtifact art2a = getWebSearchArtifact(1002, ds, query2, DAY_SECONDS * 2); + BlackboardArtifact art2b = getWebSearchArtifact(1003, ds, query2.toUpperCase(), DAY_SECONDS * 3); + BlackboardArtifact art1b = getWebSearchArtifact(1004, ds, query1.toUpperCase(), DAY_SECONDS * 4); + BlackboardArtifact art1c = getWebSearchArtifact(1005, ds, query1.toLowerCase(), DAY_SECONDS * 5); + + List artList = Arrays.asList(art1a, art2a, art2b, art1b, art1c); + + Pair tskPair = getArtifactsTSKMock(artList); + UserActivitySummary summary = getTestClass(tskPair.getLeft(), false, null); + List results = summary.getMostRecentWebSearches(ds, 10); + verifyCalled(tskPair.getRight(), ARTIFACT_TYPE.TSK_WEB_SEARCH_QUERY.getTypeID(), dataSourceId, + "Expected getRecentDevices to call getArtifacts with correct arguments."); + + Assert.assertEquals("Expected two different search queries", 2, results.size()); + Assert.assertTrue(query1.equalsIgnoreCase(results.get(0).getSearchString())); + Assert.assertEquals(DAY_SECONDS * 5, results.get(0).getDateAccessed().getTime() / 1000); + Assert.assertTrue(query2.equalsIgnoreCase(results.get(1).getSearchString())); + Assert.assertEquals(DAY_SECONDS * 3, results.get(1).getDateAccessed().getTime() / 1000); + } + + private void webSearchTranslationTest(List queries, boolean hasProvider, String translationSuffix) + throws SleuthkitCaseProviderException, TskCoreException, NoServiceProviderException, TranslationException { + + long dataSourceId = 1; + DataSource ds = TskMockUtils.getDataSource(dataSourceId); + + // create artifacts for each query where first query in the list will have most recent time. + List artList = IntStream.range(0, queries.size()) + .mapToObj((idx) -> getWebSearchArtifact(1000 + idx, ds, queries.get(idx), DAY_SECONDS * (queries.size() - idx))) + .collect(Collectors.toList()); + + Pair tskPair = getArtifactsTSKMock(artList); + + // return name with suffix if original exists and suffix is not null. + Function translator = (orig) -> { + if (orig == null || translationSuffix == null) { + return null; + } else { + return orig + translationSuffix; + } + }; + + // set up a mock TextTranslationService returning a translation + TextTranslationService translationService = TskMockUtils.getTextTranslationService(translator, hasProvider); + + UserActivitySummary summary = new UserActivitySummary( + () -> tskPair.getLeft(), + translationService, + TskMockUtils.getJavaLogger("UNIT TEST LOGGER") + ); + + List results = summary.getMostRecentWebSearches(ds, queries.size()); + + // verify translation service only called if hasProvider + if (hasProvider) { + verify(translationService, + times(queries.size()).description("Expected translation to be called for each query")) + .translate(anyString()); + } else { + verify(translationService, + never().description("Expected translation not to be called because no provider")) + .translate(anyString()); + } + + Assert.assertEquals(queries.size(), results.size()); + + // verify the translation if there should be one + for (int i = 0; i < queries.size(); i++) { + String query = queries.get(i); + TopWebSearchResult result = results.get(i); + + Assert.assertTrue(query.equalsIgnoreCase(result.getSearchString())); + if (hasProvider) { + if (StringUtils.isBlank(translationSuffix)) { + Assert.assertNull(result.getTranslatedResult()); + } else { + Assert.assertTrue((query + translationSuffix).equalsIgnoreCase(result.getTranslatedResult())); + } + } else { + Assert.assertNull(result.getTranslatedResult()); + } + } + } + + /** + * Verify that UserActivitySummary.getMostRecentWebSearches handles + * translation appropriately. + * + * @throws SleuthkitCaseProviderException + * @throws TskCoreException + * @throws NoServiceProviderException + * @throws TranslationException + */ + @Test + public void getMostRecentWebSearches_handlesTranslation() + throws SleuthkitCaseProviderException, TskCoreException, NoServiceProviderException, TranslationException { + + List queryList = Arrays.asList("query1", "query2", "query3"); + String translationSuffix = " [TRANSLATED]"; + // if no provider. + webSearchTranslationTest(queryList, false, translationSuffix); + + // if no translation. + webSearchTranslationTest(queryList, true, null); + + // if translation is the same (translation suffix doesn't change the trimmed string value) + webSearchTranslationTest(queryList, true, ""); + webSearchTranslationTest(queryList, true, " "); + + // if there is an actual translation + webSearchTranslationTest(queryList, true, translationSuffix); + } + + /** + * Ensure that UserActivitySummary.getMostRecentWebSearches results limited + * to count. + * + * @throws TskCoreException + * @throws NoServiceProviderException + * @throws SleuthkitCaseProviderException + * @throws TskCoreException + * @throws TranslationException + */ + @Test + public void getMostRecentWebSearches_limitedToCount() + throws TskCoreException, NoServiceProviderException, SleuthkitCaseProviderException, TskCoreException, TranslationException { + + int countRequested = 10; + for (int returnedCount : new int[]{1, 9, 10, 11}) { + long dataSourceId = 1L; + DataSource dataSource = TskMockUtils.getDataSource(dataSourceId); + + List returnedArtifacts = IntStream.range(0, returnedCount) + .mapToObj((idx) -> getWebSearchArtifact(1000 + idx, dataSource, "Query" + idx, DAY_SECONDS * idx + 1)) + .collect(Collectors.toList()); + + Pair tskPair = getArtifactsTSKMock(returnedArtifacts); + UserActivitySummary summary = getTestClass(tskPair.getLeft(), false, null); + + List results = summary.getMostRecentWebSearches(dataSource, countRequested); + verifyCalled(tskPair.getRight(), ARTIFACT_TYPE.TSK_WEB_SEARCH_QUERY.getTypeID(), dataSourceId, + "Expected getRecentDevices to call getArtifacts with correct arguments."); + + Assert.assertEquals(Math.min(countRequested, returnedCount), results.size()); + } + } + + private BlackboardArtifact getDomainsArtifact(DataSource dataSource, long id, String domain, Long time) { + List attributes = new ArrayList<>(); + if (domain != null) { + attributes.add(TskMockUtils.getAttribute(ATTRIBUTE_TYPE.TSK_DOMAIN, domain)); + } + + if (time != null) { + attributes.add(TskMockUtils.getAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_ACCESSED, time)); + } + + try { + return TskMockUtils.getArtifact( + new BlackboardArtifact.Type(ARTIFACT_TYPE.TSK_WEB_HISTORY), id, dataSource, + attributes); + } catch (TskCoreException e) { + fail("TskCoreException occurred while trying to mock a blackboard artifact"); + return null; + } + } + + private static final long DOMAIN_WINDOW_DAYS = 30; + + /** + * UserActivitySummary.getRecentDomains should return results within 30 days + * of the most recent access. + * + * @throws TskCoreException + * @throws SleuthkitCaseProviderException + * @throws NoServiceProviderException + * @throws TranslationException + */ + @Test + public void getRecentDomains_withinTimeWIndow() throws TskCoreException, SleuthkitCaseProviderException, NoServiceProviderException, TranslationException { + long dataSourceId = 1; + DataSource dataSource = TskMockUtils.getDataSource(dataSourceId); + String domain1 = "www.google.com"; + String domain2 = "www.basistech.com"; + String domain3 = "www.github.com"; + String domain4 = "www.stackoverflow.com"; + + BlackboardArtifact artifact1 = getDomainsArtifact(dataSource, 1000, domain1, DAY_SECONDS * DOMAIN_WINDOW_DAYS * 2); + BlackboardArtifact artifact1a = getDomainsArtifact(dataSource, 10001, domain1, DAY_SECONDS * DOMAIN_WINDOW_DAYS); + + BlackboardArtifact artifact2 = getDomainsArtifact(dataSource, 1001, domain2, DAY_SECONDS * DOMAIN_WINDOW_DAYS - 1); + + BlackboardArtifact artifact3 = getDomainsArtifact(dataSource, 1002, domain3, DAY_SECONDS * DOMAIN_WINDOW_DAYS); + BlackboardArtifact artifact3a = getDomainsArtifact(dataSource, 10021, domain3, 1L); + + BlackboardArtifact artifact4 = getDomainsArtifact(dataSource, 1003, domain4, 1L); + + List retArr = Arrays.asList(artifact1, artifact1a, artifact2, artifact3, artifact3a, artifact4); + + Pair tskPair = getArtifactsTSKMock(retArr); + + UserActivitySummary summary = getTestClass(tskPair.getLeft(), false, null); + + List domains = summary.getRecentDomains(dataSource, 10); + + verifyCalled(tskPair.getRight(), ARTIFACT_TYPE.TSK_WEB_HISTORY.getTypeID(), dataSourceId, + "Expected getRecentDomains to call getArtifacts with correct arguments."); + + Assert.assertEquals(2, domains.size()); + + Assert.assertTrue("Expected " + domain1 + " to be first domain", domain1.equalsIgnoreCase(domains.get(0).getDomain())); + Assert.assertEquals(DAY_SECONDS * DOMAIN_WINDOW_DAYS * 2, domains.get(0).getLastVisit().getTime() / 1000); + Assert.assertEquals((Long) 2L, domains.get(0).getVisitTimes()); + + Assert.assertTrue("Expected " + domain3 + " to be second domain", domain3.equalsIgnoreCase(domains.get(1).getDomain())); + Assert.assertEquals(DAY_SECONDS * DOMAIN_WINDOW_DAYS, domains.get(1).getLastVisit().getTime() / 1000); + Assert.assertEquals((Long) 1L, domains.get(1).getVisitTimes()); + } + + /** + * Ensure that items like localhost and 127.0.0.1 are removed from results. + * + * @throws TskCoreException + * @throws NoServiceProviderException + * @throws TranslationException + * @throws SleuthkitCaseProviderException + */ + @Test + public void getRecentDomains_appropriatelyFiltered() throws TskCoreException, NoServiceProviderException, TranslationException, SleuthkitCaseProviderException { + long dataSourceId = 1; + DataSource dataSource = TskMockUtils.getDataSource(dataSourceId); + String domain1 = "www.google.com"; + + // excluded + String domain2 = "localhost"; + String domain3 = "127.0.0.1"; + + BlackboardArtifact artifact1 = getDomainsArtifact(dataSource, 1000, domain1, DAY_SECONDS); + BlackboardArtifact artifact2 = getDomainsArtifact(dataSource, 1001, domain2, DAY_SECONDS * 2); + BlackboardArtifact artifact3 = getDomainsArtifact(dataSource, 1002, domain3, DAY_SECONDS * 3); + + List retArr = Arrays.asList(artifact1, artifact2, artifact3); + + Pair tskPair = getArtifactsTSKMock(retArr); + + UserActivitySummary summary = getTestClass(tskPair.getLeft(), false, null); + + List domains = summary.getRecentDomains(dataSource, 10); + + verifyCalled(tskPair.getRight(), ARTIFACT_TYPE.TSK_WEB_HISTORY.getTypeID(), dataSourceId, + "Expected getRecentDomains to call getArtifacts with correct arguments."); + + Assert.assertEquals(1, domains.size()); + + Assert.assertTrue("Expected " + domain1 + " to be most recent domain", domain1.equalsIgnoreCase(domains.get(0).getDomain())); + Assert.assertEquals(DAY_SECONDS, domains.get(0).getLastVisit().getTime() / 1000); + } + + /** + * Ensure domains are grouped by name. + * + * @throws TskCoreException + * @throws NoServiceProviderException + * @throws TranslationException + * @throws SleuthkitCaseProviderException + */ + @Test + public void getRecentDomains_groupedAppropriately() throws TskCoreException, NoServiceProviderException, TranslationException, SleuthkitCaseProviderException { + long dataSourceId = 1; + DataSource dataSource = TskMockUtils.getDataSource(dataSourceId); + String domain1 = "www.google.com"; + String domain2 = "www.basistech.com"; + + BlackboardArtifact artifact1 = getDomainsArtifact(dataSource, 1000, domain1, 1L); + BlackboardArtifact artifact1a = getDomainsArtifact(dataSource, 1001, domain1, 6L); + BlackboardArtifact artifact2 = getDomainsArtifact(dataSource, 1002, domain2, 2L); + BlackboardArtifact artifact2a = getDomainsArtifact(dataSource, 1003, domain2, 3L); + BlackboardArtifact artifact2b = getDomainsArtifact(dataSource, 1004, domain2, 4L); + + List retArr = Arrays.asList(artifact1, artifact1a, artifact2, artifact2a, artifact2b); + + Pair tskPair = getArtifactsTSKMock(retArr); + UserActivitySummary summary = getTestClass(tskPair.getLeft(), false, null); + + List domains = summary.getRecentDomains(dataSource, 10); + + verifyCalled(tskPair.getRight(), ARTIFACT_TYPE.TSK_WEB_HISTORY.getTypeID(), dataSourceId, + "Expected getRecentDomains to call getArtifacts with correct arguments."); + + Assert.assertEquals(2, domains.size()); + + Assert.assertTrue(domain1.equalsIgnoreCase(domains.get(1).getDomain())); + Assert.assertEquals(6L, domains.get(1).getLastVisit().getTime() / 1000); + Assert.assertEquals((Long) 2L, domains.get(1).getVisitTimes()); + + Assert.assertTrue(domain2.equalsIgnoreCase(domains.get(0).getDomain())); + Assert.assertEquals(4L, domains.get(0).getLastVisit().getTime() / 1000); + Assert.assertEquals((Long) 3L, domains.get(0).getVisitTimes()); + } + + /** + * Ensure that UserActivitySummary.getRecentDomains limits to count + * appropriately. + * + * @throws TskCoreException + * @throws NoServiceProviderException + * @throws TranslationException + * @throws SleuthkitCaseProviderException + */ + @Test + public void getRecentDomains_limitedAppropriately() + throws TskCoreException, NoServiceProviderException, TranslationException, SleuthkitCaseProviderException { + + int countRequested = 10; + for (int returnedCount : new int[]{1, 9, 10, 11}) { + long dataSourceId = 1L; + DataSource dataSource = TskMockUtils.getDataSource(dataSourceId); + + // create a list where there are 1 accesses for first, 2 for second, etc. + List returnedArtifacts = IntStream.range(0, returnedCount) + .mapToObj((idx) -> { + return IntStream.range(0, idx + 1) + .mapToObj((numIdx) -> { + int hash = 100 * idx + numIdx; + return getDomainsArtifact(dataSource, 1000 + hash, "Domain " + idx, 10L); + }); + }) + .flatMap((s) -> s) + .collect(Collectors.toList()); + + Pair tskPair = getArtifactsTSKMock(returnedArtifacts); + UserActivitySummary summary = getTestClass(tskPair.getLeft(), false, null); + + List results = summary.getRecentDomains(dataSource, countRequested); + verifyCalled(tskPair.getRight(), ARTIFACT_TYPE.TSK_WEB_HISTORY.getTypeID(), dataSourceId, + "Expected getRecentDevices to call getArtifacts with correct arguments."); + + Assert.assertEquals(Math.min(countRequested, returnedCount), results.size()); + } + } + + /** + * Get email artifact to be used with getRecentAccounts + * + * @param artifactId The artifact id. + * @param dataSource The datasource. + * @param dateRcvd The date received in seconds or null to exclude. + * @param dateSent The date sent in seconds or null to exclude. + * + * @return The mock artifact. + */ + private static BlackboardArtifact getEmailArtifact(long artifactId, DataSource dataSource, Long dateRcvd, Long dateSent) { + List attributes = new ArrayList<>(); + + if (dateRcvd != null) { + attributes.add(TskMockUtils.getAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_RCVD, dateRcvd)); + } + + if (dateSent != null) { + attributes.add(TskMockUtils.getAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_SENT, dateSent)); + } + + try { + return TskMockUtils.getArtifact(new BlackboardArtifact.Type(ARTIFACT_TYPE.TSK_EMAIL_MSG), + artifactId, dataSource, attributes); + } catch (TskCoreException ignored) { + fail("Something went wrong while mocking"); + return null; + } + } + + /** + * Get calllog artifact to be used with getRecentAccounts + * + * @param artifactId The artifact id. + * @param dataSource The datasource. + * @param dateStart The date start in seconds or null to exclude. + * @param dateEnd The date end in seconds or null to exclude. + * + * @return The mock artifact. + */ + private static BlackboardArtifact getCallogArtifact(long artifactId, DataSource dataSource, Long dateStart, Long dateEnd) { + List attributes = new ArrayList<>(); + + if (dateStart != null) { + attributes.add(TskMockUtils.getAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_START, dateStart)); + } + + if (dateEnd != null) { + attributes.add(TskMockUtils.getAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_END, dateEnd)); + } + + try { + return TskMockUtils.getArtifact(new BlackboardArtifact.Type(ARTIFACT_TYPE.TSK_CALLLOG), + artifactId, dataSource, attributes); + } catch (TskCoreException ignored) { + fail("Something went wrong while mocking"); + return null; + } + } + + /** + * Get message artifact to be used with getRecentAccounts + * + * @param artifactId The artifact id. + * @param dataSource The datasource. + * @param type The account type. + * @param dateSent The date of the message in seconds. + */ + private static BlackboardArtifact getMessageArtifact(long artifactId, DataSource dataSource, String type, Long dateTime) { + List attributes = new ArrayList<>(); + + if (type != null) { + attributes.add(TskMockUtils.getAttribute(ATTRIBUTE_TYPE.TSK_MESSAGE_TYPE, type)); + } + + if (dateTime != null) { + attributes.add(TskMockUtils.getAttribute(ATTRIBUTE_TYPE.TSK_DATETIME, dateTime)); + } + + try { + return TskMockUtils.getArtifact(new BlackboardArtifact.Type(ARTIFACT_TYPE.TSK_MESSAGE), + artifactId, dataSource, attributes); + } catch (TskCoreException ignored) { + fail("Something went wrong while mocking"); + return null; + } + } + + /** + * Performs a test on UserActivitySummary.getRecentAccounts. + * + * @param dataSource The datasource to use as parameter. + * @param count The count to use as a parameter. + * @param retArtifacts The artifacts to return from + * SleuthkitCase.getArtifacts. This method filters + * based on artifact type from the call. + * @param expectedResults The expected results. + * + * @throws TskCoreException + * @throws NoServiceProviderException + * @throws TranslationException + * @throws + * org.sleuthkit.autopsy.datasourcesummary.datamodel.SleuthkitCaseProvider.SleuthkitCaseProviderException + */ + private void getRecentAccountsTest(DataSource dataSource, int count, + List retArtifacts, List expectedResults) + throws TskCoreException, NoServiceProviderException, TranslationException, SleuthkitCaseProviderException { + + SleuthkitCase mockCase = mock(SleuthkitCase.class); + Blackboard mockBlackboard = mock(Blackboard.class); + when(mockCase.getBlackboard()).thenReturn(mockBlackboard); + + when(mockBlackboard.getArtifacts(anyInt(), anyLong())).thenAnswer((invocation) -> { + Object[] args = invocation.getArguments(); + int artifactType = (Integer) args[0]; + return retArtifacts.stream() + .filter((art) -> art.getArtifactTypeID() == artifactType) + .collect(Collectors.toList()); + }); + + UserActivitySummary summary = getTestClass(mockCase, false, null); + + List receivedResults = summary.getRecentAccounts(dataSource, count); + + verifyCalled(mockBlackboard, ARTIFACT_TYPE.TSK_MESSAGE.getTypeID(), dataSource.getId(), + "Expected getRecentAccounts to call getArtifacts requesting TSK_MESSAGE."); + + verifyCalled(mockBlackboard, ARTIFACT_TYPE.TSK_EMAIL_MSG.getTypeID(), dataSource.getId(), + "Expected getRecentAccounts to call getArtifacts requesting TSK_EMAIL_MSG."); + + verifyCalled(mockBlackboard, ARTIFACT_TYPE.TSK_CALLLOG.getTypeID(), dataSource.getId(), + "Expected getRecentAccounts to call getArtifacts requesting TSK_CALLLOG."); + + Assert.assertEquals(expectedResults.size(), receivedResults.size()); + for (int i = 0; i < expectedResults.size(); i++) { + TopAccountResult expectedItem = expectedResults.get(i); + TopAccountResult receivedItem = receivedResults.get(i); + + // since this may be somewhat variable + Assert.assertTrue(expectedItem.getAccountType().equalsIgnoreCase(receivedItem.getAccountType())); + Assert.assertEquals(expectedItem.getLastAccess().getTime(), receivedItem.getLastAccess().getTime()); + } + } + + private void getRecentAccountsOneArtTest(DataSource dataSource, BlackboardArtifact retArtifact, TopAccountResult expectedResult) + throws TskCoreException, NoServiceProviderException, TranslationException, SleuthkitCaseProviderException { + getRecentAccountsTest(dataSource, 10, Arrays.asList(retArtifact), expectedResult != null ? Arrays.asList(expectedResult) : Collections.emptyList()); + } + + /** + * Verify that UserActivitySummary.getRecentAccounts attempts to find a date + * but if none present, the artifact is excluded. + * + * @throws TskCoreException + * @throws NoServiceProviderException + * @throws TranslationException + * @throws SleuthkitCaseProviderException + */ + @Test + public void getRecentAccounts_filtersNoDate() + throws TskCoreException, NoServiceProviderException, TranslationException, SleuthkitCaseProviderException { + + DataSource ds1 = TskMockUtils.getDataSource(1); + BlackboardArtifact email1 = getEmailArtifact(31, ds1, DAY_SECONDS, null); + getRecentAccountsOneArtTest(ds1, email1, + new TopAccountResult( + Bundle.DataSourceUserActivitySummary_getRecentAccounts_emailMessage(), + new Date(DAY_SECONDS * 1000))); + + BlackboardArtifact email2 = getEmailArtifact(2, ds1, null, DAY_SECONDS); + getRecentAccountsOneArtTest(ds1, email2, + new TopAccountResult( + Bundle.DataSourceUserActivitySummary_getRecentAccounts_emailMessage(), + new Date(DAY_SECONDS * 1000))); + + BlackboardArtifact email3 = getEmailArtifact(3, ds1, null, null); + getRecentAccountsOneArtTest(ds1, email3, null); + + BlackboardArtifact email4 = getEmailArtifact(4, ds1, DAY_SECONDS, DAY_SECONDS * 2); + getRecentAccountsOneArtTest(ds1, email4, + new TopAccountResult( + Bundle.DataSourceUserActivitySummary_getRecentAccounts_emailMessage(), + new Date(DAY_SECONDS * 2 * 1000))); + + BlackboardArtifact callog1 = getCallogArtifact(11, ds1, DAY_SECONDS, null); + getRecentAccountsOneArtTest(ds1, callog1, + new TopAccountResult( + Bundle.DataSourceUserActivitySummary_getRecentAccounts_calllogMessage(), + new Date(DAY_SECONDS * 1000))); + + BlackboardArtifact callog2 = getCallogArtifact(12, ds1, null, DAY_SECONDS); + getRecentAccountsOneArtTest(ds1, callog2, + new TopAccountResult( + Bundle.DataSourceUserActivitySummary_getRecentAccounts_calllogMessage(), + new Date(DAY_SECONDS * 1000))); + + BlackboardArtifact callog3 = getCallogArtifact(13, ds1, null, null); + getRecentAccountsOneArtTest(ds1, callog3, null); + + BlackboardArtifact callog4 = getCallogArtifact(14, ds1, DAY_SECONDS, DAY_SECONDS * 2); + getRecentAccountsOneArtTest(ds1, callog4, + new TopAccountResult( + Bundle.DataSourceUserActivitySummary_getRecentAccounts_calllogMessage(), + new Date(DAY_SECONDS * 2 * 1000))); + + BlackboardArtifact message1 = getMessageArtifact(21, ds1, "Skype", null); + getRecentAccountsOneArtTest(ds1, message1, null); + + BlackboardArtifact message2 = getMessageArtifact(22, ds1, null, DAY_SECONDS); + getRecentAccountsOneArtTest(ds1, message2, null); + + BlackboardArtifact message3 = getMessageArtifact(23, ds1, null, null); + getRecentAccountsOneArtTest(ds1, message3, null); + + BlackboardArtifact message4 = getMessageArtifact(24, ds1, "Skype", DAY_SECONDS); + getRecentAccountsOneArtTest(ds1, message4, new TopAccountResult("Skype", new Date(DAY_SECONDS * 1000))); + + } + + /** + * Verifies that UserActivitySummary.getRecentAccounts groups appropriately + * by account type. + * + * @throws TskCoreException + * @throws NoServiceProviderException + * @throws TranslationException + * @throws SleuthkitCaseProviderException + */ + @Test + public void getRecentAccounts_rightGrouping() + throws TskCoreException, NoServiceProviderException, TranslationException, SleuthkitCaseProviderException { + DataSource ds1 = TskMockUtils.getDataSource(1); + BlackboardArtifact email1 = getEmailArtifact(11, ds1, DAY_SECONDS - 11, null); + BlackboardArtifact email2 = getEmailArtifact(12, ds1, DAY_SECONDS - 12, null); + BlackboardArtifact email3 = getEmailArtifact(13, ds1, DAY_SECONDS + 13, null); + + BlackboardArtifact callog1 = getCallogArtifact(21, ds1, DAY_SECONDS - 21, null); + BlackboardArtifact callog2 = getCallogArtifact(22, ds1, DAY_SECONDS + 22, null); + + BlackboardArtifact message1a = getMessageArtifact(31, ds1, "Skype", DAY_SECONDS - 31); + BlackboardArtifact message1b = getMessageArtifact(32, ds1, "Skype", DAY_SECONDS + 32); + + BlackboardArtifact message2a = getMessageArtifact(41, ds1, "Facebook", DAY_SECONDS - 41); + BlackboardArtifact message2b = getMessageArtifact(41, ds1, "Facebook", DAY_SECONDS + 42); + + getRecentAccountsTest(ds1, 10, + Arrays.asList(email1, email2, email3, callog1, callog2, message1a, message1b, message2a, message2b), + Arrays.asList( + new TopAccountResult("Facebook", new Date((DAY_SECONDS + 42) * 1000)), + new TopAccountResult("Skype", new Date((DAY_SECONDS + 32) * 1000)), + new TopAccountResult(Bundle.DataSourceUserActivitySummary_getRecentAccounts_calllogMessage(), new Date((DAY_SECONDS + 22) * 1000)), + new TopAccountResult(Bundle.DataSourceUserActivitySummary_getRecentAccounts_emailMessage(), new Date((DAY_SECONDS + 13) * 1000)) + )); + } + + /** + * Verifies that UserActivitySummary.getRecentAccounts properly limits + * results returned. + * + * @throws TskCoreException + * @throws NoServiceProviderException + * @throws TranslationException + * @throws SleuthkitCaseProviderException + */ + @Test + public void getRecentAccounts_rightLimit() + throws TskCoreException, NoServiceProviderException, TranslationException, SleuthkitCaseProviderException { + int countRequested = 10; + for (int returnedCount : new int[]{1, 9, 10, 11}) { + long dataSourceId = 1L; + DataSource dataSource = TskMockUtils.getDataSource(dataSourceId); + + List returnedArtifacts = IntStream.range(0, returnedCount) + .mapToObj((idx) -> getMessageArtifact(1000 + idx, dataSource, "Message Type " + idx, DAY_SECONDS * idx + 1)) + .collect(Collectors.toList()); + + Pair tskPair = getArtifactsTSKMock(returnedArtifacts); + UserActivitySummary summary = getTestClass(tskPair.getLeft(), false, null); + + List results = summary.getRecentAccounts(dataSource, countRequested); + verifyCalled(tskPair.getRight(), ARTIFACT_TYPE.TSK_MESSAGE.getTypeID(), dataSource.getId(), + "Expected getRecentAccounts to call getArtifacts requesting TSK_MESSAGE."); + + verifyCalled(tskPair.getRight(), ARTIFACT_TYPE.TSK_EMAIL_MSG.getTypeID(), dataSource.getId(), + "Expected getRecentAccounts to call getArtifacts requesting TSK_EMAIL_MSG."); + + verifyCalled(tskPair.getRight(), ARTIFACT_TYPE.TSK_CALLLOG.getTypeID(), dataSource.getId(), + "Expected getRecentAccounts to call getArtifacts requesting TSK_CALLLOG."); + + Assert.assertEquals(Math.min(countRequested, returnedCount), results.size()); + } + } + + /** + * Ensures that UserActivity.getShortFolderName handles paths appropriately + * including Program Files and AppData folders. + * + * @throws NoServiceProviderException + * @throws TskCoreException + * @throws TranslationException + */ + @Test + public void getShortFolderName_rightConversions() throws NoServiceProviderException, TskCoreException, TranslationException { + Map expected = new HashMap<>(); + expected.put("/Program Files/Item/Item.exe", "Item"); + expected.put("/Program Files (x86)/Item/Item.exe", "Item"); + expected.put("/Program_Files/Item/Item.exe", ""); + + expected.put("/User/test_user/item/AppData/Item/Item.exe", "AppData"); + expected.put("/User/test_user/item/Application Data/Item/Item.exe", "AppData"); + + expected.put("/Other Path/Item/Item.exe", ""); + + Pair tskPair = getArtifactsTSKMock(null); + UserActivitySummary summary = getTestClass(tskPair.getLeft(), false, null); + + for (Entry path : expected.entrySet()) { + Assert.assertTrue(path.getValue().equalsIgnoreCase(summary.getShortFolderName(path.getKey(), "Item.exe"))); + Assert.assertTrue(path.getValue().equalsIgnoreCase(summary.getShortFolderName(path.getKey().toUpperCase(), "Item.exe".toUpperCase()))); + Assert.assertTrue(path.getValue().equalsIgnoreCase(summary.getShortFolderName(path.getKey().toLowerCase(), "Item.exe".toLowerCase()))); + } + } + + private static BlackboardArtifact getProgramArtifact(long artifactId, DataSource dataSource, String programName, String path, Integer count, Long dateTime) { + List attributes = new ArrayList<>(); + + if (programName != null) { + attributes.add(TskMockUtils.getAttribute(ATTRIBUTE_TYPE.TSK_PROG_NAME, programName)); + } + + if (path != null) { + attributes.add(TskMockUtils.getAttribute(ATTRIBUTE_TYPE.TSK_PATH, path)); + } + + if (dateTime != null) { + attributes.add(TskMockUtils.getAttribute(ATTRIBUTE_TYPE.TSK_DATETIME, dateTime)); + } + + if (count != null) { + attributes.add(TskMockUtils.getAttribute(ATTRIBUTE_TYPE.TSK_COUNT, count)); + } + + try { + return TskMockUtils.getArtifact(new BlackboardArtifact.Type(ARTIFACT_TYPE.TSK_PROG_RUN), + artifactId, dataSource, attributes); + } catch (TskCoreException ignored) { + fail("Something went wrong while mocking"); + return null; + } + } + + /** + * Ensures that getTopPrograms filters results like ntosboot programs or + * /Windows folders. + * + * @throws TskCoreException + * @throws NoServiceProviderException + * @throws TranslationException + * @throws SleuthkitCaseProviderException + */ + @Test + public void getTopPrograms_filtered() + throws TskCoreException, NoServiceProviderException, TranslationException, SleuthkitCaseProviderException { + + DataSource ds1 = TskMockUtils.getDataSource(1); + BlackboardArtifact ntosToRemove = getProgramArtifact(1, ds1, "ntosboot", "/Program Files/etc/", 21, 21L); + BlackboardArtifact windowsToRemove = getProgramArtifact(2, ds1, "Program.exe", "/Windows/", 21, 21L); + BlackboardArtifact windowsToRemove2 = getProgramArtifact(3, ds1, "Program.exe", "/Windows/Nested/", 21, 21L); + BlackboardArtifact noProgramNameToRemove = getProgramArtifact(4, ds1, null, "/Program Files/", 21, 21L); + BlackboardArtifact noProgramNameToRemove2 = getProgramArtifact(5, ds1, " ", "/Program Files/", 21, 21L); + BlackboardArtifact successful = getProgramArtifact(6, ds1, "ProgramSuccess.exe", "/AppData/Success/", null, null); + BlackboardArtifact successful2 = getProgramArtifact(7, ds1, "ProgramSuccess2.exe", "/AppData/Success/", 22, 22L); + + Pair tskPair = getArtifactsTSKMock(Arrays.asList( + ntosToRemove, + windowsToRemove, + windowsToRemove2, + noProgramNameToRemove, + noProgramNameToRemove2, + successful, + successful2 + )); + UserActivitySummary summary = getTestClass(tskPair.getLeft(), false, null); + List results = summary.getTopPrograms(ds1, 10); + + Assert.assertEquals(2, results.size()); + Assert.assertTrue("ProgramSuccess2.exe".equalsIgnoreCase(results.get(0).getProgramName())); + Assert.assertTrue("ProgramSuccess.exe".equalsIgnoreCase(results.get(1).getProgramName())); + } + + /** + * Ensures proper grouping of programs with index of program name and path. + * + * @throws TskCoreException + * @throws NoServiceProviderException + * @throws TranslationException + * @throws SleuthkitCaseProviderException + */ + @Test + public void getTopPrograms_correctGrouping() + throws TskCoreException, NoServiceProviderException, TranslationException, SleuthkitCaseProviderException { + + DataSource ds1 = TskMockUtils.getDataSource(1); + BlackboardArtifact prog1 = getProgramArtifact(1, ds1, "program1.exe", "/Program Files/etc/", 21, 21L); + BlackboardArtifact prog1a = getProgramArtifact(1, ds1, "program1.exe", "/Program Files/etc/", 1, 31L); + BlackboardArtifact prog1b = getProgramArtifact(1, ds1, "program1.exe", "/Program Files/etc/", 2, 11L); + + BlackboardArtifact prog2 = getProgramArtifact(1, ds1, "program1.exe", "/Program Files/another/", 31, 21L); + BlackboardArtifact prog2a = getProgramArtifact(1, ds1, "program1.exe", "/Program Files/another/", 1, 31L); + BlackboardArtifact prog2b = getProgramArtifact(1, ds1, "program1.exe", "/Program Files/another/", 2, 11L); + + BlackboardArtifact prog3 = getProgramArtifact(1, ds1, "program2.exe", "/Program Files/another/", 10, 21L); + BlackboardArtifact prog3a = getProgramArtifact(1, ds1, "program2.exe", "/Program Files/another/", 1, 22L); + BlackboardArtifact prog3b = getProgramArtifact(1, ds1, "program2.exe", "/Program Files/another/", 2, 11L); + + Pair tskPair = getArtifactsTSKMock(Arrays.asList( + prog1, prog1a, prog1b, + prog2, prog2a, prog2b, + prog3, prog3a, prog3b + )); + UserActivitySummary summary = getTestClass(tskPair.getLeft(), false, null); + List results = summary.getTopPrograms(ds1, 10); + + Assert.assertEquals(3, results.size()); + Assert.assertTrue("program1.exe".equalsIgnoreCase(results.get(0).getProgramName())); + Assert.assertTrue("/Program Files/another/".equalsIgnoreCase(results.get(0).getProgramPath())); + Assert.assertEquals((Long) 31L, results.get(0).getRunTimes()); + Assert.assertEquals((Long) 31L, (Long) (results.get(0).getLastRun().getTime() / 1000)); + + Assert.assertTrue("program1.exe".equalsIgnoreCase(results.get(1).getProgramName())); + Assert.assertTrue("/Program Files/etc/".equalsIgnoreCase(results.get(1).getProgramPath())); + Assert.assertEquals((Long) 21L, results.get(1).getRunTimes()); + Assert.assertEquals((Long) 31L, (Long) (results.get(1).getLastRun().getTime() / 1000)); + + Assert.assertTrue("program2.exe".equalsIgnoreCase(results.get(2).getProgramName())); + Assert.assertTrue("/Program Files/another/".equalsIgnoreCase(results.get(2).getProgramPath())); + Assert.assertEquals((Long) 10L, results.get(2).getRunTimes()); + Assert.assertEquals((Long) 22L, (Long) (results.get(2).getLastRun().getTime() / 1000)); + } + + private void assertProgramOrder(DataSource ds1, List artifacts, List programNamesReturned) + throws TskCoreException, NoServiceProviderException, TranslationException, SleuthkitCaseProviderException { + + Pair tskPair = getArtifactsTSKMock(artifacts); + UserActivitySummary summary = getTestClass(tskPair.getLeft(), false, null); + List results = summary.getTopPrograms(ds1, 10); + + Assert.assertEquals(programNamesReturned.size(), results.size()); + for (int i = 0; i < programNamesReturned.size(); i++) { + Assert.assertTrue(programNamesReturned.get(i).equalsIgnoreCase(results.get(i).getProgramName())); + } + } + + /** + * Ensure that UserActivitySummary.getTopPrograms properly orders results + * (first by run count, then date, then program name). + * + * @throws TskCoreException + * @throws NoServiceProviderException + * @throws TranslationException + * @throws SleuthkitCaseProviderException + */ + @Test + public void getTopPrograms_correctOrdering() + throws TskCoreException, NoServiceProviderException, TranslationException, SleuthkitCaseProviderException { + + DataSource ds1 = TskMockUtils.getDataSource(1); + BlackboardArtifact sortByRunsCount1 = getProgramArtifact(1001, ds1, "Program1.exe", "/Program Files/Folder/", 8, 1L); + BlackboardArtifact sortByRunsCount2 = getProgramArtifact(1002, ds1, "Program2.exe", "/Program Files/Folder/", 9, 2L); + BlackboardArtifact sortByRunsCount3 = getProgramArtifact(1003, ds1, "Program3.exe", "/Program Files/Folder/", 10, 3L); + assertProgramOrder(ds1, Arrays.asList(sortByRunsCount1, sortByRunsCount2, sortByRunsCount3), Arrays.asList("Program3.exe", "Program2.exe", "Program1.exe")); + + BlackboardArtifact sortByRunDate1 = getProgramArtifact(1011, ds1, "Program1.exe", "/Program Files/Folder/", null, 1L); + BlackboardArtifact sortByRunDate2 = getProgramArtifact(1012, ds1, "Program2.exe", "/Program Files/Folder/", null, 3L); + BlackboardArtifact sortByRunDate3 = getProgramArtifact(1013, ds1, "Program3.exe", "/Program Files/Folder/", null, 2L); + assertProgramOrder(ds1, Arrays.asList(sortByRunDate1, sortByRunDate2, sortByRunDate3), Arrays.asList("Program2.exe", "Program3.exe", "Program1.exe")); + + BlackboardArtifact sortByProgName1 = getProgramArtifact(1021, ds1, "cProgram.exe", "/Program Files/Folder/", null, null); + BlackboardArtifact sortByProgName2 = getProgramArtifact(1022, ds1, "BProgram.exe", "/Program Files/Folder/", null, null); + BlackboardArtifact sortByProgName3 = getProgramArtifact(1023, ds1, "aProgram.exe", "/Program Files/Folder/", null, null); + assertProgramOrder(ds1, Arrays.asList(sortByProgName1, sortByProgName2, sortByProgName3), Arrays.asList("aProgram.exe", "BProgram.exe", "cProgram.exe")); + } + + /** + * Ensure that UserActivitySummary.getTopPrograms properly limits results + * (if no run count and no run date, then no limit). + * + * @throws TskCoreException + * @throws NoServiceProviderException + * @throws TranslationException + * @throws SleuthkitCaseProviderException + */ + @Test + public void getTopPrograms_limited() + throws TskCoreException, NoServiceProviderException, + TranslationException, SleuthkitCaseProviderException { + + int countRequested = 10; + for (int returnedCount : new int[]{1, 9, 10, 11}) { + long dataSourceId = 1L; + DataSource dataSource = TskMockUtils.getDataSource(dataSourceId); + + // if data is present for counts and dates, the results are limited + List returnedArtifacts = IntStream.range(0, returnedCount) + .mapToObj((idx) -> getProgramArtifact(1000 + idx, dataSource, "Program" + idx, + "/Program Files/Folder/", idx + 1, DAY_SECONDS * idx + 1)) + .collect(Collectors.toList()); + + Pair tskPair = getArtifactsTSKMock(returnedArtifacts); + UserActivitySummary summary = getTestClass(tskPair.getLeft(), false, null); + + List results = summary.getTopPrograms(dataSource, countRequested); + verifyCalled(tskPair.getRight(), ARTIFACT_TYPE.TSK_PROG_RUN.getTypeID(), dataSourceId, + "Expected getRecentDevices to call getArtifacts with correct arguments."); + + Assert.assertEquals(Math.min(countRequested, returnedCount), results.size()); + + // if that data is not present, it is not limited + List returnedArtifactsAlphabetical = IntStream.range(0, returnedCount) + .mapToObj((idx) -> getProgramArtifact(1000 + idx, dataSource, "Program" + idx, null, null, null)) + .collect(Collectors.toList()); + + Pair tskPairAlphabetical = getArtifactsTSKMock(returnedArtifactsAlphabetical); + UserActivitySummary summaryAlphabetical = getTestClass(tskPairAlphabetical.getLeft(), false, null); + + List resultsAlphabetical = summaryAlphabetical.getTopPrograms(dataSource, countRequested); + verifyCalled(tskPairAlphabetical.getRight(), ARTIFACT_TYPE.TSK_PROG_RUN.getTypeID(), dataSourceId, + "Expected getRecentDevices to call getArtifacts with correct arguments."); + + // ensure alphabetical by name + for (int i = 0; i < resultsAlphabetical.size() - 1; i++) { + Assert.assertTrue(resultsAlphabetical.get(i).getProgramName().compareToIgnoreCase(resultsAlphabetical.get(i + 1).getProgramName()) < 0); + } + + Assert.assertEquals(returnedArtifacts.size(), resultsAlphabetical.size()); + } + } +} diff --git a/Core/test/unit/src/org/sleuthkit/autopsy/discovery/search/DomainSearchTestUtils.java b/Core/test/unit/src/org/sleuthkit/autopsy/discovery/search/DomainSearchTestUtils.java index ffadb3a16b..9596ae432f 100755 --- a/Core/test/unit/src/org/sleuthkit/autopsy/discovery/search/DomainSearchTestUtils.java +++ b/Core/test/unit/src/org/sleuthkit/autopsy/discovery/search/DomainSearchTestUtils.java @@ -32,7 +32,7 @@ public class DomainSearchTestUtils { public static ResultDomain mockDomainResult(String domain, long start, long end, long totalVisits, long visits, long filesDownloaded, long dataSourceId) { - Content dataSource = TskMockUtils.mockDataSource(dataSourceId); + Content dataSource = TskMockUtils.getDataSource(dataSourceId); return new ResultDomain(domain, start, end, totalVisits, visits, filesDownloaded, dataSource); } diff --git a/Core/test/unit/src/org/sleuthkit/autopsy/testutils/RandomizationUtils.java b/Core/test/unit/src/org/sleuthkit/autopsy/testutils/RandomizationUtils.java new file mode 100644 index 0000000000..848830284b --- /dev/null +++ b/Core/test/unit/src/org/sleuthkit/autopsy/testutils/RandomizationUtils.java @@ -0,0 +1,56 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2020 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.testutils; + +import java.util.ArrayList; +import java.util.List; + +/** + * Tools for pseudo-randomization. + */ +public final class RandomizationUtils { + + /** + * Returns list in 0, n-1, 1, n-2 ... order. Deterministic so same results + * each time, but not in original order. + * + * @return Mixed up list. + */ + public static List getMixedUp(List list) { + int forward = 0; + int backward = list.size() - 1; + + List newList = new ArrayList<>(); + while (forward <= backward) { + newList.add(list.get(forward)); + + if (forward < backward) { + newList.add(list.get(backward)); + } + + forward++; + backward--; + } + + return newList; + } + + private RandomizationUtils() { + } +} diff --git a/Core/test/unit/src/org/sleuthkit/autopsy/testutils/TskMockUtils.java b/Core/test/unit/src/org/sleuthkit/autopsy/testutils/TskMockUtils.java index b47ecdece3..b6a832acb7 100644 --- a/Core/test/unit/src/org/sleuthkit/autopsy/testutils/TskMockUtils.java +++ b/Core/test/unit/src/org/sleuthkit/autopsy/testutils/TskMockUtils.java @@ -19,15 +19,26 @@ package org.sleuthkit.autopsy.testutils; import java.util.ArrayList; +import java.util.List; import java.util.Map; import java.util.function.Function; +import java.util.logging.ConsoleHandler; +import java.util.logging.Handler; +import java.util.logging.Logger; import java.util.stream.Collectors; import java.util.stream.Stream; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import org.sleuthkit.autopsy.texttranslation.NoServiceProviderException; +import org.sleuthkit.autopsy.texttranslation.TextTranslationService; +import org.sleuthkit.autopsy.texttranslation.TranslationException; +import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardAttribute; +import org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE; +import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.DataSource; import org.sleuthkit.datamodel.TskCoreException; @@ -44,7 +55,7 @@ public class TskMockUtils { * * @return The mocked datasource. */ - public static DataSource mockDataSource(long dataSourceId) { + public static DataSource getDataSource(long dataSourceId) { DataSource dataSource = mock(DataSource.class); when(dataSource.getName()).thenReturn(""); when(dataSource.getId()).thenReturn(dataSourceId); @@ -65,14 +76,35 @@ public class TskMockUtils { * * @throws TskCoreException */ - public static BlackboardArtifact mockArtifact(BlackboardArtifact.Type artifactType, long artifactId, + public static BlackboardArtifact getArtifact(BlackboardArtifact.Type artifactType, long artifactId, + DataSource dataSource, BlackboardAttribute... attributes) throws TskCoreException { + return getArtifact(artifactType, null, artifactId, dataSource, attributes); + } + + /** + * Gets a mock Blackboard artifact. + * + * @param artifactType The artifact type for the artifact. + * @param parent The parent file of the artifact. + * @param artifactId The artifact id. + * @param dataSource The datasource. + * @param attributes The attributes for the artifact. + * + * @return The mocked artifact. + * + * @throws TskCoreException + */ + public static BlackboardArtifact getArtifact(BlackboardArtifact.Type artifactType, Content parent, long artifactId, DataSource dataSource, BlackboardAttribute... attributes) throws TskCoreException { BlackboardArtifact artifact = mock(BlackboardArtifact.class); final Map attributeTypes = Stream.of(attributes) + .filter(attr -> attr != null) .collect(Collectors.toMap((attr) -> attr.getAttributeType(), Function.identity())); + when(artifact.getParent()).thenReturn(parent); + when(artifact.getArtifactID()).thenReturn(artifactId); when(artifact.getArtifactTypeID()).thenReturn(artifactType.getTypeID()); @@ -89,6 +121,146 @@ public class TskMockUtils { return artifact; } + public static BlackboardArtifact getArtifact(BlackboardArtifact.Type artifactType, long artifactId, + DataSource dataSource, List attributes) throws TskCoreException { + + return getArtifact(artifactType, artifactId, dataSource, attributes.toArray(new BlackboardAttribute[0])); + } + + private static final String DEFAULT_ATTR_SOURCE = "TEST SOURCE"; + + public static BlackboardAttribute getAttribute(ATTRIBUTE_TYPE attrType, Object value) { + + return getAttribute(new BlackboardAttribute.Type(attrType), DEFAULT_ATTR_SOURCE, value); + } + + public static BlackboardAttribute getAttribute(BlackboardAttribute.Type attrType, String source, Object value) { + switch (attrType.getValueType()) { + case STRING: + case JSON: + if (value instanceof String) { + return new BlackboardAttribute(attrType, source, (String) value); + } + break; + case DATETIME: + case LONG: + if (value instanceof Long) { + return new BlackboardAttribute(attrType, source, (Long) value); + } + break; + case INTEGER: + if (value instanceof Integer) { + return new BlackboardAttribute(attrType, source, (Integer) value); + } + break; + case DOUBLE: + if (value instanceof Double) { + return new BlackboardAttribute(attrType, source, (Double) value); + } + break; + case BYTE: + if (value instanceof byte[]) { + return new BlackboardAttribute(attrType, source, (byte[]) value); + } + break; + default: + throw new IllegalArgumentException(String.format("Unknown attribute value type: %s", attrType.getValueType())); + } + + throw new IllegalArgumentException(String.format("Attribute type expected type of %s but received argument of %s", attrType.getValueType(), value)); + } + + /** + * Returns a mock TextTranslationService. + * + * @param onTranslate A function that performs the translation. If null, a + * null result is always returned for .translate method. + * @param hasProvider What to return for the hasProvider method. + * + * @return The mocked text translation service. + * + * @throws NoServiceProviderException + * @throws TranslationException + */ + public static TextTranslationService getTextTranslationService(Function onTranslate, boolean hasProvider) + throws NoServiceProviderException, TranslationException { + TextTranslationService translationService = mock(TextTranslationService.class); + when(translationService.hasProvider()).thenReturn(hasProvider); + + when(translationService.translate(anyString())).thenAnswer((invocation) -> { + if (onTranslate == null) { + throw new NoServiceProviderException("No onTranslate function provided"); + } + + Object[] args = invocation.getArguments(); + String input = (String) args[0]; + return (input == null) ? null : onTranslate.apply(input); + }); + + return translationService; + } + + /** + * Returns an AbstractFile mocking getPath and getName. + * + * @param objId The object id. + * @param path The path for the file. + * @param name The name + * + * @return + */ + public static AbstractFile getAbstractFile(long objId, String path, String name) { + AbstractFile mocked = mock(AbstractFile.class); + when(mocked.getId()).thenReturn(objId); + when(mocked.getName()).thenReturn(name); + when(mocked.getParentPath()).thenReturn(path); + return mocked; + } + + private static void setConsoleHandler(Logger logger) { + // taken from https://stackoverflow.com/a/981230 + // Handler for console (reuse it if it already exists) + Handler consoleHandler = null; + + //see if there is already a console handler + for (Handler handler : logger.getHandlers()) { + if (handler instanceof ConsoleHandler) { + //found the console handler + consoleHandler = handler; + break; + } + } + + if (consoleHandler == null) { + //there was no console handler found, create a new one + consoleHandler = new ConsoleHandler(); + logger.addHandler(consoleHandler); + } + + //set the console handler to fine: + consoleHandler.setLevel(java.util.logging.Level.FINEST); + } + + /** + * Retrieves an autopsy logger that does not write to disk. + * + * @param loggerName The name of the logger. + * + * @return The autopsy logger for the console + * + * @throws InstantiationException + * @throws IllegalStateException + */ + public static Logger getJavaLogger(String loggerName) { + // The logger doesn't appear to respond well to mocking with mockito. + // It appears that the issue may have to do with mocking methods in the java.* packages + // since the autopsy logger extends the java.util.logging.Logger class: + // https://javadoc.io/static/org.mockito/mockito-core/3.5.13/org/mockito/Mockito.html#39 + Logger logger = Logger.getLogger(loggerName); + setConsoleHandler(logger); + return logger; + } + private TskMockUtils() { } } diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java index a6ece45667..b3a25be677 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java @@ -2930,7 +2930,9 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen String eventType = event.getPropertyName(); if (eventType.equals(IngestManager.IngestJobEvent.COMPLETED.toString()) || eventType.equals(IngestManager.IngestJobEvent.CANCELLED.toString())) { synchronized (ingestLock) { - ingestLock.notify(); + if (! IngestManager.getInstance().isIngestRunning()) { + ingestLock.notify(); + } } } } diff --git a/thirdparty/yara/ReadMe.txt b/thirdparty/yara/ReadMe.txt new file mode 100755 index 0000000000..31f38633b4 --- /dev/null +++ b/thirdparty/yara/ReadMe.txt @@ -0,0 +1,39 @@ +This folder contains the projects you need for building and testing the yarabridge.dll and YaraJNIWrapper.jar. + +bin: +Contains the built dll and jar. + +yarabridge: +VS project to create the dll that wraps the the libyara library. + +YaraJNIWrapper: +Simple jar file that contains the native JNI methods for accessing the yarabridge.dll. + + +Steps for building yarabridge, YaraJNIWrapper and YaraWrapperTest. + +1. Clone the yara repo at the same level as you have the autopsy repo. https://github.com/VirusTotal/yara +2. Build libyara: + - Open the project yara/windows/2015/yara.sln + - Build Release x64. +3. Open the yarabridge project and build Release x64. + -If you have link issues, make sure you build release x64 in the previous step. + -This project will automatically copy the built dll to the bin folder. +4. Build YaraJNIWrapper + - Open in netbeans and select Build. + - Manually move the newly build jar file to the bin folder. After building the jar file can be found in + yara/YaraJNIWrapper/dist/ + - Any matching rules will appear on the CL or the output of the project. +5. Test + - Open the YaraWrapperTest + - In the Run Properties you need to specify the path to a compiled yara rule file and a file to search. + There are sample files in YaraWrapperTest\resources. + - If you would like to make your own compiled rule file you can use the yarac tool that can be found + in yara/windows/vs2015/Release, if its not there go back to the yara project and build all of the + projects. + +Troubleshooting: +- When building libyara make sure that you are building the vs2015 project (There is a vs2017 project too). + The paths in the yarabrige package are relative, but assume + that you are building the release version of the dll with the vs2015 project. +- Don't forget to move the YaraJNIWrapper.jar after you build it. diff --git a/thirdparty/yara/YaraJNIWrapper/build.xml b/thirdparty/yara/YaraJNIWrapper/build.xml new file mode 100755 index 0000000000..b1000f923d --- /dev/null +++ b/thirdparty/yara/YaraJNIWrapper/build.xml @@ -0,0 +1,73 @@ + + + + + + + + + + + Builds, tests, and runs the project YaraJNIWrapper. + + + diff --git a/thirdparty/yara/YaraJNIWrapper/nbproject/build-impl.xml b/thirdparty/yara/YaraJNIWrapper/nbproject/build-impl.xml new file mode 100755 index 0000000000..38dd8d0c87 --- /dev/null +++ b/thirdparty/yara/YaraJNIWrapper/nbproject/build-impl.xml @@ -0,0 +1,1770 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Must set src.dir + Must set test.src.dir + Must set build.dir + Must set dist.dir + Must set build.classes.dir + Must set dist.javadoc.dir + Must set build.test.classes.dir + Must set build.test.results.dir + Must set build.classes.excludes + Must set dist.jar + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Must set javac.includes + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + No tests executed. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Must set JVM to use for profiling in profiler.info.jvm + Must set profiler agent JVM arguments in profiler.info.jvmargs.agent + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Must select some files in the IDE or set javac.includes + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + To run this application from the command line without Ant, try: + + java -jar "${dist.jar.resolved}" + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Must select one file in the IDE or set run.class + + + + Must select one file in the IDE or set run.class + + + + + + + + + + + + + + + + + + + + + + + Must select one file in the IDE or set debug.class + + + + + Must select one file in the IDE or set debug.class + + + + + Must set fix.includes + + + + + + + + + + This target only works when run from inside the NetBeans IDE. + + + + + + + + + Must select one file in the IDE or set profile.class + This target only works when run from inside the NetBeans IDE. + + + + + + + + + This target only works when run from inside the NetBeans IDE. + + + + + + + + + + + + + This target only works when run from inside the NetBeans IDE. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Must select one file in the IDE or set run.class + + + + + + Must select some files in the IDE or set test.includes + + + + + Must select one file in the IDE or set run.class + + + + + Must select one file in the IDE or set applet.url + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Must select some files in the IDE or set javac.includes + + + + + + + + + + + + + + + + + + + + + + + + Some tests failed; see details above. + + + + + + + + + Must select some files in the IDE or set test.includes + + + + Some tests failed; see details above. + + + + Must select some files in the IDE or set test.class + Must select some method in the IDE or set test.method + + + + Some tests failed; see details above. + + + + + Must select one file in the IDE or set test.class + + + + Must select one file in the IDE or set test.class + Must select some method in the IDE or set test.method + + + + + + + + + + + + + + Must select one file in the IDE or set applet.url + + + + + + + + + Must select one file in the IDE or set applet.url + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/thirdparty/yara/YaraJNIWrapper/nbproject/private/private.xml b/thirdparty/yara/YaraJNIWrapper/nbproject/private/private.xml new file mode 100755 index 0000000000..475096252c --- /dev/null +++ b/thirdparty/yara/YaraJNIWrapper/nbproject/private/private.xml @@ -0,0 +1,4 @@ + + + + diff --git a/thirdparty/yara/YaraJNIWrapper/nbproject/project.properties b/thirdparty/yara/YaraJNIWrapper/nbproject/project.properties new file mode 100755 index 0000000000..a0ef4dac37 --- /dev/null +++ b/thirdparty/yara/YaraJNIWrapper/nbproject/project.properties @@ -0,0 +1,93 @@ +annotation.processing.enabled=true +annotation.processing.enabled.in.editor=false +annotation.processing.processor.options= +annotation.processing.processors.list= +annotation.processing.run.all.processors=true +annotation.processing.source.output=${build.generated.sources.dir}/ap-source-output +build.classes.dir=${build.dir}/classes +build.classes.excludes=**/*.java,**/*.form +# This directory is removed when the project is cleaned: +build.dir=build +build.generated.dir=${build.dir}/generated +build.generated.sources.dir=${build.dir}/generated-sources +# Only compile against the classpath explicitly listed here: +build.sysclasspath=ignore +build.test.classes.dir=${build.dir}/test/classes +build.test.results.dir=${build.dir}/test/results +# Uncomment to specify the preferred debugger connection transport: +#debug.transport=dt_socket +debug.classpath=\ + ${run.classpath} +debug.modulepath=\ + ${run.modulepath} +debug.test.classpath=\ + ${run.test.classpath} +debug.test.modulepath=\ + ${run.test.modulepath} +# Files in build.classes.dir which should be excluded from distribution jar +dist.archive.excludes= +# This directory is removed when the project is cleaned: +dist.dir=dist +dist.jar=${dist.dir}/YaraJNIWrapper.jar +dist.javadoc.dir=${dist.dir}/javadoc +dist.jlink.dir=${dist.dir}/jlink +dist.jlink.output=${dist.jlink.dir}/YaraJNIWrapper +excludes= +includes=** +jar.compress=false +javac.classpath= +# Space-separated list of extra javac options +javac.compilerargs= +javac.deprecation=false +javac.external.vm=true +javac.modulepath= +javac.processormodulepath= +javac.processorpath=\ + ${javac.classpath} +javac.source=1.8 +javac.target=1.8 +javac.test.classpath=\ + ${javac.classpath}:\ + ${build.classes.dir} +javac.test.modulepath=\ + ${javac.modulepath} +javac.test.processorpath=\ + ${javac.test.classpath} +javadoc.additionalparam= +javadoc.author=false +javadoc.encoding=${source.encoding} +javadoc.html5=false +javadoc.noindex=false +javadoc.nonavbar=false +javadoc.notree=false +javadoc.private=false +javadoc.splitindex=true +javadoc.use=true +javadoc.version=false +javadoc.windowtitle= +# The jlink additional root modules to resolve +jlink.additionalmodules= +# The jlink additional command line parameters +jlink.additionalparam= +jlink.launcher=true +jlink.launcher.name=YaraJNIWrapper +meta.inf.dir=${src.dir}/META-INF +mkdist.disabled=true +platform.active=default_platform +run.classpath=\ + ${javac.classpath}:\ + ${build.classes.dir} +# Space-separated list of JVM arguments used when running the project. +# You may also define separate properties like run-sys-prop.name=value instead of -Dname=value. +# To set system properties for unit tests define test-sys-prop.name=value: +run.jvmargs= +run.modulepath=\ + ${javac.modulepath} +run.test.classpath=\ + ${javac.test.classpath}:\ + ${build.test.classes.dir} +run.test.modulepath=\ + ${javac.test.modulepath} +source.encoding=UTF-8 +src.dir=src +test.src.dir=test diff --git a/thirdparty/yara/YaraJNIWrapper/nbproject/project.xml b/thirdparty/yara/YaraJNIWrapper/nbproject/project.xml new file mode 100755 index 0000000000..df43138d7e --- /dev/null +++ b/thirdparty/yara/YaraJNIWrapper/nbproject/project.xml @@ -0,0 +1,15 @@ + + + org.netbeans.modules.java.j2seproject + + + YaraJNIWrapper + + + + + + + + + diff --git a/thirdparty/yara/YaraJNIWrapper/src/org/sleuthkit/autopsy/yara/YaraJNIWrapper.java b/thirdparty/yara/YaraJNIWrapper/src/org/sleuthkit/autopsy/yara/YaraJNIWrapper.java new file mode 100755 index 0000000000..0fc5e8f0f4 --- /dev/null +++ b/thirdparty/yara/YaraJNIWrapper/src/org/sleuthkit/autopsy/yara/YaraJNIWrapper.java @@ -0,0 +1,68 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2020 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.yara; + +import java.net.URISyntaxException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * + * native JNI interface to the yarabridge dll. + */ +public class YaraJNIWrapper { + + // Load the yarabridge.dll which should be located in the same directory as + // the jar file. If we need to use this code for debugging the dll this + // code will need to be modified to add that support. + static { + Path directoryPath = null; + try { + directoryPath = Paths.get(YaraJNIWrapper.class.getProtectionDomain().getCodeSource().getLocation().toURI()).getParent().toAbsolutePath(); + } catch (URISyntaxException ex) { + Logger.getLogger(YaraJNIWrapper.class.getName()).log(Level.SEVERE, null, ex); + } + String libraryPath = Paths.get(directoryPath != null ? directoryPath.toString() : "", "yarabridge.dll").toAbsolutePath().toString(); + System.load(libraryPath); + } + + /** + * Returns a list of rules that were found in the given byteBuffer. + * + * The rule path must be to a yara compile rule file. + * + * @param compiledRulesPath + * @param byteBuffer + * + * @return List of rules found rules. Null maybe returned if error occurred. + * + * @throws YaraWrapperException + */ + static public native List findRuleMatch(String compiledRulesPath, byte[] byteBuffer) throws YaraWrapperException; + + /** + * private constructor. + */ + private YaraJNIWrapper() { + } + +} diff --git a/thirdparty/yara/YaraJNIWrapper/src/org/sleuthkit/autopsy/yara/YaraWrapperException.java b/thirdparty/yara/YaraJNIWrapper/src/org/sleuthkit/autopsy/yara/YaraWrapperException.java new file mode 100755 index 0000000000..d93d436ffc --- /dev/null +++ b/thirdparty/yara/YaraJNIWrapper/src/org/sleuthkit/autopsy/yara/YaraWrapperException.java @@ -0,0 +1,37 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2020 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.yara; + +/** + * + * An exception class for the YaraWrapper Library + */ +public class YaraWrapperException extends Exception{ + + private static final long serialVersionUID = 1L; + + /** + * Create exception containing the error message + * + * @param msg Error message + */ + public YaraWrapperException(String msg) { + super(msg); + } +} diff --git a/thirdparty/yara/YaraWrapperTest/build.xml b/thirdparty/yara/YaraWrapperTest/build.xml new file mode 100755 index 0000000000..d108da9a59 --- /dev/null +++ b/thirdparty/yara/YaraWrapperTest/build.xml @@ -0,0 +1,73 @@ + + + + + + + + + + + Builds, tests, and runs the project YaraWrapperTest. + + + diff --git a/thirdparty/yara/YaraWrapperTest/manifest.mf b/thirdparty/yara/YaraWrapperTest/manifest.mf new file mode 100755 index 0000000000..328e8e5bc3 --- /dev/null +++ b/thirdparty/yara/YaraWrapperTest/manifest.mf @@ -0,0 +1,3 @@ +Manifest-Version: 1.0 +X-COMMENT: Main-Class will be added automatically by build + diff --git a/thirdparty/yara/YaraWrapperTest/nbproject/build-impl.xml b/thirdparty/yara/YaraWrapperTest/nbproject/build-impl.xml new file mode 100755 index 0000000000..98375a4932 --- /dev/null +++ b/thirdparty/yara/YaraWrapperTest/nbproject/build-impl.xml @@ -0,0 +1,1770 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Must set src.dir + Must set test.src.dir + Must set build.dir + Must set dist.dir + Must set build.classes.dir + Must set dist.javadoc.dir + Must set build.test.classes.dir + Must set build.test.results.dir + Must set build.classes.excludes + Must set dist.jar + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Must set javac.includes + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + No tests executed. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Must set JVM to use for profiling in profiler.info.jvm + Must set profiler agent JVM arguments in profiler.info.jvmargs.agent + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Must select some files in the IDE or set javac.includes + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + To run this application from the command line without Ant, try: + + java -jar "${dist.jar.resolved}" + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Must select one file in the IDE or set run.class + + + + Must select one file in the IDE or set run.class + + + + + + + + + + + + + + + + + + + + + + + Must select one file in the IDE or set debug.class + + + + + Must select one file in the IDE or set debug.class + + + + + Must set fix.includes + + + + + + + + + + This target only works when run from inside the NetBeans IDE. + + + + + + + + + Must select one file in the IDE or set profile.class + This target only works when run from inside the NetBeans IDE. + + + + + + + + + This target only works when run from inside the NetBeans IDE. + + + + + + + + + + + + + This target only works when run from inside the NetBeans IDE. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Must select one file in the IDE or set run.class + + + + + + Must select some files in the IDE or set test.includes + + + + + Must select one file in the IDE or set run.class + + + + + Must select one file in the IDE or set applet.url + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Must select some files in the IDE or set javac.includes + + + + + + + + + + + + + + + + + + + + + + + + Some tests failed; see details above. + + + + + + + + + Must select some files in the IDE or set test.includes + + + + Some tests failed; see details above. + + + + Must select some files in the IDE or set test.class + Must select some method in the IDE or set test.method + + + + Some tests failed; see details above. + + + + + Must select one file in the IDE or set test.class + + + + Must select one file in the IDE or set test.class + Must select some method in the IDE or set test.method + + + + + + + + + + + + + + Must select one file in the IDE or set applet.url + + + + + + + + + Must select one file in the IDE or set applet.url + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/thirdparty/yara/YaraWrapperTest/nbproject/private/private.xml b/thirdparty/yara/YaraWrapperTest/nbproject/private/private.xml new file mode 100755 index 0000000000..475096252c --- /dev/null +++ b/thirdparty/yara/YaraWrapperTest/nbproject/private/private.xml @@ -0,0 +1,4 @@ + + + + diff --git a/thirdparty/yara/YaraWrapperTest/nbproject/project.properties b/thirdparty/yara/YaraWrapperTest/nbproject/project.properties new file mode 100755 index 0000000000..c0126ab42a --- /dev/null +++ b/thirdparty/yara/YaraWrapperTest/nbproject/project.properties @@ -0,0 +1,99 @@ +annotation.processing.enabled=true +annotation.processing.enabled.in.editor=false +annotation.processing.processors.list= +annotation.processing.run.all.processors=true +annotation.processing.source.output=${build.generated.sources.dir}/ap-source-output +application.title=YaraWrapperTest +application.vendor=kelly +build.classes.dir=${build.dir}/classes +build.classes.excludes=**/*.java,**/*.form +# This directory is removed when the project is cleaned: +build.dir=build +build.generated.dir=${build.dir}/generated +build.generated.sources.dir=${build.dir}/generated-sources +# Only compile against the classpath explicitly listed here: +build.sysclasspath=ignore +build.test.classes.dir=${build.dir}/test/classes +build.test.results.dir=${build.dir}/test/results +# Uncomment to specify the preferred debugger connection transport: +#debug.transport=dt_socket +debug.classpath=\ + ${run.classpath} +debug.modulepath=\ + ${run.modulepath} +debug.test.classpath=\ + ${run.test.classpath} +debug.test.modulepath=\ + ${run.test.modulepath} +# Files in build.classes.dir which should be excluded from distribution jar +dist.archive.excludes= +# This directory is removed when the project is cleaned: +dist.dir=dist +dist.jar=${dist.dir}/YaraWrapperTest.jar +dist.javadoc.dir=${dist.dir}/javadoc +dist.jlink.dir=${dist.dir}/jlink +dist.jlink.output=${dist.jlink.dir}/YaraWrapperTest +endorsed.classpath= +excludes= +file.reference.YaraJNIWrapper.jar=../bin/YaraJNIWrapper.jar +includes=** +jar.compress=false +javac.classpath=\ + ${file.reference.YaraJNIWrapper.jar} +# Space-separated list of extra javac options +javac.compilerargs= +javac.deprecation=false +javac.external.vm=true +javac.modulepath= +javac.processormodulepath= +javac.processorpath=\ + ${javac.classpath} +javac.source=1.8 +javac.target=1.8 +javac.test.classpath=\ + ${javac.classpath}:\ + ${build.classes.dir} +javac.test.modulepath=\ + ${javac.modulepath} +javac.test.processorpath=\ + ${javac.test.classpath} +javadoc.additionalparam= +javadoc.author=false +javadoc.encoding=${source.encoding} +javadoc.html5=false +javadoc.noindex=false +javadoc.nonavbar=false +javadoc.notree=false +javadoc.private=false +javadoc.splitindex=true +javadoc.use=true +javadoc.version=false +javadoc.windowtitle= +# The jlink additional root modules to resolve +jlink.additionalmodules= +# The jlink additional command line parameters +jlink.additionalparam= +jlink.launcher=true +jlink.launcher.name=YaraWrapperTest +main.class=org.sleuthkit.autopsy.yara.YaraWrapperTest +manifest.file=manifest.mf +meta.inf.dir=${src.dir}/META-INF +mkdist.disabled=false +platform.active=default_platform +run.classpath=\ + ${javac.classpath}:\ + ${build.classes.dir} +# Space-separated list of JVM arguments used when running the project. +# You may also define separate properties like run-sys-prop.name=value instead of -Dname=value. +# To set system properties for unit tests define test-sys-prop.name=value: +run.jvmargs= +run.modulepath=\ + ${javac.modulepath} +run.test.classpath=\ + ${javac.test.classpath}:\ + ${build.test.classes.dir} +run.test.modulepath=\ + ${javac.test.modulepath} +source.encoding=UTF-8 +src.dir=src +test.src.dir=test diff --git a/thirdparty/yara/YaraWrapperTest/nbproject/project.xml b/thirdparty/yara/YaraWrapperTest/nbproject/project.xml new file mode 100755 index 0000000000..3541c3f5d7 --- /dev/null +++ b/thirdparty/yara/YaraWrapperTest/nbproject/project.xml @@ -0,0 +1,15 @@ + + + org.netbeans.modules.java.j2seproject + + + YaraWrapperTest + + + + + + + + + diff --git a/thirdparty/yara/YaraWrapperTest/resources/hello.compiled b/thirdparty/yara/YaraWrapperTest/resources/hello.compiled new file mode 100755 index 0000000000..8912235125 Binary files /dev/null and b/thirdparty/yara/YaraWrapperTest/resources/hello.compiled differ diff --git a/thirdparty/yara/YaraWrapperTest/resources/hello.txt b/thirdparty/yara/YaraWrapperTest/resources/hello.txt new file mode 100755 index 0000000000..5e1c309dae --- /dev/null +++ b/thirdparty/yara/YaraWrapperTest/resources/hello.txt @@ -0,0 +1 @@ +Hello World \ No newline at end of file diff --git a/thirdparty/yara/YaraWrapperTest/src/org/sleuthkit/autopsy/yara/YaraWrapperTest.java b/thirdparty/yara/YaraWrapperTest/src/org/sleuthkit/autopsy/yara/YaraWrapperTest.java new file mode 100755 index 0000000000..4a57abfef2 --- /dev/null +++ b/thirdparty/yara/YaraWrapperTest/src/org/sleuthkit/autopsy/yara/YaraWrapperTest.java @@ -0,0 +1,81 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2020 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.yara; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.sleuthkit.autopsy.yara.YaraJNIWrapper; +import org.sleuthkit.autopsy.yara.YaraWrapperException; + +/** + * Tests the YaraJNIWrapper code. + */ +public class YaraWrapperTest { + + private static final Logger logger = Logger.getLogger(YaraWrapperTest.class.getName()); + + public static void main(String[] args) { + if (args.length < 2) { + System.out.println("Please supply two arguments, a yara compiled rule path and a path to the file to scan."); + return; + } + + testFileRuleMatch(args[0], args[1]); + } + + /** + * Call the YaraJNIWrapper FindRuleMatch with the given path and output the + * results to the cl. + * + * @param compiledRulePath Path to yara compiled rule file + * @param filePath Path to file + */ + private static void testFileRuleMatch(String compiledRulePath, String filePath) { + Path path = Paths.get(filePath); + + try { + byte[] data = Files.readAllBytes(path); + + List list = YaraJNIWrapper.findRuleMatch(compiledRulePath, data); + + if (list != null) { + if (list.isEmpty()) { + System.out.println("FindRuleMatch return an empty list"); + } else { + System.out.println("Matching Rules:"); + for (String s : list) { + System.out.println(s); + } + } + } else { + logger.log(Level.SEVERE, "FindRuleMatch return a null list"); + } + + } catch (IOException | YaraWrapperException ex) { + logger.log(Level.SEVERE, "Error thrown from yarabridge", ex); + } + } + +} diff --git a/thirdparty/yara/bin/YaraJNIWrapper.jar b/thirdparty/yara/bin/YaraJNIWrapper.jar new file mode 100755 index 0000000000..749d7a6ae7 Binary files /dev/null and b/thirdparty/yara/bin/YaraJNIWrapper.jar differ diff --git a/thirdparty/yara/bin/yarabridge.dll b/thirdparty/yara/bin/yarabridge.dll new file mode 100755 index 0000000000..c74062a626 Binary files /dev/null and b/thirdparty/yara/bin/yarabridge.dll differ diff --git a/thirdparty/yara/yarabridge/yarabridge.sln b/thirdparty/yara/yarabridge/yarabridge.sln new file mode 100755 index 0000000000..5616f0a9d4 --- /dev/null +++ b/thirdparty/yara/yarabridge/yarabridge.sln @@ -0,0 +1,28 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio 14 +VisualStudioVersion = 14.0.25420.1 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "yarabridge", "yarabridge\yarabridge.vcxproj", "{7922D123-F27A-427B-B3EF-964F5E79FDDA}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|x64 = Debug|x64 + Debug|x86 = Debug|x86 + Release|x64 = Release|x64 + Release|x86 = Release|x86 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {7922D123-F27A-427B-B3EF-964F5E79FDDA}.Debug|x64.ActiveCfg = Debug|x64 + {7922D123-F27A-427B-B3EF-964F5E79FDDA}.Debug|x64.Build.0 = Debug|x64 + {7922D123-F27A-427B-B3EF-964F5E79FDDA}.Debug|x86.ActiveCfg = Debug|Win32 + {7922D123-F27A-427B-B3EF-964F5E79FDDA}.Debug|x86.Build.0 = Debug|Win32 + {7922D123-F27A-427B-B3EF-964F5E79FDDA}.Release|x64.ActiveCfg = Release|x64 + {7922D123-F27A-427B-B3EF-964F5E79FDDA}.Release|x64.Build.0 = Release|x64 + {7922D123-F27A-427B-B3EF-964F5E79FDDA}.Release|x86.ActiveCfg = Release|Win32 + {7922D123-F27A-427B-B3EF-964F5E79FDDA}.Release|x86.Build.0 = Release|Win32 + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/thirdparty/yara/yarabridge/yarabridge/YaraJNIWrapper.cpp b/thirdparty/yara/yarabridge/yarabridge/YaraJNIWrapper.cpp new file mode 100755 index 0000000000..0d36d2a039 --- /dev/null +++ b/thirdparty/yara/yarabridge/yarabridge/YaraJNIWrapper.cpp @@ -0,0 +1,138 @@ +/* +** YaraBridge +** +** Brian Carrier [carrier sleuthkit [dot] org] +** Copyright (c) 2010-2018 Brian Carrier. All Rights reserved +** +** This software is distributed under the Common Public License 1.0 +** +*/ + +#include +#include +#include "YaraJNIWrapper.h" +#include "yara.h" +#include +#include +#include +#include + +using std::string; +using std::vector; + + +/* + Callback method to be passed to yr_rules_scan_mem method. + user_data is expected to be a pointer to a string vector. +*/ +static int callback( + YR_SCAN_CONTEXT* context, + int message, + void* message_data, + void* user_data) +{ + if (message == CALLBACK_MSG_RULE_MATCHING) { + YR_RULE *rule = (YR_RULE *)message_data; + + ((std::vector*)user_data)->push_back(rule->identifier); + } + return CALLBACK_CONTINUE; +} + + +/* + Throw a new instance of YaraWrapperException with the given message. + + Unlike in JAVA throwing this exception will not stop the execution + of the method from which it is thrown. +*/ +static void throwException(JNIEnv *env, char * msg) { + jclass cls; + + cls = env->FindClass("org/sleuthkit/autopsy/yara/YaraWrapperException"); + if (cls == NULL) { + fprintf(stderr, "Failed to throw YaraWrapperException, cannot find class\n"); + return; + } + + env->ThrowNew(cls, msg); + +} + +/* + Generic method that will create a Java ArrayList object populating it with + the strings from the given vector. +*/ +jobject createArrayList(JNIEnv *env, std::vector vector) { + jclass cls_arrayList = env->FindClass("java/util/ArrayList"); + jmethodID constructor = env->GetMethodID(cls_arrayList, "", "(I)V"); + jmethodID method_add = env->GetMethodID(cls_arrayList, "add", "(Ljava/lang/Object;)Z"); + + jobject list = env->NewObject(cls_arrayList, constructor, vector.size()); + + for (std::string str : vector) { + jstring element = env->NewStringUTF(str.c_str()); + env->CallBooleanMethod(list, method_add, element); + env->DeleteLocalRef(element); + } + + return list; +} + +/* +* Class: org_sleuthkit_autopsy_yara_YaraJNIWrapper +* Method: FindRuleMatch +* Signature: (Ljava/lang/String;[B)Ljava/util/List; +*/ +JNIEXPORT jobject JNICALL Java_org_sleuthkit_autopsy_yara_YaraJNIWrapper_findRuleMatch +(JNIEnv * env, jclass cls, jstring compiledRulePath, jbyteArray fileByteArray) { + + char errorMessage[256]; + const char *nativeString = env->GetStringUTFChars(compiledRulePath, 0); + jobject resultList = NULL; + + int result; + if ((result = yr_initialize()) != ERROR_SUCCESS) { + sprintf_s(errorMessage, "libyara initialization error (%d)\n", result); + throwException(env, errorMessage); + return resultList; + } + + while (1) { + YR_RULES *rules = NULL; + if ((result = yr_rules_load(nativeString, &rules)) != ERROR_SUCCESS) { + sprintf_s(errorMessage, "Failed to load compiled yara rules (%d)\n", result); + throwException(env, errorMessage); + break; + } + + boolean isCopy; + int byteArrayLength = env->GetArrayLength(fileByteArray); + if (byteArrayLength == 0) { + throwException(env, "Unable to scan for matches. File byte array size was 0."); + break; + } + + jbyte* nativeByteArray = env->GetByteArrayElements(fileByteArray, &isCopy); + int flags = 0; + std::vector scanResults; + + result = yr_rules_scan_mem(rules, (unsigned char*)nativeByteArray, byteArrayLength, flags, callback, &scanResults, 1000000); + env->ReleaseByteArrayElements(fileByteArray, nativeByteArray, 0); + + if (result != ERROR_SUCCESS) { + sprintf_s(errorMessage, "Yara file scan failed (%d)\n", result); + throwException(env, errorMessage); + break; + } + + resultList = createArrayList(env, scanResults); + break; + } + + env->ReleaseStringUTFChars(compiledRulePath, nativeString); + yr_finalize(); + + return resultList; + +} \ No newline at end of file diff --git a/thirdparty/yara/yarabridge/yarabridge/YaraJNIWrapper.h b/thirdparty/yara/yarabridge/yarabridge/YaraJNIWrapper.h new file mode 100755 index 0000000000..6c5f5f5d75 --- /dev/null +++ b/thirdparty/yara/yarabridge/yarabridge/YaraJNIWrapper.h @@ -0,0 +1,21 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_sleuthkit_autopsy_yara_YaraJNIWrapper */ + +#ifndef _Included_org_sleuthkit_autopsy_yara_YaraJNIWrapper +#define _Included_org_sleuthkit_autopsy_yara_YaraJNIWrapper +#ifdef __cplusplus +extern "C" { +#endif + /* + * Class: org_sleuthkit_autopsy_yara_YaraJNIWrapper + * Method: FindRuleMatch + * Signature: (Ljava/lang/String;[B)Ljava/util/List; + */ + JNIEXPORT jobject JNICALL Java_org_sleuthkit_autopsy_yara_YaraJNIWrapper_findRuleMatch + (JNIEnv *, jclass, jstring, jbyteArray); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/thirdparty/yara/yarabridge/yarabridge/stdafx.cpp b/thirdparty/yara/yarabridge/yarabridge/stdafx.cpp new file mode 100755 index 0000000000..68193816bc --- /dev/null +++ b/thirdparty/yara/yarabridge/yarabridge/stdafx.cpp @@ -0,0 +1,8 @@ +// stdafx.cpp : source file that includes just the standard includes +// yarabridge.pch will be the pre-compiled header +// stdafx.obj will contain the pre-compiled type information + +#include "stdafx.h" + +// TODO: reference any additional headers you need in STDAFX.H +// and not in this file diff --git a/thirdparty/yara/yarabridge/yarabridge/stdafx.h b/thirdparty/yara/yarabridge/yarabridge/stdafx.h new file mode 100755 index 0000000000..f3a07375c7 --- /dev/null +++ b/thirdparty/yara/yarabridge/yarabridge/stdafx.h @@ -0,0 +1,16 @@ +// stdafx.h : include file for standard system include files, +// or project specific include files that are used frequently, but +// are changed infrequently +// + +#pragma once + +#include "targetver.h" + +#define WIN32_LEAN_AND_MEAN // Exclude rarely-used stuff from Windows headers +// Windows Header Files: +#include + + + +// TODO: reference additional headers your program requires here diff --git a/thirdparty/yara/yarabridge/yarabridge/targetver.h b/thirdparty/yara/yarabridge/yarabridge/targetver.h new file mode 100755 index 0000000000..87c0086de7 --- /dev/null +++ b/thirdparty/yara/yarabridge/yarabridge/targetver.h @@ -0,0 +1,8 @@ +#pragma once + +// Including SDKDDKVer.h defines the highest available Windows platform. + +// If you wish to build your application for a previous Windows platform, include WinSDKVer.h and +// set the _WIN32_WINNT macro to the platform you wish to support before including SDKDDKVer.h. + +#include diff --git a/thirdparty/yara/yarabridge/yarabridge/yarabridge.vcxproj b/thirdparty/yara/yarabridge/yarabridge/yarabridge.vcxproj new file mode 100755 index 0000000000..ce5dd10c80 --- /dev/null +++ b/thirdparty/yara/yarabridge/yarabridge/yarabridge.vcxproj @@ -0,0 +1,174 @@ + + + + + Debug + Win32 + + + Release + Win32 + + + Debug + x64 + + + Release + x64 + + + + {7922D123-F27A-427B-B3EF-964F5E79FDDA} + Win32Proj + yarabridge + 8.1 + + + + DynamicLibrary + true + v140 + Unicode + + + DynamicLibrary + false + v140 + true + Unicode + + + DynamicLibrary + true + v140 + Unicode + + + DynamicLibrary + false + v140 + true + Unicode + + + + + + + + + + + + + + + + + + + + + true + + + true + $(JDK_HOME)\include\win32;$(JDK_HOME)\include;..\..\..\..\..\yara\libyara\include;$(IncludePath) + $(LibraryPath) + + + false + + + false + $(JDK_HOME)\include\win32;$(JDK_HOME)\include;..\..\..\..\..\yara\libyara\include;$(IncludePath) + $(LibraryPath) + + + + + + Level3 + Disabled + WIN32;_DEBUG;_WINDOWS;_USRDLL;YARABRIDGE_EXPORTS;%(PreprocessorDefinitions) + + + Windows + true + + + + + + + Level3 + Disabled + _DEBUG;_WINDOWS;_USRDLL;YARABRIDGE_EXPORTS;%(PreprocessorDefinitions) + + + Windows + true + $(ProjectDir)..\..\..\..\..\yara\windows\vs2015\libyara\Release;%(AdditionalLibraryDirectories) + ws2_32.lib;crypt32.lib;libyara64.lib;%(AdditionalDependencies) + + + copy "$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName).dll" "$(SolutionDir)..\bin\$(ProjectName).dll" + + + + + Level3 + + + MaxSpeed + true + true + WIN32;NDEBUG;_WINDOWS;_USRDLL;YARABRIDGE_EXPORTS;%(PreprocessorDefinitions) + + + Windows + true + true + true + + + + + Level3 + + + MaxSpeed + true + true + NDEBUG;_WINDOWS;_USRDLL;YARABRIDGE_EXPORTS;%(PreprocessorDefinitions) + $(JDK_HOME)\include;%(AdditionalIncludeDirectories) + + + Windows + true + true + true + $(ProjectDir)..\..\..\..\..\yara\windows\vs2015\libyara\Release;%(AdditionalLibraryDirectories) + ws2_32.lib;crypt32.lib;libyara64.lib;%(AdditionalDependencies) + + + copy "$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName).dll" "$(SolutionDir)..\bin\$(ProjectName).dll" + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java index 9957c52454..12f3342dac 100644 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java @@ -23,9 +23,11 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.logging.Level; import java.util.regex.Matcher; @@ -79,7 +81,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { private static final int MBOX_SIZE_TO_SPLIT = 1048576000; private Case currentCase; - + /** * Empty constructor. */ @@ -152,7 +154,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { return ProcessResult.ERROR; } } - + if (isMbox) { return processMBox(abstractFile); } @@ -168,7 +170,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { if (isVcardFile) { return processVcard(abstractFile); } - + return ProcessResult.OK; } @@ -211,12 +213,13 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { PstParser parser = new PstParser(services); PstParser.ParseResult result = parser.open(file, abstractFile.getId()); - + + switch( result) { case OK: Iterator pstMsgIterator = parser.getEmailMessageIterator(); if (pstMsgIterator != null) { - processEmails(parser.getPartialEmailMessages(), pstMsgIterator , abstractFile); + processEmails(parser.getPartialEmailMessages(), pstMsgIterator, abstractFile); if (context.fileIngestIsCancelled()) { return ProcessResult.OK; } @@ -272,7 +275,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { if (file.delete() == false) { logger.log(Level.INFO, "Failed to delete temp file: {0}", file.getName()); //NON-NLS } - + return ProcessResult.OK; } @@ -451,7 +454,9 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { List derivedFiles = new ArrayList<>(); - BlackboardArtifact msgArtifact = addEmailArtifact(message, abstractFile); + AccountFileInstanceCache accountFileInstanceCache = new AccountFileInstanceCache(abstractFile, currentCase); + BlackboardArtifact msgArtifact = addEmailArtifact(message, abstractFile, accountFileInstanceCache); + accountFileInstanceCache.clear(); if ((msgArtifact != null) && (message.hasAttachment())) { derivedFiles.addAll(handleAttachments(message.getAttachments(), abstractFile, msgArtifact)); @@ -527,7 +532,11 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { * @param fullMessageIterator * @param abstractFile */ - private void processEmails(List partialEmailsForThreading, Iterator fullMessageIterator, AbstractFile abstractFile) { + private void processEmails(List partialEmailsForThreading, Iterator fullMessageIterator, + AbstractFile abstractFile) { + + // Create cache for accounts + AccountFileInstanceCache accountFileInstanceCache = new AccountFileInstanceCache(abstractFile, currentCase); // Putting try/catch around this to catch any exception and still allow // the creation of the artifacts to continue. @@ -560,7 +569,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { } } - BlackboardArtifact msgArtifact = addEmailArtifact(current, abstractFile); + BlackboardArtifact msgArtifact = addEmailArtifact(current, abstractFile, accountFileInstanceCache); if ((msgArtifact != null) && (current.hasAttachment())) { derivedFiles.addAll(handleAttachments(current.getAttachments(), abstractFile, msgArtifact )); @@ -676,7 +685,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { * @return The generated e-mail message artifact. */ @Messages({"ThunderbirdMboxFileIngestModule.addArtifact.indexError.message=Failed to index email message detected artifact for keyword search."}) - private BlackboardArtifact addEmailArtifact(EmailMessage email, AbstractFile abstractFile) { + private BlackboardArtifact addEmailArtifact(EmailMessage email, AbstractFile abstractFile, AccountFileInstanceCache accountFileInstanceCache) { BlackboardArtifact bbart = null; List bbattributes = new ArrayList<>(); String to = email.getRecipients(); @@ -706,7 +715,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { if (senderAddressList.size() == 1) { senderAddress = senderAddressList.get(0); try { - senderAccountInstance = currentCase.getSleuthkitCase().getCommunicationsManager().createAccountFileInstance(Account.Type.EMAIL, senderAddress, EmailParserModuleFactory.getModuleName(), abstractFile); + senderAccountInstance = accountFileInstanceCache.getAccountInstance(senderAddress); } catch(TskCoreException ex) { logger.log(Level.WARNING, "Failed to create account for email address " + senderAddress, ex); //NON-NLS @@ -731,9 +740,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { return null; } try { - AccountFileInstance recipientAccountInstance = - currentCase.getSleuthkitCase().getCommunicationsManager().createAccountFileInstance(Account.Type.EMAIL, addr, - EmailParserModuleFactory.getModuleName(), abstractFile); + AccountFileInstance recipientAccountInstance = accountFileInstanceCache.getAccountInstance(addr); recipientAccountInstances.add(recipientAccountInstance); } catch(TskCoreException ex) { @@ -835,6 +842,56 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { } } + /** + * Cache for storing AccountFileInstance. + * The idea is that emails will be used multiple times in a file and + * we shouldn't do a database lookup each time. + */ + static private class AccountFileInstanceCache { + private final Map cacheMap; + private final AbstractFile file; + private final Case currentCase; + + /** + * Create a new cache. Caches are linked to a specific file. + * @param file + * @param currentCase + */ + AccountFileInstanceCache(AbstractFile file, Case currentCase) { + cacheMap= new HashMap<>(); + this.file = file; + this.currentCase = currentCase; + } + + /** + * Get the account file instance from the cache or the database. + * + * @param email The email for this account. + * + * @return The corresponding AccountFileInstance + * + * @throws TskCoreException + */ + AccountFileInstance getAccountInstance(String email) throws TskCoreException { + if (cacheMap.containsKey(email)) { + return cacheMap.get(email); + } + + AccountFileInstance accountInstance = + currentCase.getSleuthkitCase().getCommunicationsManager().createAccountFileInstance(Account.Type.EMAIL, email, + EmailParserModuleFactory.getModuleName(), file); + cacheMap.put(email, accountInstance); + return accountInstance; + } + + /** + * Clears the cache. + */ + void clear() { + cacheMap.clear(); + } + } + /** * Post an error message for the user. *