diff --git a/CentralRepository/src/org/sleuthkit/autopsy/centralrepository/contentviewer/DataContentViewerOtherCases.java b/CentralRepository/src/org/sleuthkit/autopsy/centralrepository/contentviewer/DataContentViewerOtherCases.java index c820788349..4c804cf703 100644 --- a/CentralRepository/src/org/sleuthkit/autopsy/centralrepository/contentviewer/DataContentViewerOtherCases.java +++ b/CentralRepository/src/org/sleuthkit/autopsy/centralrepository/contentviewer/DataContentViewerOtherCases.java @@ -134,7 +134,7 @@ public class DataContentViewerOtherCases extends javax.swing.JPanel implements D try { EamDb dbManager = EamDb.getInstance(); for (EamArtifact eamArtifact : correlatedArtifacts) { - percentage = dbManager.getCommonalityPercentageForTypeValue(eamArtifact); + percentage = dbManager.getCommonalityPercentageForTypeValue(eamArtifact.getCorrelationType(), eamArtifact.getCorrelationValue()); msg.append(Bundle.DataContentViewerOtherCases_correlatedArtifacts_byType(percentage, eamArtifact.getCorrelationType().getDisplayName(), eamArtifact.getCorrelationValue())); @@ -366,8 +366,7 @@ public class DataContentViewerOtherCases extends javax.swing.JPanel implements D /** * Scan a Node for blackboard artifacts / content that we can correlate on - * and create the corresponding Central Repository artifacts for - * display + * and create the corresponding Central Repository artifacts for display * * @param node The node to view * @@ -388,10 +387,7 @@ public class DataContentViewerOtherCases extends javax.swing.JPanel implements D EamDb dbManager = EamDb.getInstance(); artifactTypes = dbManager.getCorrelationTypes(); if (bbArtifact != null) { - EamArtifact eamArtifact = EamArtifactUtil.fromBlackboardArtifact(bbArtifact, false, artifactTypes, false); - if (eamArtifact != null) { - ret.add(eamArtifact); - } + ret.addAll(EamArtifactUtil.fromBlackboardArtifact(bbArtifact, false, artifactTypes, false)); } } catch (EamDbException ex) { LOGGER.log(Level.SEVERE, "Error retrieving correlation types", ex); // NON-NLS @@ -460,11 +456,11 @@ public class DataContentViewerOtherCases extends javax.swing.JPanel implements D * * @return A collection of correlated artifact instances from other cases */ - private Collection getCorrelatedInstances(EamArtifact eamArtifact, String dataSourceName, String deviceId) { + private Collection getCorrelatedInstances(EamArtifact.Type aType, String value, String dataSourceName, String deviceId) { String caseUUID = Case.getCurrentCase().getName(); try { EamDb dbManager = EamDb.getInstance(); - Collection artifactInstances = dbManager.getArtifactInstancesByTypeValue(eamArtifact).stream() + Collection artifactInstances = dbManager.getArtifactInstancesByTypeValue(aType, value).stream() .filter(artifactInstance -> !artifactInstance.getEamCase().getCaseUUID().equals(caseUUID) || !artifactInstance.getEamDataSource().getName().equals(dataSourceName) || !artifactInstance.getEamDataSource().getDeviceID().equals(deviceId)) @@ -483,8 +479,8 @@ public class DataContentViewerOtherCases extends javax.swing.JPanel implements D * * @param eamArtifact Artifact to use for ArtifactTypeEnum matching * - * @return List of Central Repository Artifact Instances, empty - * list if none found + * @return List of Central Repository Artifact Instances, empty list if none + * found */ public Collection getReferenceInstancesAsArtifactInstances(EamArtifact eamArtifact) { Collection eamArtifactInstances = new ArrayList<>(); @@ -542,7 +538,7 @@ public class DataContentViewerOtherCases extends javax.swing.JPanel implements D correlatedArtifacts.addAll(getArtifactsFromCorrelatableAttributes(node)); correlatedArtifacts.forEach((eamArtifact) -> { // get local instances - Collection eamArtifactInstances = getCorrelatedInstances(eamArtifact, dataSourceName, deviceId); + Collection eamArtifactInstances = getCorrelatedInstances(eamArtifact.getCorrelationType(), eamArtifact.getCorrelationValue(), dataSourceName, deviceId); // get global instances eamArtifactInstances.addAll(getReferenceInstancesAsArtifactInstances(eamArtifact)); diff --git a/CentralRepository/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java b/CentralRepository/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java index 4e78438199..8f25f471c6 100644 --- a/CentralRepository/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java +++ b/CentralRepository/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java @@ -570,7 +570,7 @@ public abstract class AbstractSqlEamDb implements EamDb { * @return List of artifact instances for a given type/value */ @Override - public List getArtifactInstancesByTypeValue(EamArtifact eamArtifact) throws EamDbException { + public List getArtifactInstancesByTypeValue(EamArtifact.Type aType, String value) throws EamDbException { Connection conn = connect(); List artifactInstances = new ArrayList<>(); @@ -579,7 +579,7 @@ public abstract class AbstractSqlEamDb implements EamDb { PreparedStatement preparedStatement = null; ResultSet resultSet = null; - String tableName = EamDbUtil.correlationTypeToInstanceTableName(eamArtifact.getCorrelationType()); + String tableName = EamDbUtil.correlationTypeToInstanceTableName(aType); StringBuilder sql = new StringBuilder(); sql.append("SELECT cases.case_name, cases.case_uid, data_sources.name, device_id, file_path, known_status, comment FROM "); sql.append(tableName); @@ -593,7 +593,7 @@ public abstract class AbstractSqlEamDb implements EamDb { try { preparedStatement = conn.prepareStatement(sql.toString()); - preparedStatement.setString(1, eamArtifact.getCorrelationValue()); + preparedStatement.setString(1, value); resultSet = preparedStatement.executeQuery(); while (resultSet.next()) { artifactInstance = getEamArtifactInstanceFromResultSet(resultSet); @@ -673,14 +673,14 @@ public abstract class AbstractSqlEamDb implements EamDb { * ArtifactValue. */ @Override - public Long getCountArtifactInstancesByTypeValue(EamArtifact eamArtifact) throws EamDbException { + public Long getCountArtifactInstancesByTypeValue(EamArtifact.Type aType, String value) throws EamDbException { Connection conn = connect(); Long instanceCount = 0L; PreparedStatement preparedStatement = null; ResultSet resultSet = null; - String tableName = EamDbUtil.correlationTypeToInstanceTableName(eamArtifact.getCorrelationType()); + String tableName = EamDbUtil.correlationTypeToInstanceTableName(aType); StringBuilder sql = new StringBuilder(); sql.append("SELECT count(*) FROM "); sql.append(tableName); @@ -688,7 +688,7 @@ public abstract class AbstractSqlEamDb implements EamDb { try { preparedStatement = conn.prepareStatement(sql.toString()); - preparedStatement.setString(1, eamArtifact.getCorrelationValue()); + preparedStatement.setString(1, value); resultSet = preparedStatement.executeQuery(); resultSet.next(); instanceCount = resultSet.getLong(1); @@ -715,8 +715,8 @@ public abstract class AbstractSqlEamDb implements EamDb { * @return Int between 0 and 100 */ @Override - public int getCommonalityPercentageForTypeValue(EamArtifact eamArtifact) throws EamDbException { - Double uniqueTypeValueTuples = getCountUniqueCaseDataSourceTuplesHavingTypeValue(eamArtifact).doubleValue(); + public int getCommonalityPercentageForTypeValue(EamArtifact.Type aType, String value) throws EamDbException { + Double uniqueTypeValueTuples = getCountUniqueCaseDataSourceTuplesHavingTypeValue(aType, value).doubleValue(); Double uniqueCaseDataSourceTuples = getCountUniqueCaseDataSourceTuples().doubleValue(); Double commonalityPercentage = uniqueTypeValueTuples / uniqueCaseDataSourceTuples * 100; return commonalityPercentage.intValue(); @@ -733,14 +733,14 @@ public abstract class AbstractSqlEamDb implements EamDb { * @return Number of unique tuples */ @Override - public Long getCountUniqueCaseDataSourceTuplesHavingTypeValue(EamArtifact eamArtifact) throws EamDbException { + public Long getCountUniqueCaseDataSourceTuplesHavingTypeValue(EamArtifact.Type aType, String value) throws EamDbException { Connection conn = connect(); Long instanceCount = 0L; PreparedStatement preparedStatement = null; ResultSet resultSet = null; - String tableName = EamDbUtil.correlationTypeToInstanceTableName(eamArtifact.getCorrelationType()); + String tableName = EamDbUtil.correlationTypeToInstanceTableName(aType); StringBuilder sql = new StringBuilder(); sql.append("SELECT count(*) FROM (SELECT DISTINCT case_id, data_source_id FROM "); sql.append(tableName); @@ -750,7 +750,7 @@ public abstract class AbstractSqlEamDb implements EamDb { try { preparedStatement = conn.prepareStatement(sql.toString()); - preparedStatement.setString(1, eamArtifact.getCorrelationValue()); + preparedStatement.setString(1, value); resultSet = preparedStatement.executeQuery(); resultSet.next(); instanceCount = resultSet.getLong(1); @@ -814,16 +814,14 @@ public abstract class AbstractSqlEamDb implements EamDb { * associated with the caseDisplayName and dataSource of the given * eamArtifact instance. * - * @param eamInstance Instance with caseName and dataSource to search for - * - * @param eamInstance Instance with caseDisplayName and dataSource to search - * for + * @param caseUUID Case ID to search for + * @param dataSourceID Data source ID to search for * * @return Number of artifact instances having caseDisplayName and * dataSource */ @Override - public Long getCountArtifactInstancesByCaseDataSource(EamArtifactInstance eamInstance) throws EamDbException { + public Long getCountArtifactInstancesByCaseDataSource(String caseUUID, String dataSourceID) throws EamDbException { Connection conn = connect(); Long instanceCount = 0L; @@ -847,8 +845,8 @@ public abstract class AbstractSqlEamDb implements EamDb { preparedStatement = conn.prepareStatement(sql.toString()); for (int i = 0; i < artifactTypes.size(); ++i) { - preparedStatement.setString(2 * i + 1, eamInstance.getEamCase().getCaseUUID()); - preparedStatement.setString(2 * i + 2, eamInstance.getEamDataSource().getDeviceID()); + preparedStatement.setString(2 * i + 1, caseUUID); + preparedStatement.setString(2 * i + 2, dataSourceID); } resultSet = preparedStatement.executeQuery(); @@ -1111,12 +1109,13 @@ public abstract class AbstractSqlEamDb implements EamDb { * Gets list of matching eamArtifact instances that have knownStatus = * "Bad". * - * @param eamArtifact Artifact containing Type and Value - * + * @param aType EamArtifact.Type to search for + * @param value Value to search for + * * @return List with 0 or more matching eamArtifact instances. */ @Override - public List getArtifactInstancesKnownBad(EamArtifact eamArtifact) throws EamDbException { + public List getArtifactInstancesKnownBad(EamArtifact.Type aType, String value) throws EamDbException { Connection conn = connect(); List artifactInstances = new ArrayList<>(); @@ -1125,7 +1124,7 @@ public abstract class AbstractSqlEamDb implements EamDb { PreparedStatement preparedStatement = null; ResultSet resultSet = null; - String tableName = EamDbUtil.correlationTypeToInstanceTableName(eamArtifact.getCorrelationType()); + String tableName = EamDbUtil.correlationTypeToInstanceTableName(aType); StringBuilder sql = new StringBuilder(); sql.append("SELECT cases.case_name, cases.case_uid, data_sources.name, device_id, file_path, known_status, comment FROM "); sql.append(tableName); @@ -1139,7 +1138,7 @@ public abstract class AbstractSqlEamDb implements EamDb { try { preparedStatement = conn.prepareStatement(sql.toString()); - preparedStatement.setString(1, eamArtifact.getCorrelationValue()); + preparedStatement.setString(1, value); preparedStatement.setString(2, TskData.FileKnown.BAD.name()); resultSet = preparedStatement.executeQuery(); while (resultSet.next()) { @@ -1160,19 +1159,20 @@ public abstract class AbstractSqlEamDb implements EamDb { /** * Count matching eamArtifacts instances that have knownStatus = "Bad". * - * @param eamArtifact Artifact containing Type and Value + * @param aType EamArtifact.Type to search for + * @param value Value to search for * * @return Number of matching eamArtifacts */ @Override - public Long getCountArtifactInstancesKnownBad(EamArtifact eamArtifact) throws EamDbException { + public Long getCountArtifactInstancesKnownBad(EamArtifact.Type aType, String value) throws EamDbException { Connection conn = connect(); Long badInstances = 0L; PreparedStatement preparedStatement = null; ResultSet resultSet = null; - String tableName = EamDbUtil.correlationTypeToInstanceTableName(eamArtifact.getCorrelationType()); + String tableName = EamDbUtil.correlationTypeToInstanceTableName(aType); StringBuilder sql = new StringBuilder(); sql.append("SELECT count(*) FROM "); sql.append(tableName); @@ -1180,7 +1180,7 @@ public abstract class AbstractSqlEamDb implements EamDb { try { preparedStatement = conn.prepareStatement(sql.toString()); - preparedStatement.setString(1, eamArtifact.getCorrelationValue()); + preparedStatement.setString(1, value); preparedStatement.setString(2, TskData.FileKnown.BAD.name()); resultSet = preparedStatement.executeQuery(); resultSet.next(); @@ -1200,7 +1200,8 @@ public abstract class AbstractSqlEamDb implements EamDb { * Gets list of distinct case display names, where each case has 1+ Artifact * Instance matching eamArtifact with knownStatus = "Bad". * - * @param eamArtifact Artifact containing Type and Value + * @param aType EamArtifact.Type to search for + * @param value Value to search for * * @return List of cases containing this artifact with instances marked as * bad @@ -1208,7 +1209,7 @@ public abstract class AbstractSqlEamDb implements EamDb { * @throws EamDbException */ @Override - public List getListCasesHavingArtifactInstancesKnownBad(EamArtifact eamArtifact) throws EamDbException { + public List getListCasesHavingArtifactInstancesKnownBad(EamArtifact.Type aType, String value) throws EamDbException { Connection conn = connect(); Collection caseNames = new LinkedHashSet<>(); @@ -1216,7 +1217,7 @@ public abstract class AbstractSqlEamDb implements EamDb { PreparedStatement preparedStatement = null; ResultSet resultSet = null; - String tableName = EamDbUtil.correlationTypeToInstanceTableName(eamArtifact.getCorrelationType()); + String tableName = EamDbUtil.correlationTypeToInstanceTableName(aType); StringBuilder sql = new StringBuilder(); sql.append("SELECT DISTINCT case_name FROM "); sql.append(tableName); @@ -1230,7 +1231,7 @@ public abstract class AbstractSqlEamDb implements EamDb { try { preparedStatement = conn.prepareStatement(sql.toString()); - preparedStatement.setString(1, eamArtifact.getCorrelationValue()); + preparedStatement.setString(1, value); preparedStatement.setString(2, TskData.FileKnown.BAD.name()); resultSet = preparedStatement.executeQuery(); while (resultSet.next()) { @@ -1250,15 +1251,16 @@ public abstract class AbstractSqlEamDb implements EamDb { /** * Is the artifact known as bad according to the reference entries? * - * @param eamArtifact Artifact containing Type and Value + * @param aType EamArtifact.Type to search for + * @param value Value to search for * * @return Global known status of the artifact */ @Override - public boolean isArtifactlKnownBadByReference(EamArtifact eamArtifact) throws EamDbException { + public boolean isArtifactlKnownBadByReference(EamArtifact.Type aType, String value) throws EamDbException { // TEMP: Only support file correlation type - if (eamArtifact.getCorrelationType().getId() != EamArtifact.FILES_TYPE_ID) { + if (aType.getId() != EamArtifact.FILES_TYPE_ID) { return false; } @@ -1270,8 +1272,8 @@ public abstract class AbstractSqlEamDb implements EamDb { String sql = "SELECT count(*) FROM %s WHERE value=? AND known_status=?"; try { - preparedStatement = conn.prepareStatement(String.format(sql, EamDbUtil.correlationTypeToReferenceTableName(eamArtifact.getCorrelationType()))); - preparedStatement.setString(1, eamArtifact.getCorrelationValue()); + preparedStatement = conn.prepareStatement(String.format(sql, EamDbUtil.correlationTypeToReferenceTableName(aType))); + preparedStatement.setString(1, value); preparedStatement.setString(2, TskData.FileKnown.BAD.name()); resultSet = preparedStatement.executeQuery(); resultSet.next(); @@ -1468,7 +1470,8 @@ public abstract class AbstractSqlEamDb implements EamDb { * Add a new reference instance * * @param eamGlobalFileInstance The reference instance to add - * @param correlationType Correlation Type that this Reference Instance is + * @param correlationType Correlation Type that this Reference + * Instance is * * @throws EamDbException */ @@ -1508,7 +1511,7 @@ public abstract class AbstractSqlEamDb implements EamDb { try { // FUTURE: have a separate global_files table for each Type. String sql = "INSERT INTO %s(reference_set_id, value, known_status, comment) VALUES (?, ?, ?, ?) " - + getConflictClause(); + + getConflictClause(); bulkPs = conn.prepareStatement(String.format(sql, EamDbUtil.correlationTypeToReferenceTableName(contentType))); @@ -1532,7 +1535,7 @@ public abstract class AbstractSqlEamDb implements EamDb { /** * Get all reference entries having a given correlation type and value * - * @param aType Type to use for matching + * @param aType Type to use for matching * @param aValue Value to use for matching * * @return List of all global file instances with a type and value @@ -1572,6 +1575,7 @@ public abstract class AbstractSqlEamDb implements EamDb { * @param newType New type to add. * * @return ID of this new Correlation Type + * * @throws EamDbException */ @Override diff --git a/CentralRepository/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifact.java b/CentralRepository/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifact.java index aeca81911f..3c2db37e39 100644 --- a/CentralRepository/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifact.java +++ b/CentralRepository/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifact.java @@ -44,7 +44,7 @@ public class EamArtifact implements Serializable { public static final int EMAIL_TYPE_ID = 2; public static final int PHONE_TYPE_ID = 3; public static final int USBID_TYPE_ID = 4; - + /** * Load the default correlation types * @@ -68,7 +68,8 @@ public class EamArtifact implements Serializable { public EamArtifact(Type correlationType, String correlationValue) { this.ID = ""; this.correlationType = correlationType; - this.correlationValue = correlationValue; + // Lower-case all values to normalize and improve correlation hits, going forward make sure this makes sense for all correlation types + this.correlationValue = correlationValue.toLowerCase(); this.artifactInstances = new ArrayList<>(); } @@ -113,7 +114,8 @@ public class EamArtifact implements Serializable { * @param correlationValue the correlationValue to set */ public void setCorrelationValue(String correlationValue) { - this.correlationValue = correlationValue; + // Lower-case all values to normalize and improve correlation hits, going forward make sure this makes sense for all correlation types + this.correlationValue = correlationValue.toLowerCase(); } /** @@ -330,7 +332,7 @@ public class EamArtifact implements Serializable { * When custom Types are added in the future, they are already supported * by just giving the desired value for the table name for each custom * Type. Possibly having all custom Types use a common table name. - * + * * @return the dbTableName */ public String getDbTableName() { diff --git a/CentralRepository/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactInstance.java b/CentralRepository/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactInstance.java index f782b7e2f6..4e73c7562b 100644 --- a/CentralRepository/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactInstance.java +++ b/CentralRepository/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactInstance.java @@ -107,7 +107,8 @@ public class EamArtifactInstance implements Serializable { this.ID = ID; this.eamCase = eamCase; this.eamDataSource = eamDataSource; - this.filePath = filePath; + // Lower case paths to normalize paths and improve correlation results, if this causes significant issues on case-sensitive file systems, remove + this.filePath = filePath.toLowerCase(); this.comment = comment; this.knownStatus = knownStatus; this.globalStatus = globalStatus; @@ -187,7 +188,8 @@ public class EamArtifactInstance implements Serializable { * @param filePath the filePath to set */ public void setFilePath(String filePath) { - this.filePath = filePath; + // Lower case paths to normalize paths and improve correlation results, if this causes significant issues on case-sensitive file systems, remove + this.filePath = filePath.toLowerCase(); } /** diff --git a/CentralRepository/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactUtil.java b/CentralRepository/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactUtil.java index 21acbc4fb4..a60d0b5251 100644 --- a/CentralRepository/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactUtil.java +++ b/CentralRepository/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactUtil.java @@ -18,6 +18,7 @@ */ package org.sleuthkit.autopsy.centralrepository.datamodel; +import java.util.ArrayList; import java.util.List; import java.util.logging.Level; import org.openide.util.NbBundle.Messages; @@ -46,31 +47,32 @@ public class EamArtifactUtil { return Bundle.EamArtifactUtil_emailaddresses_text(); } - /* + /** * Static factory method to examine a BlackboardArtifact to determine if it * has contents that can be used for Correlation. If so, return a * EamArtifact with a single EamArtifactInstance within. If not, return * null. * - * @param bbArtifact BlackboardArtifact to examine @return EamArtifact or - * null + * @param bbArtifact BlackboardArtifact to examine + * @return List of EamArtifacts */ - public static EamArtifact fromBlackboardArtifact(BlackboardArtifact bbArtifact, + public static List fromBlackboardArtifact(BlackboardArtifact bbArtifact, boolean includeInstances, List artifactTypes, boolean checkEnabled) { - EamArtifact eamArtifact = null; + List eamArtifacts = new ArrayList<>(); + for (EamArtifact.Type aType : artifactTypes) { if ((checkEnabled && aType.isEnabled()) || !checkEnabled) { - eamArtifact = getTypeFromBlackboardArtifact(aType, bbArtifact); - } - if (null != eamArtifact) { - break; + EamArtifact eamArtifact = getTypeFromBlackboardArtifact(aType, bbArtifact); + if (eamArtifact != null) { + eamArtifacts.add(eamArtifact); + } } } - if (null != eamArtifact && includeInstances) { + if (!eamArtifacts.isEmpty() && includeInstances) { try { AbstractFile af = Case.getCurrentCase().getSleuthkitCase().getAbstractFileById(bbArtifact.getObjectID()); if (null == af) { @@ -92,14 +94,17 @@ public class EamArtifactUtil { TskData.FileKnown.UNKNOWN, EamArtifactInstance.GlobalStatus.LOCAL ); - eamArtifact.addInstance(eamInstance); + + for (EamArtifact eamArtifact : eamArtifacts) { + eamArtifact.addInstance(eamInstance); + } } catch (TskCoreException ex) { LOGGER.log(Level.SEVERE, "Error creating artifact instance.", ex); // NON-NLS return null; } } - return eamArtifact; + return eamArtifacts; } /** @@ -129,8 +134,8 @@ public class EamArtifactUtil { || BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_DOWNLOAD.getTypeID() == artifactTypeID || BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_HISTORY.getTypeID() == artifactTypeID)) { + // Lower-case this to normalize domains value = bbArtifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DOMAIN)).getValueString(); - } else if (aType.getId() == EamArtifact.PHONE_TYPE_ID && (BlackboardArtifact.ARTIFACT_TYPE.TSK_CONTACT.getTypeID() == artifactTypeID || BlackboardArtifact.ARTIFACT_TYPE.TSK_CALLLOG.getTypeID() == artifactTypeID @@ -144,6 +149,15 @@ public class EamArtifactUtil { value = bbArtifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_TO)).getValueString(); } + // Remove all non-numeric symbols to semi-normalize phone numbers, preserving leading "+" character + if (value != null) { + String newValue = value.replaceAll("\\D", ""); + if (value.startsWith("+")) { + newValue = "+" + newValue; + } + + value = newValue; + } } else if (aType.getId() == EamArtifact.USBID_TYPE_ID && BlackboardArtifact.ARTIFACT_TYPE.TSK_DEVICE_ATTACHED.getTypeID() == artifactTypeID) { diff --git a/CentralRepository/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDb.java b/CentralRepository/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDb.java index 97226ddb29..dc52ea6a1f 100644 --- a/CentralRepository/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDb.java +++ b/CentralRepository/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDb.java @@ -25,6 +25,7 @@ import java.util.Set; * Main interface for interacting with the database */ public interface EamDb { + public static final int SCHEMA_VERSION = 1; /** @@ -50,24 +51,22 @@ public interface EamDb { /** * Shutdown the connection pool. - * + * * This closes the connection pool including all idle database connections. - * It will not close active/in-use connections. - * Thus, it is vital that there are no in-use connections - * when you call this method. - * + * It will not close active/in-use connections. Thus, it is vital that there + * are no in-use connections when you call this method. + * * @throws EamDbException if there is a problem closing the connection pool. */ void shutdownConnections() throws EamDbException; /** * Update settings - * - * When using updateSettings, - * if any database settings have changed, you should call - * shutdownConnections() before using any API methods. - * That will ensure that any old connections are closed - * and all new connections will be made using the new settings. + * + * When using updateSettings, if any database settings have changed, you + * should call shutdownConnections() before using any API methods. That will + * ensure that any old connections are closed and all new connections will + * be made using the new settings. */ void updateSettings(); @@ -206,18 +205,19 @@ public interface EamDb { void addArtifact(EamArtifact eamArtifact) throws EamDbException; /** - * Retrieves eamArtifact instances from the database that are associated with - * the eamArtifactType and eamArtifactValue of the given eamArtifact. + * Retrieves eamArtifact instances from the database that are associated + * with the eamArtifactType and eamArtifactValue of the given eamArtifact. * - * @param eamArtifact The type/value to look up (artifact with 0 instances) + * @param aType EamArtifact.Type to search for + * @param value Value to search for * * @return List of artifact instances for a given type/value */ - List getArtifactInstancesByTypeValue(EamArtifact eamArtifact) throws EamDbException; + List getArtifactInstancesByTypeValue(EamArtifact.Type aType, String value) throws EamDbException; /** - * Retrieves eamArtifact instances from the database that are associated with - * the aType and filePath + * Retrieves eamArtifact instances from the database that are associated + * with the aType and filePath * * @param aType EamArtifact.Type to search for * @param filePath File path to search for @@ -232,13 +232,13 @@ public interface EamDb { * Retrieves number of artifact instances in the database that are * associated with the ArtifactType and artifactValue of the given artifact. * - * @param eamArtifact Artifact with artifactType and artifactValue to search - * for + * @param aType EamArtifact.Type to search for + * @param value Value to search for * * @return Number of artifact instances having ArtifactType and * ArtifactValue. */ - Long getCountArtifactInstancesByTypeValue(EamArtifact eamArtifact) throws EamDbException; + Long getCountArtifactInstancesByTypeValue(EamArtifact.Type aType, String value) throws EamDbException; /** * Using the ArtifactType and ArtifactValue from the given eamArtfact, @@ -246,24 +246,24 @@ public interface EamDb { * where Type/Value is found) divided by (The total number of unique * case_id/datasource_id tuples in the database) expressed as a percentage. * - * @param eamArtifact Artifact with artifactType and artifactValue to search - * for + * @param aType EamArtifact.Type to search for + * @param value Value to search for * * @return Int between 0 and 100 */ - int getCommonalityPercentageForTypeValue(EamArtifact eamArtifact) throws EamDbException; + int getCommonalityPercentageForTypeValue(EamArtifact.Type aType, String value) throws EamDbException; /** * Retrieves number of unique caseDisplayName / dataSource tuples in the * database that are associated with the artifactType and artifactValue of * the given artifact. * - * @param eamArtifact Artifact with artifactType and artifactValue to search - * for + * @param aType EamArtifact.Type to search for + * @param value Value to search for * * @return Number of unique tuples */ - Long getCountUniqueCaseDataSourceTuplesHavingTypeValue(EamArtifact eamArtifact) throws EamDbException; + Long getCountUniqueCaseDataSourceTuplesHavingTypeValue(EamArtifact.Type aType, String value) throws EamDbException; /** * Retrieves number of unique caseDisplayName/dataSource tuples in the @@ -278,15 +278,13 @@ public interface EamDb { * associated with the caseDisplayName and dataSource of the given * eamArtifact instance. * - * @param eamInstance Instance with caseName and dataSource to search for - * - * @param eamInstance Instance with caseDisplayName and dataSource to search - * for + * @param caseUUID Case ID to search for + * @param dataSourceID Data source ID to search for * * @return Number of artifact instances having caseDisplayName and * dataSource */ - Long getCountArtifactInstancesByCaseDataSource(EamArtifactInstance eamInstance) throws EamDbException; + Long getCountArtifactInstancesByCaseDataSource(String caseUUID, String dataSourceID) throws EamDbException; /** * Adds an eamArtifact to an internal list to be later added to DB. Artifact @@ -309,52 +307,57 @@ public interface EamDb { void bulkInsertCases(List cases) throws EamDbException; /** - * Sets an eamArtifact instance as knownStatus = "Bad". If eamArtifact exists, - * it is updated. If eamArtifact does not exist nothing happens + * Sets an eamArtifact instance as knownStatus = "Bad". If eamArtifact + * exists, it is updated. If eamArtifact does not exist nothing happens * * @param eamArtifact Artifact containing exactly one (1) ArtifactInstance. */ void setArtifactInstanceKnownBad(EamArtifact eamArtifact) throws EamDbException; /** - * Gets list of matching eamArtifact instances that have knownStatus = "Bad". + * Gets list of matching eamArtifact instances that have knownStatus = + * "Bad". * - * @param eamArtifact Artifact containing Type and Value + * @param aType EamArtifact.Type to search for + * @param value Value to search for * * @return List with 0 or more matching eamArtifact instances. */ - List getArtifactInstancesKnownBad(EamArtifact eamArtifact) throws EamDbException; + List getArtifactInstancesKnownBad(EamArtifact.Type aType, String value) throws EamDbException; /** * Count matching eamArtifacts instances that have knownStatus = "Bad". * - * @param eamArtifact Artifact containing Type and Value + * @param aType EamArtifact.Type to search for + * @param value Value to search for * * @return Number of matching eamArtifacts */ - Long getCountArtifactInstancesKnownBad(EamArtifact eamArtifact) throws EamDbException; + Long getCountArtifactInstancesKnownBad(EamArtifact.Type aType, String value) throws EamDbException; /** * Gets list of distinct case display names, where each case has 1+ Artifact * Instance matching eamArtifact with knownStatus = "Bad". * - * @param eamArtifact Artifact containing Type and Value + * @param aType EamArtifact.Type to search for + * @param value Value to search for * * @return List of cases containing this artifact with instances marked as * bad * * @throws EamDbException */ - List getListCasesHavingArtifactInstancesKnownBad(EamArtifact eamArtifact) throws EamDbException; + List getListCasesHavingArtifactInstancesKnownBad(EamArtifact.Type aType, String value) throws EamDbException; /** * Is the artifact known as bad according to the reference entries? * - * @param eamArtifact Artifact containing Type and Value + * @param aType EamArtifact.Type to search for + * @param value Value to search for * * @return Global known status of the artifact */ - boolean isArtifactlKnownBadByReference(EamArtifact eamArtifact) throws EamDbException; + boolean isArtifactlKnownBadByReference(EamArtifact.Type aType, String value) throws EamDbException; /** * Add a new organization @@ -411,12 +414,13 @@ public interface EamDb { * Add a new reference instance * * @param eamGlobalFileInstance The reference instance to add - * @param correlationType Correlation Type that this Reference Instance is + * @param correlationType Correlation Type that this Reference + * Instance is * * @throws EamDbException */ - void addReferenceInstance(EamGlobalFileInstance eamGlobalFileInstance, EamArtifact.Type correlationType) throws EamDbException ; - + void addReferenceInstance(EamGlobalFileInstance eamGlobalFileInstance, EamArtifact.Type correlationType) throws EamDbException; + /** * Add a new global file instance to the bulk collection * @@ -425,13 +429,13 @@ public interface EamDb { * @throws EamDbException */ // void prepareGlobalFileInstance(EamGlobalFileInstance eamGlobalFileInstance) throws EamDbException; - /** * Insert the bulk collection of Global File Instances * - * @param globalInstances a Set of EamGlobalFileInstances to insert into the db. - * @param contentType the Type of the global instances - * + * @param globalInstances a Set of EamGlobalFileInstances to insert into the + * db. + * @param contentType the Type of the global instances + * * @throws EamDbException */ void bulkInsertReferenceTypeEntries(Set globalInstances, EamArtifact.Type contentType) throws EamDbException; @@ -439,7 +443,7 @@ public interface EamDb { /** * Get all reference entries having a given correlation type and value * - * @param aType Type to use for matching + * @param aType Type to use for matching * @param aValue Value to use for matching * * @return List of all global file instances with a type and value @@ -454,6 +458,7 @@ public interface EamDb { * @param newType New type to add. * * @return Type.ID for newType + * * @throws EamDbException */ public int newCorrelationType(EamArtifact.Type newType) throws EamDbException; diff --git a/CentralRepository/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamGlobalFileInstance.java b/CentralRepository/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamGlobalFileInstance.java index 70690cad38..87d974f353 100644 --- a/CentralRepository/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamGlobalFileInstance.java +++ b/CentralRepository/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamGlobalFileInstance.java @@ -32,6 +32,14 @@ public class EamGlobalFileInstance { private TskData.FileKnown knownStatus; private String comment; + public EamGlobalFileInstance( + int globalSetID, + String MD5Hash, + TskData.FileKnown knownStatus, + String comment) { + this(-1, globalSetID, MD5Hash, knownStatus, comment); + } + public EamGlobalFileInstance( int instanceID, int globalSetID, @@ -40,19 +48,12 @@ public class EamGlobalFileInstance { String comment) { this.instanceID = instanceID; this.globalSetID = globalSetID; - this.MD5Hash = MD5Hash; + // Normalize hashes by lower casing + this.MD5Hash = MD5Hash.toLowerCase(); this.knownStatus = knownStatus; this.comment = comment; } - public EamGlobalFileInstance( - int globalSetID, - String MD5Hash, - TskData.FileKnown knownStatus, - String comment) { - this(-1, globalSetID, MD5Hash, knownStatus, comment); - } - @Override public boolean equals(Object otherInstance) { if (this == otherInstance) { @@ -111,7 +112,8 @@ public class EamGlobalFileInstance { * @param MD5Hash the MD5Hash to set */ public void setMD5Hash(String MD5Hash) { - this.MD5Hash = MD5Hash; + // Normalize hashes by lower casing + this.MD5Hash = MD5Hash.toLowerCase(); } /** diff --git a/CentralRepository/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/CaseEventListener.java b/CentralRepository/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/CaseEventListener.java index de04b5aa61..2bacb9234f 100644 --- a/CentralRepository/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/CaseEventListener.java +++ b/CentralRepository/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/CaseEventListener.java @@ -20,6 +20,7 @@ package org.sleuthkit.autopsy.centralrepository.eventlisteners; import java.beans.PropertyChangeEvent; import java.beans.PropertyChangeListener; +import java.util.List; import java.util.logging.Level; import java.util.stream.Collectors; import org.openide.util.NbBundle.Messages; @@ -149,8 +150,8 @@ public class CaseEventListener implements PropertyChangeListener { if (dbManager.getBadTags().contains(tagName.getDisplayName())) { try { - EamArtifact eamArtifact = EamArtifactUtil.fromBlackboardArtifact(bbArtifact, true, dbManager.getCorrelationTypes(), true); - if (null != eamArtifact) { + List convertedArtifacts = EamArtifactUtil.fromBlackboardArtifact(bbArtifact, true, dbManager.getCorrelationTypes(), true); + for (EamArtifact eamArtifact : convertedArtifacts) { eamArtifact.getInstances().get(0).setComment(bbTagAdded.getComment()); Runnable r = new BadFileTagRunner(eamArtifact); // TODO: send r into a thread pool instead diff --git a/CentralRepository/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java b/CentralRepository/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java index 5e7502c1b5..5f1ff75a24 100644 --- a/CentralRepository/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java +++ b/CentralRepository/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java @@ -44,14 +44,15 @@ import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.autopsy.centralrepository.datamodel.EamDb; /** - * Listen for ingest events and update entries in the Central Repository database accordingly + * Listen for ingest events and update entries in the Central Repository + * database accordingly */ public class IngestEventsListener { private static final Logger LOGGER = Logger.getLogger(EamArtifact.class.getName()); final Collection addedCeArtifactTrackerSet = new LinkedHashSet<>(); - + private final PropertyChangeListener pcl1 = new IngestModuleEventListener(); private final PropertyChangeListener pcl2 = new IngestJobEventListener(); @@ -98,9 +99,8 @@ public class IngestEventsListener { try { for (BlackboardArtifact bbArtifact : bbArtifacts) { // eamArtifact will be null OR a EamArtifact containing one EamArtifactInstance. - EamArtifact eamArtifact = EamArtifactUtil.fromBlackboardArtifact(bbArtifact, true, dbManager.getCorrelationTypes(), true); - if (null != eamArtifact) { - + List convertedArtifacts = EamArtifactUtil.fromBlackboardArtifact(bbArtifact, true, dbManager.getCorrelationTypes(), true); + for (EamArtifact eamArtifact : convertedArtifacts) { try { // Only do something with this artifact if it's unique within the job if (addedCeArtifactTrackerSet.add(eamArtifact.toString())) { @@ -108,10 +108,10 @@ public class IngestEventsListener { // query db for artifact instances having this TYPE/VALUE and knownStatus = "Bad". // if gettKnownStatus() is "Unknown" and this artifact instance was marked bad in a previous case, // create TSK_INTERESTING_ARTIFACT_HIT artifact on BB. - List caseDisplayNames = dbManager.getListCasesHavingArtifactInstancesKnownBad(eamArtifact); + List caseDisplayNames = dbManager.getListCasesHavingArtifactInstancesKnownBad(eamArtifact.getCorrelationType(), eamArtifact.getCorrelationValue()); if (!caseDisplayNames.isEmpty()) { postCorrelatedBadArtifactToBlackboard(bbArtifact, - caseDisplayNames); + caseDisplayNames); } eamArtifacts.add(eamArtifact); } @@ -146,7 +146,7 @@ public class IngestEventsListener { // @@@ This isnt' entirely accurate to do here. We could have multiple // ingest jobs at the same time addedCeArtifactTrackerSet.clear(); - + } // DATA_SOURCE_ANALYSIS_COMPLETED break; } @@ -159,7 +159,7 @@ public class IngestEventsListener { try { AbstractFile af = bbArtifact.getSleuthkitCase().getAbstractFileById(bbArtifact.getObjectID()); - + String MODULE_NAME = Bundle.IngestEventsListener_ingestmodule_name(); BlackboardArtifact tifArtifact = af.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_ARTIFACT_HIT); BlackboardAttribute att = new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME, MODULE_NAME, @@ -169,7 +169,7 @@ public class IngestEventsListener { tifArtifact.addAttribute(att); tifArtifact.addAttribute(att2); tifArtifact.addAttribute(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_ASSOCIATED_ARTIFACT, MODULE_NAME, bbArtifact.getArtifactID())); - + try { // index the artifact for keyword search Blackboard blackboard = Case.getCurrentCase().getServices().getBlackboard(); diff --git a/CentralRepository/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java b/CentralRepository/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java index 6cacff6c3e..3eb5b1de37 100644 --- a/CentralRepository/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java +++ b/CentralRepository/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java @@ -107,15 +107,13 @@ class IngestModule implements FileIngestModule { return ProcessResult.OK; } - EamArtifact eamArtifact = new EamArtifact(filesType, md5); - // If unknown to both the hash module and as a globally known artifact in the EAM DB, correlate to other cases if (af.getKnown() == TskData.FileKnown.UNKNOWN) { // query db for artifact instances having this MD5 and knownStatus = "Bad". try { // if af.getKnown() is "UNKNOWN" and this artifact instance was marked bad in a previous case, // create TSK_INTERESTING_FILE artifact on BB. - List caseDisplayNames = dbManager.getListCasesHavingArtifactInstancesKnownBad(eamArtifact); + List caseDisplayNames = dbManager.getListCasesHavingArtifactInstancesKnownBad(filesType, md5); if (!caseDisplayNames.isEmpty()) { postCorrelatedBadFileToBlackboard(af, caseDisplayNames); } @@ -127,7 +125,7 @@ class IngestModule implements FileIngestModule { // Make a TSK_HASHSET_HIT blackboard artifact for global known bad files try { - if (dbManager.isArtifactlKnownBadByReference(eamArtifact)) { + if (dbManager.isArtifactlKnownBadByReference(filesType, md5)) { postCorrelatedHashHitToBlackboard(af); } } catch (EamDbException ex) { @@ -136,6 +134,7 @@ class IngestModule implements FileIngestModule { } try { + EamArtifact eamArtifact = new EamArtifact(filesType, md5); EamArtifactInstance cefi = new EamArtifactInstance( eamCase, eamDataSource, @@ -179,7 +178,7 @@ class IngestModule implements FileIngestModule { LOGGER.log(Level.SEVERE, "Error doing bulk insert of artifacts.", ex); // NON-NLS } try { - Long count = dbManager.getCountArtifactInstancesByCaseDataSource(new EamArtifactInstance(eamCase, eamDataSource)); + Long count = dbManager.getCountArtifactInstancesByCaseDataSource(eamCase.getCaseUUID(), eamDataSource.getDeviceID()); LOGGER.log(Level.INFO, "{0} artifacts in db for case: {1} ds:{2}", new Object[]{count, eamCase.getDisplayName(), eamDataSource.getName()}); // NON-NLS } catch (EamDbException ex) { LOGGER.log(Level.SEVERE, "Error counting artifacts.", ex); // NON-NLS diff --git a/Core/src/org/sleuthkit/autopsy/ingest/FactoryClassNameNormalizer.java b/Core/src/org/sleuthkit/autopsy/ingest/FactoryClassNameNormalizer.java index a03abea38a..8fbeb1d19c 100755 --- a/Core/src/org/sleuthkit/autopsy/ingest/FactoryClassNameNormalizer.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/FactoryClassNameNormalizer.java @@ -1,14 +1,27 @@ /* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. + * Autopsy Forensic Browser + * + * Copyright 2016-2017 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.sleuthkit.autopsy.ingest; import org.sleuthkit.autopsy.coreutils.Logger; /** - * Used to strip python ids on factory class names. + * Used to strip Python IDs on factory class names. */ class FactoryClassNameNormalizer { @@ -17,16 +30,18 @@ class FactoryClassNameNormalizer { static String normalize(String canonicalClassName) { if (isPythonModuleSettingsFile(canonicalClassName)) { - // compiled python modules have variable instance number as a part of their file name. - // This block of code gets rid of that variable instance number and helps maitains constant module name over multiple runs. - String moduleClassName = canonicalClassName.replaceAll("[$][\\d]", ""); //NON-NLS NON-NLS + // Compiled Python modules have variable instance number as a part + // of their file name. This block of code gets rid of that variable + // instance number and helps maitains constant module name over + // multiple runs. + String moduleClassName = canonicalClassName.replaceAll("\\d*$", ""); //NON-NLS NON-NLS return moduleClassName; } return canonicalClassName; } /** - * Determines if the moduleSettingsFilePath is that of a serialized jython + * Determines if the moduleSettingsFilePath is that of a serialized Jython * instance. Serialized Jython instances (settings saved on the disk) * contain "org.python.proxies." in their fileName based on the current * implementation. diff --git a/README_LINUX_OSX.md b/README_LINUX_OSX.md new file mode 100644 index 0000000000..834288b201 --- /dev/null +++ b/README_LINUX_OSX.md @@ -0,0 +1,22 @@ +Most of the Autopsy development occurs to be run on Windows systems, but it is possible to run Autopsy on Linux and OS X. This file contains the instructions for building Autopsy on those platforms and getting it working. + +# Prerequisites + +You need the following: +- X +- Y + +TODO + + +# Building +## Building The Sleuth Kit + +TODO + +## Building Autopsy + +TODO + + + diff --git a/branding/core/core.jar/org/netbeans/core/startup/Bundle.properties b/branding/core/core.jar/org/netbeans/core/startup/Bundle.properties index db8f10ef9f..9fa4b48338 100644 --- a/branding/core/core.jar/org/netbeans/core/startup/Bundle.properties +++ b/branding/core/core.jar/org/netbeans/core/startup/Bundle.properties @@ -1,5 +1,5 @@ #Updated by build script -#Tue, 13 Jun 2017 12:57:15 -0400 +#Thu, 22 Jun 2017 08:50:21 -0400 LBL_splash_window_title=Starting Autopsy SPLASH_HEIGHT=314 SPLASH_WIDTH=538 diff --git a/branding/modules/org-netbeans-core-windows.jar/org/netbeans/core/windows/view/ui/Bundle.properties b/branding/modules/org-netbeans-core-windows.jar/org/netbeans/core/windows/view/ui/Bundle.properties index 501ed875b4..9207917601 100644 --- a/branding/modules/org-netbeans-core-windows.jar/org/netbeans/core/windows/view/ui/Bundle.properties +++ b/branding/modules/org-netbeans-core-windows.jar/org/netbeans/core/windows/view/ui/Bundle.properties @@ -1,4 +1,4 @@ #Updated by build script -#Tue, 13 Jun 2017 12:57:15 -0400 +#Thu, 22 Jun 2017 08:50:21 -0400 CTL_MainWindow_Title=Autopsy 4.4.1 CTL_MainWindow_Title_No_Project=Autopsy 4.4.1