From c882aed613f6a3611b936838aeccf94673c1955c Mon Sep 17 00:00:00 2001 From: Maxwell Koo Date: Fri, 30 Jun 2017 11:14:11 -0400 Subject: [PATCH] Lower-case all correlation values and paths at the data model layer as discussed in meeting, preserve leading + character in phone number normalization --- .../datamodel/EamArtifact.java | 53 ++++++++++--------- .../datamodel/EamArtifactInstance.java | 6 ++- .../datamodel/EamArtifactUtil.java | 12 +++-- .../datamodel/EamGlobalFileInstance.java | 22 ++++---- 4 files changed, 52 insertions(+), 41 deletions(-) diff --git a/CentralRepository/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifact.java b/CentralRepository/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifact.java index f8ad30e051..239afe9df4 100644 --- a/CentralRepository/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifact.java +++ b/CentralRepository/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifact.java @@ -43,7 +43,7 @@ public class EamArtifact implements Serializable { public static final int EMAIL_TYPE_ID = 2; public static final int PHONE_TYPE_ID = 3; public static final int USBID_TYPE_ID = 4; - + /** * Load the default correlation types */ @@ -65,7 +65,8 @@ public class EamArtifact implements Serializable { public EamArtifact(Type correlationType, String correlationValue) { this.ID = ""; this.correlationType = correlationType; - this.correlationValue = correlationValue; + // Lower-case all values to normalize and improve correlation hits, going forward make sure this makes sense for all correlation types + this.correlationValue = correlationValue.toLowerCase(); this.artifactInstances = new ArrayList<>(); } @@ -110,7 +111,8 @@ public class EamArtifact implements Serializable { * @param correlationValue the correlationValue to set */ public void setCorrelationValue(String correlationValue) { - this.correlationValue = correlationValue; + // Lower-case all values to normalize and improve correlation hits, going forward make sure this makes sense for all correlation types + this.correlationValue = correlationValue.toLowerCase(); } /** @@ -161,12 +163,13 @@ public class EamArtifact implements Serializable { private Boolean enabled; /** - * - * @param id Unique ID for this Correlation Type - * @param displayName Name of this type displayed in the UI. - * @param dbTableName Central Repository db table where data of this type is stored - * @param supported Is this Type currently supported - * @param enabled Is this Type currentl enabled. + * + * @param id Unique ID for this Correlation Type + * @param displayName Name of this type displayed in the UI. + * @param dbTableName Central Repository db table where data of this + * type is stored + * @param supported Is this Type currently supported + * @param enabled Is this Type currentl enabled. */ public Type(int id, String displayName, String dbTableName, Boolean supported, Boolean enabled) { this.id = id; @@ -178,13 +181,14 @@ public class EamArtifact implements Serializable { /** * Constructior for custom types where we do not know the Type ID until - * the row has been entered into the correlation_types table - * in the Central Repository. - * - * @param displayName Name of this type displayed in the UI. - * @param dbTableName Central Repository db table where data of this type is stored - * @param supported Is this Type currently supported - * @param enabled Is this Type currentl enabled. + * the row has been entered into the correlation_types table in the + * Central Repository. + * + * @param displayName Name of this type displayed in the UI. + * @param dbTableName Central Repository db table where data of this + * type is stored + * @param supported Is this Type currently supported + * @param enabled Is this Type currentl enabled. */ public Type(String displayName, String dbTableName, Boolean supported, Boolean enabled) { this(-1, displayName, dbTableName, supported, enabled); @@ -308,18 +312,17 @@ public class EamArtifact implements Serializable { } /** - * To support having different database tables for each Type, - * this field provides the prefix/suffix of the table name, - * which allows us to automatically compute the table names - * and indicies. - * - * It is the prefix for the instances tables *_instances. - * It is the suffix for the reference tables reference_*. - * + * To support having different database tables for each Type, this field + * provides the prefix/suffix of the table name, which allows us to + * automatically compute the table names and indicies. + * + * It is the prefix for the instances tables *_instances. It is the + * suffix for the reference tables reference_*. + * * When custom Types are added in the future, they are already supported * by just giving the desired value for the table name for each custom * Type. Possibly having all custom Types use a common table name. - * + * * @return the dbTableName */ public String getDbTableName() { diff --git a/CentralRepository/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactInstance.java b/CentralRepository/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactInstance.java index 27fe3ebee2..cacb2fd368 100644 --- a/CentralRepository/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactInstance.java +++ b/CentralRepository/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactInstance.java @@ -124,7 +124,8 @@ public class EamArtifactInstance implements Serializable { this.ID = ID; this.eamCase = eamCase; this.eamDataSource = eamDataSource; - this.filePath = filePath; + // Lower case paths to normalize paths and improve correlation results, if this causes significant issues on case-sensitive file systems, remove + this.filePath = filePath.toLowerCase(); this.comment = comment; this.knownStatus = knownStatus; this.globalStatus = globalStatus; @@ -204,7 +205,8 @@ public class EamArtifactInstance implements Serializable { * @param filePath the filePath to set */ public void setFilePath(String filePath) { - this.filePath = filePath; + // Lower case paths to normalize paths and improve correlation results, if this causes significant issues on case-sensitive file systems, remove + this.filePath = filePath.toLowerCase(); } /** diff --git a/CentralRepository/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactUtil.java b/CentralRepository/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactUtil.java index 1561df72d2..35dfc4dcea 100644 --- a/CentralRepository/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactUtil.java +++ b/CentralRepository/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactUtil.java @@ -130,8 +130,7 @@ public class EamArtifactUtil { || BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_HISTORY.getTypeID() == artifactTypeID)) { // Lower-case this to normalize domains - value = bbArtifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DOMAIN)).getValueString().toLowerCase(); - + value = bbArtifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DOMAIN)).getValueString(); } else if (aType.getId() == EamArtifact.PHONE_TYPE_ID && (BlackboardArtifact.ARTIFACT_TYPE.TSK_CONTACT.getTypeID() == artifactTypeID || BlackboardArtifact.ARTIFACT_TYPE.TSK_CALLLOG.getTypeID() == artifactTypeID @@ -145,9 +144,14 @@ public class EamArtifactUtil { value = bbArtifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_TO)).getValueString(); } + // Remove all non-numeric symbols to semi-normalize phone numbers, preserving leading "+" character if (value != null) { - // Remove all non-numeric symbols to semi-normalize phone numbers - value = value.replaceAll("\\D", ""); + String newValue = value.replaceAll("\\D", ""); + if (value.startsWith("+")) { + newValue = "+" + newValue; + } + + value = newValue; } } else if (aType.getId() == EamArtifact.USBID_TYPE_ID && BlackboardArtifact.ARTIFACT_TYPE.TSK_DEVICE_ATTACHED.getTypeID() == artifactTypeID) { diff --git a/CentralRepository/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamGlobalFileInstance.java b/CentralRepository/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamGlobalFileInstance.java index 2a1e4ce24d..4765fb3822 100644 --- a/CentralRepository/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamGlobalFileInstance.java +++ b/CentralRepository/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamGlobalFileInstance.java @@ -32,6 +32,14 @@ public class EamGlobalFileInstance { private TskData.FileKnown knownStatus; private String comment; + public EamGlobalFileInstance( + int globalSetID, + String MD5Hash, + TskData.FileKnown knownStatus, + String comment) { + this(-1, globalSetID, MD5Hash, knownStatus, comment); + } + public EamGlobalFileInstance( int instanceID, int globalSetID, @@ -40,19 +48,12 @@ public class EamGlobalFileInstance { String comment) { this.instanceID = instanceID; this.globalSetID = globalSetID; - this.MD5Hash = MD5Hash; + // Normalize hashes by lower casing + this.MD5Hash = MD5Hash.toLowerCase(); this.knownStatus = knownStatus; this.comment = comment; } - public EamGlobalFileInstance( - int globalSetID, - String MD5Hash, - TskData.FileKnown knownStatus, - String comment) { - this(-1, globalSetID, MD5Hash, knownStatus, comment); - } - @Override public boolean equals(Object otherInstance) { if (this == otherInstance) { @@ -111,7 +112,8 @@ public class EamGlobalFileInstance { * @param MD5Hash the MD5Hash to set */ public void setMD5Hash(String MD5Hash) { - this.MD5Hash = MD5Hash; + // Normalize hashes by lower casing + this.MD5Hash = MD5Hash.toLowerCase(); } /**