From 1c12218cdfcd347346ad224981f055dfc14a84f8 Mon Sep 17 00:00:00 2001 From: millmanorama Date: Fri, 9 Sep 2016 15:58:53 +0200 Subject: [PATCH 1/3] WIP --- .../sleuthkit/autopsy/datamodel/Accounts.java | 38 ++++++++++++++----- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/Accounts.java b/Core/src/org/sleuthkit/autopsy/datamodel/Accounts.java index 9a8d8b0f8e..7459469ae3 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/Accounts.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/Accounts.java @@ -169,7 +169,7 @@ public class Accounts extends Observable implements AutopsyVisitableItem { * based on the state of showRejected. */ private String getRejectedArtifactFilterClause() { - return showRejected ? "" : " AND blackboard_artifacts.review_status_id != " + BlackboardArtifact.ReviewStatus.REJECTED.getID(); //NON-NLS + return showRejected ? " " : " AND blackboard_artifacts.review_status_id != " + BlackboardArtifact.ReviewStatus.REJECTED.getID(); //NON-NLS } /** @@ -1020,6 +1020,7 @@ public class Accounts extends Observable implements AutopsyVisitableItem { @Override protected boolean createKeys(List list) { + String query = "SELECT SUBSTR(blackboard_attributes.value_text,1,8) AS BIN, " //NON-NLS + " COUNT(blackboard_artifacts.artifact_id) AS count " //NON-NLS @@ -1055,23 +1056,40 @@ public class Accounts extends Observable implements AutopsyVisitableItem { */ private class BinResult implements IINInfo { - private final Integer bin; /** * The number of accounts with this BIN */ - private final Long count; + private Long count; + private final IINInfo iinInfo; + private Range iinRange; - private BinResult(Integer bin, Long count) { - this.bin = bin; + private BinResult(int start, int end, Long count, IINInfo iinInfo) { + this.iinRange = Range.closed(start, end); this.count = count; - iinInfo = getIINInfo(bin); + this.iinInfo = iinInfo; + } + public void setIINStart(int IINStart) { + this.IINStart = IINStart; } - public Integer getBIN() { - return bin; + public void setIINEnd(int IINEnd) { + this.IINEnd = IINEnd; } + public void setCount(Long count) { + this.count = count; + } + + int getIINstart() { + return IINStart; + } + + int getIINend() { + return IINEnd; + } + + public Long getCount() { return count; } @@ -1139,13 +1157,15 @@ public class Accounts extends Observable implements AutopsyVisitableItem { @Override protected boolean createKeys(List list) { + private final static RangeMap presentRanges = TreeRangeMap.create(); + String query = "SELECT blackboard_artifacts.artifact_id " //NON-NLS + " FROM blackboard_artifacts " //NON-NLS + " JOIN blackboard_attributes ON blackboard_artifacts.artifact_id = blackboard_attributes.artifact_id " //NON-NLS + " WHERE blackboard_artifacts.artifact_type_id = " + BlackboardArtifact.ARTIFACT_TYPE.TSK_CREDIT_CARD_ACCOUNT.getTypeID() //NON-NLS + " AND blackboard_attributes.attribute_type_id = " + BlackboardAttribute.ATTRIBUTE_TYPE.TSK_ACCOUNT_NUMBER.getTypeID() //NON-NLS - + " AND blackboard_attributes.value_text LIKE \"" + bin.getBIN() + "%\" " //NON-NLS + + " AND blackboard_attributes.value_text BETWEEN " + bin.getIINstart() + " AND " + bin.getIINend() //NON-NLS + getRejectedArtifactFilterClause() + " ORDER BY blackboard_attributes.value_text"; //NON-NLS try (SleuthkitCase.CaseDbQuery results = skCase.executeQuery(query); From 72b3078d71485dff73728bba7dda3a3035d91a6b Mon Sep 17 00:00:00 2001 From: millmanorama Date: Mon, 12 Sep 2016 15:11:47 +0200 Subject: [PATCH 2/3] more WIP to group bins by range in UI and also some bug fixes for when there is more than one valid BIN in a keyword search snippet --- .../sleuthkit/autopsy/datamodel/Accounts.java | 100 +++++++++++------- .../keywordsearch/TermComponentQuery.java | 24 +++-- 2 files changed, 80 insertions(+), 44 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/Accounts.java b/Core/src/org/sleuthkit/autopsy/datamodel/Accounts.java index 7459469ae3..8868045e9c 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/Accounts.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/Accounts.java @@ -43,6 +43,7 @@ import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; import java.util.stream.Stream; +import javax.annotation.Nonnull; import javax.annotation.concurrent.GuardedBy; import javax.swing.AbstractAction; import javax.swing.Action; @@ -933,9 +934,15 @@ public class Accounts extends Observable implements AutopsyVisitableItem { } private void updateDisplayName() { - ArrayList keys = new ArrayList<>(); - accountFactory.createKeys(keys); - setDisplayName(bin.getBIN().toString() + " (" + keys.size() + ")"); //NON-NLS + setDisplayName(getBinRangeString() + " (" + bin.getCount() + ")"); //NON-NLS + } + + private String getBinRangeString() { + if (bin.getIINStart() == bin.getIINEnd()) { + return Integer.toString(bin.getIINStart()); + } else { + return bin.getIINStart() + "-" + StringUtils.difference(bin.getIINStart() + "", bin.getIINEnd() + ""); + } } @Override @@ -977,7 +984,7 @@ public class Accounts extends Observable implements AutopsyVisitableItem { properties.put(new NodeProperty<>(Bundle.Accounts_BINNode_binProperty_displayName(), Bundle.Accounts_BINNode_binProperty_displayName(), Bundle.Accounts_BINNode_noDescription(), - bin.getBIN())); + getBinRangeString())); properties.put(new NodeProperty<>(Bundle.Accounts_BINNode_accountsProperty_displayName(), Bundle.Accounts_BINNode_accountsProperty_displayName(), Bundle.Accounts_BINNode_noDescription(), bin.getCount())); @@ -1020,6 +1027,7 @@ public class Accounts extends Observable implements AutopsyVisitableItem { @Override protected boolean createKeys(List list) { + RangeMap presentRanges = TreeRangeMap.create(); String query = "SELECT SUBSTR(blackboard_attributes.value_text,1,8) AS BIN, " //NON-NLS @@ -1034,9 +1042,27 @@ public class Accounts extends Observable implements AutopsyVisitableItem { try (SleuthkitCase.CaseDbQuery results = skCase.executeQuery(query)) { ResultSet resultSet = results.getResultSet(); while (resultSet.next()) { - list.add(new BinResult(Integer.valueOf(resultSet.getString("BIN")), //NON-NLS - resultSet.getLong("count"))); //NON-NLS + final Integer bin = Integer.valueOf(resultSet.getString("BIN")); + final long count = resultSet.getLong("count"); + IINRange iinRange = (IINRange) getIINInfo(bin); + + BinResult previousResult = presentRanges.get(bin); + + if (previousResult != null) { + presentRanges.remove(Range.closed(previousResult.getIINStart(), previousResult.getIINEnd())); + BinResult merged = new BinResult(previousResult.getCount() + count, previousResult.getIINRange()); + presentRanges.put(Range.closed(merged.getIINStart(), merged.getIINEnd()), merged); + } else if (iinRange == null) { + BinResult merged = new BinResult(count, bin, bin); + if (merged.hasDetails()) { + presentRanges.put(Range.closed(iinRange.getIINstart(), iinRange.getIINend()), merged); + } else { + presentRanges.put(Range.closed(bin, bin), merged); + } + } + } } + presentRanges.asMapOfRanges().values().forEach(list::add); } catch (TskCoreException | SQLException ex) { LOGGER.log(Level.SEVERE, "Error querying for BINs.", ex); //NON-NLS return false; @@ -1059,88 +1085,89 @@ public class Accounts extends Observable implements AutopsyVisitableItem { /** * The number of accounts with this BIN */ - private Long count; + private final Long count; - private final IINInfo iinInfo; - private Range iinRange; + private final IINRange iinRange; + private final int iinEnd; + private final int iinStart; - private BinResult(int start, int end, Long count, IINInfo iinInfo) { - this.iinRange = Range.closed(start, end); + private BinResult(Long count, @Nonnull IINRange iinRange) { this.count = count; - this.iinInfo = iinInfo; - } - public void setIINStart(int IINStart) { - this.IINStart = IINStart; + this.iinRange = iinRange; + iinStart = iinRange.getIINstart(); + iinEnd = iinRange.getIINend(); } - public void setIINEnd(int IINEnd) { - this.IINEnd = IINEnd; - } - - public void setCount(Long count) { + private BinResult(Long count, int start, int end) { this.count = count; + this.iinRange = null; + iinStart = start; + iinEnd = end; } - int getIINstart() { - return IINStart; + int getIINStart() { + return iinStart; } - int getIINend() { - return IINEnd; + int getIINEnd() { + return iinEnd; } - public Long getCount() { return count; } boolean hasDetails() { - return iinInfo != null; + return iinRange != null; } @Override public Optional getNumberLength() { - return iinInfo.getNumberLength(); + return iinRange.getNumberLength(); } @Override public Optional getBankCity() { - return iinInfo.getBankCity(); + return iinRange.getBankCity(); } @Override public Optional getBankName() { - return iinInfo.getBankName(); + return iinRange.getBankName(); } @Override public Optional getBankPhoneNumber() { - return iinInfo.getBankPhoneNumber(); + return iinRange.getBankPhoneNumber(); } @Override public Optional getBankURL() { - return iinInfo.getBankURL(); + return iinRange.getBankURL(); } @Override public Optional getBrand() { - return iinInfo.getBrand(); + return iinRange.getBrand(); } @Override public Optional getCardType() { - return iinInfo.getCardType(); + return iinRange.getCardType(); } @Override public Optional getCountry() { - return iinInfo.getCountry(); + return iinRange.getCountry(); } @Override public Optional getScheme() { - return iinInfo.getScheme(); + return iinRange.getScheme(); + } + + private IINRange getIINRange() { + throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates. } } @@ -1157,7 +1184,6 @@ public class Accounts extends Observable implements AutopsyVisitableItem { @Override protected boolean createKeys(List list) { - private final static RangeMap presentRanges = TreeRangeMap.create(); String query = "SELECT blackboard_artifacts.artifact_id " //NON-NLS @@ -1165,7 +1191,7 @@ public class Accounts extends Observable implements AutopsyVisitableItem { + " JOIN blackboard_attributes ON blackboard_artifacts.artifact_id = blackboard_attributes.artifact_id " //NON-NLS + " WHERE blackboard_artifacts.artifact_type_id = " + BlackboardArtifact.ARTIFACT_TYPE.TSK_CREDIT_CARD_ACCOUNT.getTypeID() //NON-NLS + " AND blackboard_attributes.attribute_type_id = " + BlackboardAttribute.ATTRIBUTE_TYPE.TSK_ACCOUNT_NUMBER.getTypeID() //NON-NLS - + " AND blackboard_attributes.value_text BETWEEN " + bin.getIINstart() + " AND " + bin.getIINend() //NON-NLS + + " AND blackboard_attributes.value_text >= \"" + bin.getIINStart() + "\" AND blackboard_attributes.value_text < \"" + (bin.getIINEnd() + 1) + "\"" //NON-NLS + getRejectedArtifactFilterClause() + " ORDER BY blackboard_attributes.value_text"; //NON-NLS try (SleuthkitCase.CaseDbQuery results = skCase.executeQuery(query); diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermComponentQuery.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermComponentQuery.java index 4ed91c142f..b067186252 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermComponentQuery.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermComponentQuery.java @@ -19,6 +19,7 @@ // package org.sleuthkit.autopsy.keywordsearch; +import com.google.common.base.CharMatcher; import java.util.ArrayList; import java.util.Collection; import java.util.HashSet; @@ -181,21 +182,30 @@ final class TermComponentQuery implements KeywordSearchQuery { @Override public KeywordCachedArtifact writeSingleFileHitsToBlackBoard(String termHit, KeywordHit hit, String snippet, String listName) { BlackboardArtifact newArtifact; + Collection attributes = new ArrayList<>(); try { //if the keyword hit matched the credit card number keyword/regex... if (keyword.getType() == ATTRIBUTE_TYPE.TSK_ACCOUNT_NUMBER) { + termHit = CharMatcher.inRange('0', '9').negate().trimFrom(termHit); newArtifact = hit.getContent().newArtifact(ARTIFACT_TYPE.TSK_CREDIT_CARD_ACCOUNT); // make account artifact //try to match it against the track 1 regex Matcher matcher = TRACK1_PATTERN.matcher(hit.getSnippet()); - if (matcher.find()) { - parseTrack1Data(newArtifact, matcher); + while (matcher.find()) { + if (termHit.equals(matcher.group("accountNumber"))) { + parseTrack1Data(newArtifact, matcher); + break; + } } //then try to match it against the track 2 regex matcher = TRACK2_PATTERN.matcher(hit.getSnippet()); - if (matcher.find()) { - parseTrack2Data(newArtifact, matcher); + while (matcher.find()) { + final String group = matcher.group("accountNumber"); + if (termHit.equals(group)) { + parseTrack2Data(newArtifact, matcher); + break; + } } if (hit.getContent() instanceof AbstractFile) { AbstractFile file = (AbstractFile) hit.getContent(); @@ -329,9 +339,9 @@ final class TermComponentQuery implements KeywordSearchQuery { continue; //if the hit does not pass the luhn check, skip it. } final int iin = Integer.parseInt(ccn.substring(0, 8)); - if (false == Accounts.isIINKnown(iin)) { - continue; - } +// if (false == Accounts.isIINKnown(iin)) { +// continue; +// } } /* From d132957758b587fc9267e1b162cb5bdfddedcc29 Mon Sep 17 00:00:00 2001 From: millmanorama Date: Mon, 12 Sep 2016 15:51:36 +0200 Subject: [PATCH 3/3] group bins by range in tree. --- .../sleuthkit/autopsy/datamodel/Accounts.java | 49 +++++++++---------- .../keywordsearch/TermComponentQuery.java | 6 +-- 2 files changed, 26 insertions(+), 29 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/Accounts.java b/Core/src/org/sleuthkit/autopsy/datamodel/Accounts.java index 8868045e9c..b30c831c29 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/Accounts.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/Accounts.java @@ -45,6 +45,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import javax.annotation.Nonnull; import javax.annotation.concurrent.GuardedBy; +import javax.annotation.concurrent.Immutable; import javax.swing.AbstractAction; import javax.swing.Action; import org.apache.commons.csv.CSVFormat; @@ -298,6 +299,7 @@ public class Accounts extends Observable implements AutopsyVisitableItem { * Details of a range of Issuer/Bank Identifiaction Number(s) (IIN/BIN) used * by a bank. */ + @Immutable static private class IINRange implements IINInfo { private final int IINStart; //start of IIN range, 8 digits @@ -703,6 +705,7 @@ public class Accounts extends Observable implements AutopsyVisitableItem { * DataModel for a child of the ByFileNode. Represents a file(chunk) and its * associated accounts. */ + @Immutable private static class FileWithCCN { private final long objID; @@ -1027,7 +1030,7 @@ public class Accounts extends Observable implements AutopsyVisitableItem { @Override protected boolean createKeys(List list) { - RangeMap presentRanges = TreeRangeMap.create(); + RangeMap ranges = TreeRangeMap.create(); String query = "SELECT SUBSTR(blackboard_attributes.value_text,1,8) AS BIN, " //NON-NLS @@ -1043,26 +1046,23 @@ public class Accounts extends Observable implements AutopsyVisitableItem { ResultSet resultSet = results.getResultSet(); while (resultSet.next()) { final Integer bin = Integer.valueOf(resultSet.getString("BIN")); - final long count = resultSet.getLong("count"); - IINRange iinRange = (IINRange) getIINInfo(bin); + long count = resultSet.getLong("count"); - BinResult previousResult = presentRanges.get(bin); + IINRange iinRange = (IINRange) getIINInfo(bin); + BinResult previousResult = ranges.get(bin); if (previousResult != null) { - presentRanges.remove(Range.closed(previousResult.getIINStart(), previousResult.getIINEnd())); - BinResult merged = new BinResult(previousResult.getCount() + count, previousResult.getIINRange()); - presentRanges.put(Range.closed(merged.getIINStart(), merged.getIINEnd()), merged); - } else if (iinRange == null) { - BinResult merged = new BinResult(count, bin, bin); - if (merged.hasDetails()) { - presentRanges.put(Range.closed(iinRange.getIINstart(), iinRange.getIINend()), merged); - } else { - presentRanges.put(Range.closed(bin, bin), merged); - } - } + ranges.remove(Range.closed(previousResult.getIINStart(), previousResult.getIINEnd())); + count += previousResult.getCount(); + } + + if (iinRange != null) { + ranges.put(Range.closed(iinRange.getIINstart(), iinRange.getIINend()), new BinResult(count, iinRange)); + } else { + ranges.put(Range.closed(bin, bin), new BinResult(count, bin, bin)); } } - presentRanges.asMapOfRanges().values().forEach(list::add); + ranges.asMapOfRanges().values().forEach(list::add); } catch (TskCoreException | SQLException ex) { LOGGER.log(Level.SEVERE, "Error querying for BINs.", ex); //NON-NLS return false; @@ -1080,25 +1080,26 @@ public class Accounts extends Observable implements AutopsyVisitableItem { * Data model item to back the BINNodes in the tree. Has the number of * accounts found with the BIN. */ - private class BinResult implements IINInfo { + @Immutable + static private class BinResult implements IINInfo { /** * The number of accounts with this BIN */ - private final Long count; + private final long count; private final IINRange iinRange; private final int iinEnd; private final int iinStart; - private BinResult(Long count, @Nonnull IINRange iinRange) { + private BinResult(long count, @Nonnull IINRange iinRange) { this.count = count; this.iinRange = iinRange; iinStart = iinRange.getIINstart(); iinEnd = iinRange.getIINend(); } - private BinResult(Long count, int start, int end) { + private BinResult(long count, int start, int end) { this.count = count; this.iinRange = null; iinStart = start; @@ -1113,7 +1114,8 @@ public class Accounts extends Observable implements AutopsyVisitableItem { return iinEnd; } - public Long getCount() { + + public long getCount() { return count; } @@ -1165,10 +1167,6 @@ public class Accounts extends Observable implements AutopsyVisitableItem { public Optional getScheme() { return iinRange.getScheme(); } - - private IINRange getIINRange() { - throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates. - } } /** @@ -1299,7 +1297,6 @@ public class Accounts extends Observable implements AutopsyVisitableItem { RejectAccounts(Collection artifacts) { super(Bundle.RejectAccountsAction_name()); - this.artifacts = artifacts; } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermComponentQuery.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermComponentQuery.java index b067186252..a034794c21 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermComponentQuery.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermComponentQuery.java @@ -339,9 +339,9 @@ final class TermComponentQuery implements KeywordSearchQuery { continue; //if the hit does not pass the luhn check, skip it. } final int iin = Integer.parseInt(ccn.substring(0, 8)); -// if (false == Accounts.isIINKnown(iin)) { -// continue; -// } + if (false == Accounts.isIINKnown(iin)) { + continue; + } } /*