From b8f892ab6adacafd43d73c5e3c775667868e49b4 Mon Sep 17 00:00:00 2001 From: Ann Priestman Date: Mon, 18 Jul 2016 12:34:17 -0400 Subject: [PATCH 1/3] Fix a couple of warnings from our CallableSystemActions --- .../org/sleuthkit/autopsy/actions/OpenOutputFolderAction.java | 2 +- .../sleuthkit/autopsy/diagnostics/PerformancePanelAction.java | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/actions/OpenOutputFolderAction.java b/Core/src/org/sleuthkit/autopsy/actions/OpenOutputFolderAction.java index e5d922af2d..d05baf425d 100644 --- a/Core/src/org/sleuthkit/autopsy/actions/OpenOutputFolderAction.java +++ b/Core/src/org/sleuthkit/autopsy/actions/OpenOutputFolderAction.java @@ -38,7 +38,7 @@ import org.sleuthkit.autopsy.coreutils.Logger; * Action in menu to open the folder containing the output files */ @ActionRegistration( - displayName = "#CTL_OpenOutputFolder", iconInMenu = true) + displayName = "#CTL_OpenOutputFolder", iconInMenu = true, lazy=true) @ActionReference(path = "Menu/Tools", position = 1850, separatorBefore = 1849) @ActionID(id = "org.sleuthkit.autopsy.actions.OpenOutputFolderAction", category = "Help") public final class OpenOutputFolderAction extends CallableSystemAction { diff --git a/Core/src/org/sleuthkit/autopsy/diagnostics/PerformancePanelAction.java b/Core/src/org/sleuthkit/autopsy/diagnostics/PerformancePanelAction.java index 9ecb6644ac..e1af3697a6 100755 --- a/Core/src/org/sleuthkit/autopsy/diagnostics/PerformancePanelAction.java +++ b/Core/src/org/sleuthkit/autopsy/diagnostics/PerformancePanelAction.java @@ -32,7 +32,8 @@ import org.sleuthkit.autopsy.casemodule.Case; id = "org.sleuthkit.autopsy.diagnostics.PerformancePanelAction" ) @ActionRegistration( - displayName = "#CTL_PerformancePanelAction" + displayName = "#CTL_PerformancePanelAction", + lazy=true ) @ActionReference(path = "Menu/Help", position = 1437) public final class PerformancePanelAction extends CallableSystemAction { From 4b45b2481512372d6348174a5cd043fec31e22db Mon Sep 17 00:00:00 2001 From: jmillman Date: Mon, 18 Jul 2016 16:06:14 -0400 Subject: [PATCH 2/3] rearrange code so diffing is easier --- .../keywordsearch/TermComponentQuery.java | 273 +++++++++--------- 1 file changed, 132 insertions(+), 141 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermComponentQuery.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermComponentQuery.java index b00f3d59af..ed2b06fc34 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermComponentQuery.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermComponentQuery.java @@ -102,16 +102,15 @@ final class TermComponentQuery implements KeywordSearchQuery { private static final int MAX_TERMS_RESULTS = 20000; private String escapedQuery; - private boolean isEscaped = false; private final KeywordList keywordList; private final Keyword keyword; + private boolean isEscaped; private final List filters = new ArrayList<>(); TermComponentQuery(KeywordList keywordList, Keyword keyword) { this.keyword = keyword; this.keywordList = keywordList; this.escapedQuery = keyword.getQuery(); - isEscaped = false; } @Override @@ -174,6 +173,137 @@ final class TermComponentQuery implements KeywordSearchQuery { return keyword.getQuery(); } + @Override + public KeywordCachedArtifact writeSingleFileHitsToBlackBoard(String termHit, KeywordHit hit, String snippet, String listName) { + BlackboardArtifact bba; + Collection attributes = new ArrayList<>(); + try { + //if the keyword hit matched the credit card number keyword/regex... + if (keyword.getType() == ATTRIBUTE_TYPE.TSK_ACCOUNT_NUMBER) { + bba = hit.getContent().newArtifact(ARTIFACT_TYPE.TSK_CREDIT_CARD_ACCOUNT); + // make account artifact + //try to match it against the track 1 regex + Matcher matcher = TRACK1_PATTERN.matcher(hit.getSnippet()); + if (matcher.find()) { + parseTrackData(bba, matcher, hit, true); + } + //then try to match it against the track 2 regex + matcher = TRACK2_PATTERN.matcher(hit.getSnippet()); + if (matcher.find()) { + parseTrackData(bba, matcher, hit, false); + } + } else { + //make keyword hit artifact + bba = hit.getContent().newArtifact(ARTIFACT_TYPE.TSK_KEYWORD_HIT); + + //regex match + attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD, MODULE_NAME, termHit)); + //regex keyword + attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD_REGEXP, MODULE_NAME, keyword.getQuery())); + + if (StringUtils.isNotEmpty(listName)) { + attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_SET_NAME, MODULE_NAME, listName)); + } + } + } catch (TskCoreException e) { + LOGGER.log(Level.WARNING, "Error adding bb artifact for keyword hit", e); //NON-NLS + return null; + } + //preview + if (snippet != null) { + attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD_PREVIEW, MODULE_NAME, snippet)); + } + + if (hit.isArtifactHit()) { + attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_ASSOCIATED_ARTIFACT, MODULE_NAME, hit.getArtifact().getArtifactID())); + } + + try { + //TODO: do we still/really need this KeywordCachedArtifact class? + bba.addAttributes(attributes); + KeywordCachedArtifact writeResult = new KeywordCachedArtifact(bba); + writeResult.add(attributes); + return writeResult; + } catch (TskCoreException e) { + LOGGER.log(Level.WARNING, "Error adding bb attributes for terms search artifact", e); //NON-NLS + return null; + } + } + + @Override + public QueryResults performQuery() throws NoOpenCoreException { + /* + * Execute the regex query to get a list of terms that match the regex. + * Note that the field that is being searched is tokenized based on + * whitespace. + */ + //create the query + final SolrQuery q = new SolrQuery(); + q.setRequestHandler(TERMS_HANDLER); + q.setTerms(true); + q.setTermsRegexFlag(CASE_INSENSITIVE); + q.setTermsRegex(escapedQuery); + q.addTermsField(TERMS_SEARCH_FIELD); + q.setTimeAllowed(TERMS_TIMEOUT); + q.setShowDebugInfo(DEBUG); + q.setTermsLimit(MAX_TERMS_RESULTS); + LOGGER.log(Level.INFO, "Query: {0}", q.toString()); //NON-NLS + + //execute the query + List terms = null; + try { + terms = KeywordSearch.getServer().queryTerms(q).getTerms(TERMS_SEARCH_FIELD); + } catch (KeywordSearchModuleException ex) { + LOGGER.log(Level.WARNING, "Error executing the regex terms query: " + keyword.getQuery(), ex); //NON-NLS + //TODO: this is almost certainly wrong and guaranteed to throw a NPE at some point!!!! + } + + /* + * For each term that matched the regex, query for full set of document + * hits for that term. + */ + QueryResults results = new QueryResults(this, keywordList); + int resultSize = 0; + for (Term term : terms) { + final String termStr = KeywordSearchUtil.escapeLuceneQuery(term.getTerm()); + + if (keyword.getType() == ATTRIBUTE_TYPE.TSK_ACCOUNT_NUMBER) { + //If the keyword is a credit card number, pass it through luhn validator + Matcher matcher = CCN_PATTERN.matcher(term.getTerm()); + matcher.find(); + if (false == LUHN_CHECK.isValid(matcher.group("ccn"))) { + continue; //if the hit does not pass the luhn check, skip it. + } + } + + /* + * Note: we can't set filter query on terms query but setting filter + * query on fileResults query will yield the same result + */ + LuceneQuery filesQuery = new LuceneQuery(keywordList, new Keyword(termStr, true)); + filters.forEach(filesQuery::addFilter); + + try { + QueryResults fileQueryResults = filesQuery.performQuery(); + Set filesResults = new HashSet<>(); + for (Keyword key : fileQueryResults.getKeywords()) { //flatten results into a single list + List keyRes = fileQueryResults.getResults(key); + resultSize += keyRes.size(); + filesResults.addAll(keyRes); + } + results.addResult(new Keyword(term.getTerm(), false), new ArrayList<>(filesResults)); + } catch (NoOpenCoreException | RuntimeException e) { + LOGGER.log(Level.WARNING, "Error executing Solr query,", e); //NON-NLS + throw e; + } + } + + //TODO limit how many results we store, not to hit memory limits + LOGGER.log(Level.INFO, "Regex # results: {0}", resultSize); //NON-NLS + + return results; + } + @Override public KeywordList getKeywordList() { return keywordList; @@ -231,143 +361,4 @@ final class TermComponentQuery implements KeywordSearchQuery { artifact.addAttribute(new BlackboardAttribute(SOLR_DOCUMENT_ID_TYPE, MODULE_NAME, hit.getSolrDocumentId())); } } - - @Override - public KeywordCachedArtifact writeSingleFileHitsToBlackBoard(String termHit, KeywordHit hit, String snippet, String listName) { - try { - BlackboardArtifact bba; - - Collection attributes = new ArrayList<>(); - - //if the keyword hit matched the credit card number keyword/regex... - if (keyword.getType() == ATTRIBUTE_TYPE.TSK_ACCOUNT_NUMBER) { - bba = hit.getContent().newArtifact(ARTIFACT_TYPE.TSK_CREDIT_CARD_ACCOUNT); - // make account artifact - //try to match it against the track 1 regex - Matcher matcher = TRACK1_PATTERN.matcher(hit.getSnippet()); - if (matcher.find()) { - parseTrackData(bba, matcher, hit, true); - } - //then try to match it against the track 2 regex - matcher = TRACK2_PATTERN.matcher(hit.getSnippet()); - if (matcher.find()) { - parseTrackData(bba, matcher, hit, false); - } - } else { - //make keyword hit artifact - bba = hit.getContent().newArtifact(ARTIFACT_TYPE.TSK_KEYWORD_HIT); - //TODO: move most of the following into the if branch for non-account keyword hits - //regex match - attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD, MODULE_NAME, termHit)); - - if (StringUtils.isNotEmpty(listName)) { - attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_SET_NAME, MODULE_NAME, listName)); - } - - //regex keyword - attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD_REGEXP, MODULE_NAME, keyword.getQuery())); - } - - //preview - if (snippet != null) { - attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD_PREVIEW, MODULE_NAME, snippet)); - } - - if (hit.isArtifactHit()) { - attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_ASSOCIATED_ARTIFACT, MODULE_NAME, hit.getArtifact().getArtifactID())); - } - - try { - //TODO: do we still/really need this KeywordCachedArtifact class? - bba.addAttributes(attributes); - KeywordCachedArtifact writeResult = new KeywordCachedArtifact(bba); - writeResult.add(attributes); - return writeResult; - } catch (TskCoreException e) { - LOGGER.log(Level.WARNING, "Error adding bb attributes for terms search artifact", e); //NON-NLS - } - } catch (TskCoreException e) { - LOGGER.log(Level.WARNING, "Error adding bb artifact for keyword hit", e); //NON-NLS - } - - return null; - } - - @Override - public QueryResults performQuery() throws NoOpenCoreException { - - /* - * Execute the regex query to get a list of terms that match the regex. - * Note that the field that is being searched is tokenized based on - * whitespace. - */ - final SolrQuery termsQuery = new SolrQuery(); - termsQuery.setShowDebugInfo(DEBUG); - termsQuery.setRequestHandler(TERMS_HANDLER); - termsQuery.setTerms(true); - termsQuery.setTermsRegexFlag(CASE_INSENSITIVE); - termsQuery.setTermsRegex(escapedQuery); - termsQuery.addTermsField(TERMS_SEARCH_FIELD); - termsQuery.setTimeAllowed(TERMS_TIMEOUT); - termsQuery.setTermsLimit(MAX_TERMS_RESULTS); - LOGGER.log(Level.INFO, "Query: {0}", termsQuery.toString()); //NON-NLS - - List terms; - try { - terms = KeywordSearch.getServer().queryTerms(termsQuery).getTerms(TERMS_SEARCH_FIELD); - } catch (KeywordSearchModuleException ex) { - LOGGER.log(Level.WARNING, "Error executing the regex terms query: " + keyword.getQuery(), ex); //NON-NLS - //TODO: this is almost certainly wrong and guaranteed to throw a NPE at some point!!!! - return null; //no need to create result view, just display error dialog - } - - /* - * For each term that matched the regex, query to get the full set of - * document hits for that term. - */ - QueryResults results = new QueryResults(this, keywordList); - int resultSize = 0; - for (Term term : terms) { - String escapedTermString = null; - - if (keyword.getType() == ATTRIBUTE_TYPE.TSK_ACCOUNT_NUMBER) { - //If the keyword is a credit card number, pass it through luhn validator - Matcher matcher = CCN_PATTERN.matcher(term.getTerm()); - matcher.find(); - if (false == LUHN_CHECK.isValid(matcher.group("ccn"))) { - continue; //if the hit does not pass the luhn check, skip it. - } - } - - escapedTermString = KeywordSearchUtil.escapeLuceneQuery(term.getTerm()); - - /* - * Note: we can't set filter query on terms query but setting filter - * query on fileResults query will yield the same result - */ - LuceneQuery filesQuery = new LuceneQuery(keywordList, new Keyword(escapedTermString, true)); - filters.forEach(filesQuery::addFilter); - - try { - QueryResults fileQueryResults = filesQuery.performQuery(); - Set filesResults = new HashSet<>(); - - //flatten results into a single list - for (Keyword key : fileQueryResults.getKeywords()) { - List keyRes = fileQueryResults.getResults(key); - resultSize += keyRes.size(); - filesResults.addAll(keyRes); - } - results.addResult(new Keyword(escapedTermString, false), new ArrayList<>(filesResults)); - } catch (NoOpenCoreException | RuntimeException e) { - LOGGER.log(Level.WARNING, "Error executing Solr query,", e); //NON-NLS - throw e; - } - } - - //TODO limit how many results we store, not to hit memory limits - LOGGER.log(Level.INFO, "Regex # results: {0}", resultSize); //NON-NLS - - return results; - } } From 42c962eaf12355aa8d4edc6168a7fbd395423106 Mon Sep 17 00:00:00 2001 From: jmillman Date: Mon, 18 Jul 2016 17:54:13 -0400 Subject: [PATCH 3/3] elevate log messages from warnings to severe. make seperate methods to parse track 1 and track 2 --- .../keywordsearch/TermComponentQuery.java | 38 ++++++++++++------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermComponentQuery.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermComponentQuery.java index ed2b06fc34..63b701bc2c 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermComponentQuery.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermComponentQuery.java @@ -185,12 +185,12 @@ final class TermComponentQuery implements KeywordSearchQuery { //try to match it against the track 1 regex Matcher matcher = TRACK1_PATTERN.matcher(hit.getSnippet()); if (matcher.find()) { - parseTrackData(bba, matcher, hit, true); + parseTrack1Data(bba, matcher, hit); } //then try to match it against the track 2 regex matcher = TRACK2_PATTERN.matcher(hit.getSnippet()); if (matcher.find()) { - parseTrackData(bba, matcher, hit, false); + parseTrack2Data(bba, matcher, hit); } } else { //make keyword hit artifact @@ -206,7 +206,7 @@ final class TermComponentQuery implements KeywordSearchQuery { } } } catch (TskCoreException e) { - LOGGER.log(Level.WARNING, "Error adding bb artifact for keyword hit", e); //NON-NLS + LOGGER.log(Level.SEVERE, "Error adding bb artifact for keyword hit", e); //NON-NLS return null; } //preview @@ -225,7 +225,7 @@ final class TermComponentQuery implements KeywordSearchQuery { writeResult.add(attributes); return writeResult; } catch (TskCoreException e) { - LOGGER.log(Level.WARNING, "Error adding bb attributes for terms search artifact", e); //NON-NLS + LOGGER.log(Level.SEVERE, "Error adding bb attributes for terms search artifact", e); //NON-NLS return null; } } @@ -254,7 +254,7 @@ final class TermComponentQuery implements KeywordSearchQuery { try { terms = KeywordSearch.getServer().queryTerms(q).getTerms(TERMS_SEARCH_FIELD); } catch (KeywordSearchModuleException ex) { - LOGGER.log(Level.WARNING, "Error executing the regex terms query: " + keyword.getQuery(), ex); //NON-NLS + LOGGER.log(Level.SEVERE, "Error executing the regex terms query: " + keyword.getQuery(), ex); //NON-NLS //TODO: this is almost certainly wrong and guaranteed to throw a NPE at some point!!!! } @@ -333,19 +333,17 @@ final class TermComponentQuery implements KeywordSearchQuery { } /** - * Parse the track 1/2 data from a KeywordHit and add it to the given + * Parse the track 2 data from a KeywordHit and add it to the given * artifact. * * @param artifact * @param matcher * @param hit - * @param tryName True if this mehtod should try to parse the name - * attribute. * * @throws IllegalArgumentException * @throws TskCoreException */ - static private void parseTrackData(BlackboardArtifact artifact, Matcher matcher, KeywordHit hit, boolean tryName) throws IllegalArgumentException, TskCoreException { + static private void parseTrack2Data(BlackboardArtifact artifact, Matcher matcher, KeywordHit hit) throws IllegalArgumentException, TskCoreException { //try to add all the attrributes common to track 1 and 2 addAttributeIfNotAlreadyCaptured(artifact, ATTRIBUTE_TYPE.TSK_ACCOUNT_NUMBER, "accountNumber", matcher); addAttributeIfNotAlreadyCaptured(artifact, ATTRIBUTE_TYPE.TSK_CREDIT_CARD_EXPIRATION, "expiration", matcher); @@ -353,12 +351,26 @@ final class TermComponentQuery implements KeywordSearchQuery { addAttributeIfNotAlreadyCaptured(artifact, ATTRIBUTE_TYPE.TSK_CREDIT_CARD_DISCRETIONARY, "discretionary", matcher); addAttributeIfNotAlreadyCaptured(artifact, ATTRIBUTE_TYPE.TSK_CREDIT_CARD_LRC, "LRC", matcher); - if (tryName) { - //only try to add the name for track 1 - addAttributeIfNotAlreadyCaptured(artifact, ATTRIBUTE_TYPE.TSK_NAME_PERSON, "name", matcher); - } if (artifact.getAttribute(SOLR_DOCUMENT_ID_TYPE) == null) { artifact.addAttribute(new BlackboardAttribute(SOLR_DOCUMENT_ID_TYPE, MODULE_NAME, hit.getSolrDocumentId())); } } + + /** + * Parse the track 1 data from a KeywordHit and add it to the given + * artifact. + * + * @param artifact + * @param matcher + * @param hit + * + * @throws IllegalArgumentException + * @throws TskCoreException + */ + static private void parseTrack1Data(BlackboardArtifact artifact, Matcher matcher, KeywordHit hit) throws IllegalArgumentException, TskCoreException { + // track 1 has all the fields present in track 2 + parseTrack2Data(artifact, matcher, hit); + //plus it also has the account holders name + addAttributeIfNotAlreadyCaptured(artifact, ATTRIBUTE_TYPE.TSK_NAME_PERSON, "name", matcher); + } }