diff --git a/Core/src/org/sleuthkit/autopsy/actions/OpenOutputFolderAction.java b/Core/src/org/sleuthkit/autopsy/actions/OpenOutputFolderAction.java index e5d922af2d..d05baf425d 100644 --- a/Core/src/org/sleuthkit/autopsy/actions/OpenOutputFolderAction.java +++ b/Core/src/org/sleuthkit/autopsy/actions/OpenOutputFolderAction.java @@ -38,7 +38,7 @@ import org.sleuthkit.autopsy.coreutils.Logger; * Action in menu to open the folder containing the output files */ @ActionRegistration( - displayName = "#CTL_OpenOutputFolder", iconInMenu = true) + displayName = "#CTL_OpenOutputFolder", iconInMenu = true, lazy=true) @ActionReference(path = "Menu/Tools", position = 1850, separatorBefore = 1849) @ActionID(id = "org.sleuthkit.autopsy.actions.OpenOutputFolderAction", category = "Help") public final class OpenOutputFolderAction extends CallableSystemAction { diff --git a/Core/src/org/sleuthkit/autopsy/diagnostics/PerformancePanelAction.java b/Core/src/org/sleuthkit/autopsy/diagnostics/PerformancePanelAction.java index 9ecb6644ac..e1af3697a6 100755 --- a/Core/src/org/sleuthkit/autopsy/diagnostics/PerformancePanelAction.java +++ b/Core/src/org/sleuthkit/autopsy/diagnostics/PerformancePanelAction.java @@ -32,7 +32,8 @@ import org.sleuthkit.autopsy.casemodule.Case; id = "org.sleuthkit.autopsy.diagnostics.PerformancePanelAction" ) @ActionRegistration( - displayName = "#CTL_PerformancePanelAction" + displayName = "#CTL_PerformancePanelAction", + lazy=true ) @ActionReference(path = "Menu/Help", position = 1437) public final class PerformancePanelAction extends CallableSystemAction { diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermComponentQuery.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermComponentQuery.java index a525b90a52..bdc68e7236 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermComponentQuery.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermComponentQuery.java @@ -102,16 +102,15 @@ final class TermComponentQuery implements KeywordSearchQuery { private static final int MAX_TERMS_RESULTS = 20000; private String escapedQuery; - private boolean isEscaped = false; private final KeywordList keywordList; private final Keyword keyword; + private boolean isEscaped; private final List filters = new ArrayList<>(); TermComponentQuery(KeywordList keywordList, Keyword keyword) { this.keyword = keyword; this.keywordList = keywordList; this.escapedQuery = keyword.getQuery(); - isEscaped = false; } @Override @@ -174,6 +173,138 @@ final class TermComponentQuery implements KeywordSearchQuery { return keyword.getQuery(); } + @Override + public KeywordCachedArtifact writeSingleFileHitsToBlackBoard(String termHit, KeywordHit hit, String snippet, String listName) { + BlackboardArtifact bba; + Collection attributes = new ArrayList<>(); + try { + //if the keyword hit matched the credit card number keyword/regex... + if (keyword.getType() == ATTRIBUTE_TYPE.TSK_ACCOUNT_NUMBER) { + bba = hit.getContent().newArtifact(ARTIFACT_TYPE.TSK_CREDIT_CARD_ACCOUNT); + // make account artifact + //try to match it against the track 1 regex + Matcher matcher = TRACK1_PATTERN.matcher(hit.getSnippet()); + if (matcher.find()) { + parseTrack1Data(bba, matcher, hit); + } + //then try to match it against the track 2 regex + matcher = TRACK2_PATTERN.matcher(hit.getSnippet()); + if (matcher.find()) { + parseTrack2Data(bba, matcher, hit); + } + } else { + //make keyword hit artifact + bba = hit.getContent().newArtifact(ARTIFACT_TYPE.TSK_KEYWORD_HIT); + + //regex match + attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD, MODULE_NAME, termHit)); + //regex keyword + attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD_REGEXP, MODULE_NAME, keyword.getQuery())); + + if (StringUtils.isNotEmpty(listName)) { + attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_SET_NAME, MODULE_NAME, listName)); + } + } + } catch (TskCoreException e) { + LOGGER.log(Level.SEVERE, "Error adding bb artifact for keyword hit", e); //NON-NLS + return null; + } + //preview + if (snippet != null) { + attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD_PREVIEW, MODULE_NAME, snippet)); + } + + if (hit.isArtifactHit()) { + attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_ASSOCIATED_ARTIFACT, MODULE_NAME, hit.getArtifact().getArtifactID())); + } + + try { + //TODO: do we still/really need this KeywordCachedArtifact class? + bba.addAttributes(attributes); + KeywordCachedArtifact writeResult = new KeywordCachedArtifact(bba); + writeResult.add(attributes); + return writeResult; + } catch (TskCoreException e) { + LOGGER.log(Level.SEVERE, "Error adding bb attributes for terms search artifact", e); //NON-NLS + return null; + } + } + + @Override + public QueryResults performQuery() throws NoOpenCoreException { + /* + * Execute the regex query to get a list of terms that match the regex. + * Note that the field that is being searched is tokenized based on + * whitespace. + */ + //create the query + final SolrQuery q = new SolrQuery(); + q.setRequestHandler(TERMS_HANDLER); + q.setTerms(true); + q.setTermsRegexFlag(CASE_INSENSITIVE); + q.setTermsRegex(escapedQuery); + q.addTermsField(TERMS_SEARCH_FIELD); + q.setTimeAllowed(TERMS_TIMEOUT); + q.setShowDebugInfo(DEBUG); + q.setTermsLimit(MAX_TERMS_RESULTS); + LOGGER.log(Level.INFO, "Query: {0}", q.toString()); //NON-NLS + + //execute the query + List terms = null; + try { + terms = KeywordSearch.getServer().queryTerms(q).getTerms(TERMS_SEARCH_FIELD); + } catch (KeywordSearchModuleException ex) { + LOGGER.log(Level.SEVERE, "Error executing the regex terms query: " + keyword.getQuery(), ex); //NON-NLS + //TODO: this is almost certainly wrong and guaranteed to throw a NPE at some point!!!! + } + + /* + * For each term that matched the regex, query for full set of document + * hits for that term. + */ + QueryResults results = new QueryResults(this, keywordList); + int resultSize = 0; + for (Term term : terms) { + final String termStr = KeywordSearchUtil.escapeLuceneQuery(term.getTerm()); + + if (keyword.getType() == ATTRIBUTE_TYPE.TSK_ACCOUNT_NUMBER) { + //If the keyword is a credit card number, pass it through luhn validator + Matcher matcher = CCN_PATTERN.matcher(term.getTerm()); + matcher.find(); + String ccn = matcher.group("ccn"); + if (false == LUHN_CHECK.isValid(ccn)) { + continue; //if the hit does not pass the luhn check, skip it. + } + } + + /* + * Note: we can't set filter query on terms query but setting filter + * query on fileResults query will yield the same result + */ + LuceneQuery filesQuery = new LuceneQuery(keywordList, new Keyword(termStr, true)); + filters.forEach(filesQuery::addFilter); + + try { + QueryResults fileQueryResults = filesQuery.performQuery(); + Set filesResults = new HashSet<>(); + for (Keyword key : fileQueryResults.getKeywords()) { //flatten results into a single list + List keyRes = fileQueryResults.getResults(key); + resultSize += keyRes.size(); + filesResults.addAll(keyRes); + } + results.addResult(new Keyword(term.getTerm(), false), new ArrayList<>(filesResults)); + } catch (NoOpenCoreException | RuntimeException e) { + LOGGER.log(Level.WARNING, "Error executing Solr query,", e); //NON-NLS + throw e; + } + } + + //TODO limit how many results we store, not to hit memory limits + LOGGER.log(Level.INFO, "Regex # results: {0}", resultSize); //NON-NLS + + return results; + } + @Override public KeywordList getKeywordList() { return keywordList; @@ -203,19 +334,17 @@ final class TermComponentQuery implements KeywordSearchQuery { } /** - * Parse the track 1/2 data from a KeywordHit and add it to the given + * Parse the track 2 data from a KeywordHit and add it to the given * artifact. * * @param artifact * @param matcher * @param hit - * @param tryName True if this mehtod should try to parse the name - * attribute. * * @throws IllegalArgumentException * @throws TskCoreException */ - static private void parseTrackData(BlackboardArtifact artifact, Matcher matcher, KeywordHit hit, boolean tryName) throws IllegalArgumentException, TskCoreException { + static private void parseTrack2Data(BlackboardArtifact artifact, Matcher matcher, KeywordHit hit) throws IllegalArgumentException, TskCoreException { //try to add all the attrributes common to track 1 and 2 addAttributeIfNotAlreadyCaptured(artifact, ATTRIBUTE_TYPE.TSK_ACCOUNT_NUMBER, "accountNumber", matcher); addAttributeIfNotAlreadyCaptured(artifact, ATTRIBUTE_TYPE.TSK_CREDIT_CARD_EXPIRATION, "expiration", matcher); @@ -223,152 +352,26 @@ final class TermComponentQuery implements KeywordSearchQuery { addAttributeIfNotAlreadyCaptured(artifact, ATTRIBUTE_TYPE.TSK_CREDIT_CARD_DISCRETIONARY, "discretionary", matcher); addAttributeIfNotAlreadyCaptured(artifact, ATTRIBUTE_TYPE.TSK_CREDIT_CARD_LRC, "LRC", matcher); - if (tryName) { - //only try to add the name for track 1 - addAttributeIfNotAlreadyCaptured(artifact, ATTRIBUTE_TYPE.TSK_NAME_PERSON, "name", matcher); - } if (artifact.getAttribute(SOLR_DOCUMENT_ID_TYPE) == null) { artifact.addAttribute(new BlackboardAttribute(SOLR_DOCUMENT_ID_TYPE, MODULE_NAME, hit.getSolrDocumentId())); } } - @Override - public KeywordCachedArtifact writeSingleFileHitsToBlackBoard(String termHit, KeywordHit hit, String snippet, String listName) { - try { - BlackboardArtifact bba; - - Collection attributes = new ArrayList<>(); - - //if the keyword hit matched the credit card number keyword/regex... - if (keyword.getType() == ATTRIBUTE_TYPE.TSK_ACCOUNT_NUMBER) { - bba = hit.getContent().newArtifact(ARTIFACT_TYPE.TSK_CREDIT_CARD_ACCOUNT); - // make account artifact - //try to match it against the track 1 regex - Matcher matcher = TRACK1_PATTERN.matcher(hit.getSnippet()); - if (matcher.find()) { - parseTrackData(bba, matcher, hit, true); - } - //then try to match it against the track 2 regex - matcher = TRACK2_PATTERN.matcher(hit.getSnippet()); - if (matcher.find()) { - parseTrackData(bba, matcher, hit, false); - } - } else { - //make keyword hit artifact - bba = hit.getContent().newArtifact(ARTIFACT_TYPE.TSK_KEYWORD_HIT); - //TODO: move most of the following into the if branch for non-account keyword hits - //regex match - attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD, MODULE_NAME, termHit)); - - if (StringUtils.isNotEmpty(listName)) { - attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_SET_NAME, MODULE_NAME, listName)); - } - - //regex keyword - attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD_REGEXP, MODULE_NAME, keyword.getQuery())); - } - - //preview - if (snippet != null) { - attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD_PREVIEW, MODULE_NAME, snippet)); - } - - if (hit.isArtifactHit()) { - attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_ASSOCIATED_ARTIFACT, MODULE_NAME, hit.getArtifact().getArtifactID())); - } - - try { - //TODO: do we still/really need this KeywordCachedArtifact class? - bba.addAttributes(attributes); - KeywordCachedArtifact writeResult = new KeywordCachedArtifact(bba); - writeResult.add(attributes); - return writeResult; - } catch (TskCoreException e) { - LOGGER.log(Level.WARNING, "Error adding bb attributes for terms search artifact", e); //NON-NLS - } - } catch (TskCoreException e) { - LOGGER.log(Level.WARNING, "Error adding bb artifact for keyword hit", e); //NON-NLS - } - - return null; - } - - @Override - public QueryResults performQuery() throws NoOpenCoreException { - - /* - * Execute the regex query to get a list of terms that match the regex. - * Note that the field that is being searched is tokenized based on - * whitespace. - */ - final SolrQuery termsQuery = new SolrQuery(); - termsQuery.setShowDebugInfo(DEBUG); - termsQuery.setRequestHandler(TERMS_HANDLER); - termsQuery.setTerms(true); - termsQuery.setTermsRegexFlag(CASE_INSENSITIVE); - termsQuery.setTermsRegex(escapedQuery); - termsQuery.addTermsField(TERMS_SEARCH_FIELD); - termsQuery.setTimeAllowed(TERMS_TIMEOUT); - termsQuery.setTermsLimit(MAX_TERMS_RESULTS); - LOGGER.log(Level.INFO, "Query: {0}", termsQuery.toString()); //NON-NLS - - List terms; - try { - terms = KeywordSearch.getServer().queryTerms(termsQuery).getTerms(TERMS_SEARCH_FIELD); - } catch (KeywordSearchModuleException ex) { - LOGGER.log(Level.WARNING, "Error executing the regex terms query: " + keyword.getQuery(), ex); //NON-NLS - //TODO: this is almost certainly wrong and guaranteed to throw a NPE at some point!!!! - return null; //no need to create result view, just display error dialog - } - - /* - * For each term that matched the regex, query to get the full set of - * document hits for that term. - */ - QueryResults results = new QueryResults(this, keywordList); - int resultSize = 0; - for (Term term : terms) { - String escapedTermString = null; - - if (keyword.getType() == ATTRIBUTE_TYPE.TSK_ACCOUNT_NUMBER) { - //If the keyword is a credit card number, pass it through luhn validator - Matcher matcher = CCN_PATTERN.matcher(term.getTerm()); - matcher.find(); - String ccn = matcher.group("ccn"); - if (false == LUHN_CHECK.isValid(ccn)) { - continue; //if the hit does not pass the luhn check, skip it. - } - } - - escapedTermString = KeywordSearchUtil.escapeLuceneQuery(term.getTerm()); - - /* - * Note: we can't set filter query on terms query but setting filter - * query on fileResults query will yield the same result - */ - LuceneQuery filesQuery = new LuceneQuery(keywordList, new Keyword(escapedTermString, true)); - filters.forEach(filesQuery::addFilter); - - try { - QueryResults fileQueryResults = filesQuery.performQuery(); - Set filesResults = new HashSet<>(); - - //flatten results into a single list - for (Keyword key : fileQueryResults.getKeywords()) { - List keyRes = fileQueryResults.getResults(key); - resultSize += keyRes.size(); - filesResults.addAll(keyRes); - } - results.addResult(new Keyword(escapedTermString, false), new ArrayList<>(filesResults)); - } catch (NoOpenCoreException | RuntimeException e) { - LOGGER.log(Level.WARNING, "Error executing Solr query,", e); //NON-NLS - throw e; - } - } - - //TODO limit how many results we store, not to hit memory limits - LOGGER.log(Level.INFO, "Regex # results: {0}", resultSize); //NON-NLS - - return results; + /** + * Parse the track 1 data from a KeywordHit and add it to the given + * artifact. + * + * @param artifact + * @param matcher + * @param hit + * + * @throws IllegalArgumentException + * @throws TskCoreException + */ + static private void parseTrack1Data(BlackboardArtifact artifact, Matcher matcher, KeywordHit hit) throws IllegalArgumentException, TskCoreException { + // track 1 has all the fields present in track 2 + parseTrack2Data(artifact, matcher, hit); + //plus it also has the account holders name + addAttributeIfNotAlreadyCaptured(artifact, ATTRIBUTE_TYPE.TSK_NAME_PERSON, "name", matcher); } }