Merge remote-tracking branch 'upstream/credit_card_account_search' into 825-accounts-node

Core/src/org/sleuthkit/autopsy/actions/OpenOutputFolderAction.java

@@ -38,7 +38,7 @@ import org.sleuthkit.autopsy.coreutils.Logger;
  * Action in menu to open the folder containing the output files
  */
 @ActionRegistration(
-        displayName = "#CTL_OpenOutputFolder", iconInMenu = true)
+        displayName = "#CTL_OpenOutputFolder", iconInMenu = true, lazy=true)
 @ActionReference(path = "Menu/Tools", position = 1850, separatorBefore = 1849)
 @ActionID(id = "org.sleuthkit.autopsy.actions.OpenOutputFolderAction", category = "Help")
 public final class OpenOutputFolderAction extends CallableSystemAction {

Core/src/org/sleuthkit/autopsy/diagnostics/PerformancePanelAction.java

@@ -32,7 +32,8 @@ import org.sleuthkit.autopsy.casemodule.Case;
         id = "org.sleuthkit.autopsy.diagnostics.PerformancePanelAction"
 )
 @ActionRegistration(
-        displayName = "#CTL_PerformancePanelAction"
+        displayName = "#CTL_PerformancePanelAction",
+        lazy=true
 )
 @ActionReference(path = "Menu/Help", position = 1437)
 public final class PerformancePanelAction extends CallableSystemAction {

KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermComponentQuery.java

@@ -102,16 +102,15 @@ final class TermComponentQuery implements KeywordSearchQuery {
     private static final int MAX_TERMS_RESULTS = 20000;
 
     private String escapedQuery;
-    private boolean isEscaped = false;
     private final KeywordList keywordList;
     private final Keyword keyword;
+    private boolean isEscaped;
     private final List<KeywordQueryFilter> filters = new ArrayList<>();
 
     TermComponentQuery(KeywordList keywordList, Keyword keyword) {
         this.keyword = keyword;
         this.keywordList = keywordList;
         this.escapedQuery = keyword.getQuery();
-        isEscaped = false;
     }
 
     @Override
@@ -174,6 +173,138 @@ final class TermComponentQuery implements KeywordSearchQuery {
         return keyword.getQuery();
     }
 
+    @Override
+    public KeywordCachedArtifact writeSingleFileHitsToBlackBoard(String termHit, KeywordHit hit, String snippet, String listName) {
+        BlackboardArtifact bba;
+        Collection<BlackboardAttribute> attributes = new ArrayList<>();
+        try {
+            //if the keyword hit matched the  credit card number keyword/regex...
+            if (keyword.getType() == ATTRIBUTE_TYPE.TSK_ACCOUNT_NUMBER) {
+                bba = hit.getContent().newArtifact(ARTIFACT_TYPE.TSK_CREDIT_CARD_ACCOUNT);
+                // make account artifact
+                //try to match it against the track 1 regex
+                Matcher matcher = TRACK1_PATTERN.matcher(hit.getSnippet());
+                if (matcher.find()) {
+                    parseTrack1Data(bba, matcher, hit);
+                }
+                //then try to match it against the track 2 regex
+                matcher = TRACK2_PATTERN.matcher(hit.getSnippet());
+                if (matcher.find()) {
+                    parseTrack2Data(bba, matcher, hit);
+                }
+            } else {
+                //make keyword hit artifact
+                bba = hit.getContent().newArtifact(ARTIFACT_TYPE.TSK_KEYWORD_HIT);
+
+                //regex match
+                attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD, MODULE_NAME, termHit));
+                //regex keyword
+                attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD_REGEXP, MODULE_NAME, keyword.getQuery()));
+
+                if (StringUtils.isNotEmpty(listName)) {
+                    attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_SET_NAME, MODULE_NAME, listName));
+                }
+            }
+        } catch (TskCoreException e) {
+            LOGGER.log(Level.SEVERE, "Error adding bb artifact for keyword hit", e); //NON-NLS
+            return null;
+        }
+        //preview
+        if (snippet != null) {
+            attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD_PREVIEW, MODULE_NAME, snippet));
+        }
+
+        if (hit.isArtifactHit()) {
+            attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_ASSOCIATED_ARTIFACT, MODULE_NAME, hit.getArtifact().getArtifactID()));
+        }
+
+        try {
+            //TODO: do we still/really need this KeywordCachedArtifact class? 
+            bba.addAttributes(attributes);
+            KeywordCachedArtifact writeResult = new KeywordCachedArtifact(bba);
+            writeResult.add(attributes);
+            return writeResult;
+        } catch (TskCoreException e) {
+            LOGGER.log(Level.SEVERE, "Error adding bb attributes for terms search artifact", e); //NON-NLS
+            return null;
+        }
+    }
+
+    @Override
+    public QueryResults performQuery() throws NoOpenCoreException {
+        /*
+         * Execute the regex query to get a list of terms that match the regex.
+         * Note that the field that is being searched is tokenized based on
+         * whitespace.
+         */
+        //create the query
+        final SolrQuery q = new SolrQuery();
+        q.setRequestHandler(TERMS_HANDLER);
+        q.setTerms(true);
+        q.setTermsRegexFlag(CASE_INSENSITIVE);
+        q.setTermsRegex(escapedQuery);
+        q.addTermsField(TERMS_SEARCH_FIELD);
+        q.setTimeAllowed(TERMS_TIMEOUT);
+        q.setShowDebugInfo(DEBUG);
+        q.setTermsLimit(MAX_TERMS_RESULTS);
+        LOGGER.log(Level.INFO, "Query: {0}", q.toString()); //NON-NLS
+
+        //execute the query
+        List<Term> terms = null;
+        try {
+            terms = KeywordSearch.getServer().queryTerms(q).getTerms(TERMS_SEARCH_FIELD);
+        } catch (KeywordSearchModuleException ex) {
+            LOGGER.log(Level.SEVERE, "Error executing the regex terms query: " + keyword.getQuery(), ex); //NON-NLS
+            //TODO: this is almost certainly wrong and guaranteed to throw a NPE at some point!!!!
+        }
+
+        /*
+         * For each term that matched the regex, query for full set of document
+         * hits for that term.
+         */
+        QueryResults results = new QueryResults(this, keywordList);
+        int resultSize = 0;
+        for (Term term : terms) {
+            final String termStr = KeywordSearchUtil.escapeLuceneQuery(term.getTerm());
+
+            if (keyword.getType() == ATTRIBUTE_TYPE.TSK_ACCOUNT_NUMBER) {
+                //If the keyword is a credit card number, pass it through luhn validator
+                Matcher matcher = CCN_PATTERN.matcher(term.getTerm());
+                matcher.find();
+                String ccn = matcher.group("ccn");
+                if (false == LUHN_CHECK.isValid(ccn)) {
+                    continue; //if the hit does not pass the luhn check, skip it.
+                }
+            }
+
+            /*
+             * Note: we can't set filter query on terms query but setting filter
+             * query on fileResults query will yield the same result
+             */
+            LuceneQuery filesQuery = new LuceneQuery(keywordList, new Keyword(termStr, true));
+            filters.forEach(filesQuery::addFilter);
+
+            try {
+                QueryResults fileQueryResults = filesQuery.performQuery();
+                Set<KeywordHit> filesResults = new HashSet<>();
+                for (Keyword key : fileQueryResults.getKeywords()) {                //flatten results into a single list
+                    List<KeywordHit> keyRes = fileQueryResults.getResults(key);
+                    resultSize += keyRes.size();
+                    filesResults.addAll(keyRes);
+                }
+                results.addResult(new Keyword(term.getTerm(), false), new ArrayList<>(filesResults));
+            } catch (NoOpenCoreException | RuntimeException e) {
+                LOGGER.log(Level.WARNING, "Error executing Solr query,", e); //NON-NLS
+                throw e;
+            }
+        }
+
+        //TODO limit how many results we store, not to hit memory limits
+        LOGGER.log(Level.INFO, "Regex # results: {0}", resultSize); //NON-NLS
+
+        return results;
+    }
+
     @Override
     public KeywordList getKeywordList() {
         return keywordList;
@@ -203,19 +334,17 @@ final class TermComponentQuery implements KeywordSearchQuery {
     }
 
     /**
-     * Parse the track 1/2 data from a KeywordHit and add it to the given
+     * Parse the track 2 data from a KeywordHit and add it to the given
      * artifact.
      *
      * @param artifact
      * @param matcher
      * @param hit
-     * @param tryName  True if this mehtod should try to parse the name
-     *                 attribute.
      *
      * @throws IllegalArgumentException
      * @throws TskCoreException
      */
-    static private void parseTrackData(BlackboardArtifact artifact, Matcher matcher, KeywordHit hit, boolean tryName) throws IllegalArgumentException, TskCoreException {
+    static private void parseTrack2Data(BlackboardArtifact artifact, Matcher matcher, KeywordHit hit) throws IllegalArgumentException, TskCoreException {
         //try to add all the attrributes common to track 1 and 2
         addAttributeIfNotAlreadyCaptured(artifact, ATTRIBUTE_TYPE.TSK_ACCOUNT_NUMBER, "accountNumber", matcher);
         addAttributeIfNotAlreadyCaptured(artifact, ATTRIBUTE_TYPE.TSK_CREDIT_CARD_EXPIRATION, "expiration", matcher);
@@ -223,152 +352,26 @@ final class TermComponentQuery implements KeywordSearchQuery {
         addAttributeIfNotAlreadyCaptured(artifact, ATTRIBUTE_TYPE.TSK_CREDIT_CARD_DISCRETIONARY, "discretionary", matcher);
         addAttributeIfNotAlreadyCaptured(artifact, ATTRIBUTE_TYPE.TSK_CREDIT_CARD_LRC, "LRC", matcher);
 
-        if (tryName) {
-            //only try to add the name for track 1
-            addAttributeIfNotAlreadyCaptured(artifact, ATTRIBUTE_TYPE.TSK_NAME_PERSON, "name", matcher);
-        }
         if (artifact.getAttribute(SOLR_DOCUMENT_ID_TYPE) == null) {
             artifact.addAttribute(new BlackboardAttribute(SOLR_DOCUMENT_ID_TYPE, MODULE_NAME, hit.getSolrDocumentId()));
         }
     }
 
-    @Override
+    /**
-    public KeywordCachedArtifact writeSingleFileHitsToBlackBoard(String termHit, KeywordHit hit, String snippet, String listName) {
+     * Parse the track 1 data from a KeywordHit and add it to the given
-        try {
+     * artifact.
-            BlackboardArtifact bba;
+     *
-
+     * @param artifact
-            Collection<BlackboardAttribute> attributes = new ArrayList<>();
+     * @param matcher
-
+     * @param hit
-            //if the keyword hit matched the  credit card number keyword/regex...
+     *
-            if (keyword.getType() == ATTRIBUTE_TYPE.TSK_ACCOUNT_NUMBER) {
+     * @throws IllegalArgumentException
-                bba = hit.getContent().newArtifact(ARTIFACT_TYPE.TSK_CREDIT_CARD_ACCOUNT);
+     * @throws TskCoreException
-                // make account artifact
+     */
-                //try to match it against the track 1 regex
+    static private void parseTrack1Data(BlackboardArtifact artifact, Matcher matcher, KeywordHit hit) throws IllegalArgumentException, TskCoreException {
-                Matcher matcher = TRACK1_PATTERN.matcher(hit.getSnippet());
+        // track 1 has all the fields present in track 2
-                if (matcher.find()) {
+        parseTrack2Data(artifact, matcher, hit);
-                    parseTrackData(bba, matcher, hit, true);
+        //plus it also has the account holders name
-                }
+        addAttributeIfNotAlreadyCaptured(artifact, ATTRIBUTE_TYPE.TSK_NAME_PERSON, "name", matcher);
-                //then try to match it against the track 2 regex
-                matcher = TRACK2_PATTERN.matcher(hit.getSnippet());
-                if (matcher.find()) {
-                    parseTrackData(bba, matcher, hit, false);
-                }
-            } else {
-                //make keyword hit artifact
-                bba = hit.getContent().newArtifact(ARTIFACT_TYPE.TSK_KEYWORD_HIT);
-                //TODO: move most of the following into the if branch for non-account keyword hits
-                //regex match
-                attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD, MODULE_NAME, termHit));
-
-                if (StringUtils.isNotEmpty(listName)) {
-                    attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_SET_NAME, MODULE_NAME, listName));
-                }
-
-                //regex keyword
-                attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD_REGEXP, MODULE_NAME, keyword.getQuery()));
-            }
-
-            //preview
-            if (snippet != null) {
-                attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD_PREVIEW, MODULE_NAME, snippet));
-            }
-
-            if (hit.isArtifactHit()) {
-                attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_ASSOCIATED_ARTIFACT, MODULE_NAME, hit.getArtifact().getArtifactID()));
-            }
-
-            try {
-                //TODO: do we still/really need this KeywordCachedArtifact class? 
-                bba.addAttributes(attributes);
-                KeywordCachedArtifact writeResult = new KeywordCachedArtifact(bba);
-                writeResult.add(attributes);
-                return writeResult;
-            } catch (TskCoreException e) {
-                LOGGER.log(Level.WARNING, "Error adding bb attributes for terms search artifact", e); //NON-NLS
-            }
-        } catch (TskCoreException e) {
-            LOGGER.log(Level.WARNING, "Error adding bb artifact for keyword hit", e); //NON-NLS
-        }
-
-        return null;
-    }
-
-    @Override
-    public QueryResults performQuery() throws NoOpenCoreException {
-
-        /*
-         * Execute the regex query to get a list of terms that match the regex.
-         * Note that the field that is being searched is tokenized based on
-         * whitespace.
-         */
-        final SolrQuery termsQuery = new SolrQuery();
-        termsQuery.setShowDebugInfo(DEBUG);
-        termsQuery.setRequestHandler(TERMS_HANDLER);
-        termsQuery.setTerms(true);
-        termsQuery.setTermsRegexFlag(CASE_INSENSITIVE);
-        termsQuery.setTermsRegex(escapedQuery);
-        termsQuery.addTermsField(TERMS_SEARCH_FIELD);
-        termsQuery.setTimeAllowed(TERMS_TIMEOUT);
-        termsQuery.setTermsLimit(MAX_TERMS_RESULTS);
-        LOGGER.log(Level.INFO, "Query: {0}", termsQuery.toString()); //NON-NLS
-
-        List<Term> terms;
-        try {
-            terms = KeywordSearch.getServer().queryTerms(termsQuery).getTerms(TERMS_SEARCH_FIELD);
-        } catch (KeywordSearchModuleException ex) {
-            LOGGER.log(Level.WARNING, "Error executing the regex terms query: " + keyword.getQuery(), ex); //NON-NLS
-            //TODO: this is almost certainly wrong and guaranteed to throw a NPE at some point!!!!
-            return null;  //no need to create result view, just display error dialog
-        }
-
-        /*
-         * For each term that matched the regex, query to get the full set of
-         * document hits for that term.
-         */
-        QueryResults results = new QueryResults(this, keywordList);
-        int resultSize = 0;
-        for (Term term : terms) {
-            String escapedTermString = null;
-
-            if (keyword.getType() == ATTRIBUTE_TYPE.TSK_ACCOUNT_NUMBER) {
-                //If the keyword is a credit card number, pass it through luhn validator
-                Matcher matcher = CCN_PATTERN.matcher(term.getTerm());
-                matcher.find();
-                String ccn = matcher.group("ccn");
-                if (false == LUHN_CHECK.isValid(ccn)) {
-                    continue; //if the hit does not pass the luhn check, skip it.
-                }
-            }
-
-            escapedTermString = KeywordSearchUtil.escapeLuceneQuery(term.getTerm());
-
-            /*
-             * Note: we can't set filter query on terms query but setting filter
-             * query on fileResults query will yield the same result
-             */
-            LuceneQuery filesQuery = new LuceneQuery(keywordList, new Keyword(escapedTermString, true));
-            filters.forEach(filesQuery::addFilter);
-
-            try {
-                QueryResults fileQueryResults = filesQuery.performQuery();
-                Set<KeywordHit> filesResults = new HashSet<>();
-
-                //flatten results into a single list
-                for (Keyword key : fileQueryResults.getKeywords()) {
-                    List<KeywordHit> keyRes = fileQueryResults.getResults(key);
-                    resultSize += keyRes.size();
-                    filesResults.addAll(keyRes);
-                }
-                results.addResult(new Keyword(escapedTermString, false), new ArrayList<>(filesResults));
-            } catch (NoOpenCoreException | RuntimeException e) {
-                LOGGER.log(Level.WARNING, "Error executing Solr query,", e); //NON-NLS
-                throw e;
-            }
-        }
-
-        //TODO limit how many results we store, not to hit memory limits
-        LOGGER.log(Level.INFO, "Regex # results: {0}", resultSize); //NON-NLS
-
-        return results;
     }
 }