From 9d7e4b3fdd39da031d857029a5719eee688ced96 Mon Sep 17 00:00:00 2001 From: esaunders Date: Tue, 18 Sep 2018 16:06:12 -0400 Subject: [PATCH 1/5] For backwards compatibility purposes, only lowercase the query string when we know that the Solr schema version has a lowercase content_str field. --- .../autopsy/keywordsearch/RegexQuery.java | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/RegexQuery.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/RegexQuery.java index c76255123c..7664e60f8c 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/RegexQuery.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/RegexQuery.java @@ -28,6 +28,7 @@ import java.util.logging.Level; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.math.NumberUtils; import org.apache.commons.validator.routines.DomainValidator; import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.SolrQuery.SortClause; @@ -225,11 +226,32 @@ final class RegexQuery implements KeywordSearchQuery { boolean skipWildcardPrefix = queryStringContainsWildcardPrefix || getQueryString().startsWith("^"); boolean skipWildcardSuffix = queryStringContainsWildcardSuffix || (getQueryString().endsWith("$") && (!getQueryString().endsWith("\\$"))); + + /** + * The query string to use depends on whether this is a substring or + * regex search. For substring searches, we want to escape the string. + * We may have been asked to perform a substring search on a phone + * number fragment containing special characters (e.g. (555)-) which + * requires us to escape the ( and -. + * + * Additionally, if we are querying a Solr index which is version 2.1 or + * above (where the content_str field is normalized to lowercase) we + * also need to convert the query string to lowercase. For Solr indexes + * that predate version 2.1, we do not lowercase the query string + * thereby allowing queries against existing indexes to behave the same + * way they did in previous versions. + */ + String queryString = (originalKeyword.searchTermIsLiteral() ? getEscapedQueryString() : getQueryString()); + double indexSchemaVersion = NumberUtils.toDouble(solrServer.getIndexInfo().getSchemaVersion()); + if (indexSchemaVersion >= 2.1) { + queryString = queryString.toLowerCase(); + } + solrQuery.setQuery((field == null ? Server.Schema.CONTENT_STR.toString() : field) + ":/" + (skipWildcardPrefix ? "" : ".*") // if the query is for a substring (i.e. literal search term) we want // to escape characters such as ()[]-. - + (originalKeyword.searchTermIsLiteral() ? getEscapedQueryString().toLowerCase() : getQueryString().toLowerCase()) + + queryString + (skipWildcardSuffix ? "" : ".*") + "/"); // Set the fields we want to have returned by the query. From dbe6fd5e4cd8160d26db130a6eb119095c25ca55 Mon Sep 17 00:00:00 2001 From: esaunders Date: Tue, 18 Sep 2018 17:43:58 -0400 Subject: [PATCH 2/5] Reverted changes to email and URL regular expressions to support backwards compatible querying. --- .../sleuthkit/autopsy/keywordsearch/KeywordSearchList.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchList.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchList.java index f13c4ab275..c2536f52a2 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchList.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchList.java @@ -47,8 +47,8 @@ abstract class KeywordSearchList { static final String BOUNDARY_CHARACTERS = "[ \t\r\n\\.\\-\\?\\,\\;\\\\!\\:\\[\\]\\/\\(\\)\\\"\\\'\\>\\{\\}]"; private static final String PHONE_NUMBER_REGEX = BOUNDARY_CHARACTERS + "(\\([0-9]{3}\\)|[0-9]{3})([ \\-\\.])[0-9]{3}([ \\-\\.])[0-9]{4}" + BOUNDARY_CHARACTERS; //NON-NLS private static final String IP_ADDRESS_REGEX = BOUNDARY_CHARACTERS + "(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(1[0-9]{2}|2[0-4][0-9]|25[0-5]|[1-9][0-9]|[0-9])" + BOUNDARY_CHARACTERS; //NON-NLS - private static final String EMAIL_ADDRESS_REGEX = "(\\{?)[a-z0-9%+_\\-]+(\\.[a-z0-9%+_\\-]+)*(\\}?)\\@([a-z0-9]([a-z0-9\\-]*[a-z0-9])?\\.)+[a-z]{2,4}"; //NON-NLS - private static final String URL_REGEX = "(((((h)(t))|(f))(t)(p)(s?)\\:\\/\\/)|(w){3,3}\\.)[a-z0-9\\-\\.]+\\.([a-z]{2,5})(\\:[0-9]+)*(\\/($|[a-z0-9\\.\\,\\;\\?\\'\\\\+&%\\$#\\=~_\\-]+))*"; //NON-NLS + private static final String EMAIL_ADDRESS_REGEX = "(\\{?)[a-zA-Z0-9%+_\\-]+(\\.[a-zA-Z0-9%+_\\-]+)*(\\}?)\\@([a-zA-Z0-9]([a-zA-Z0-9\\-]*[a-zA-Z0-9])?\\.)+[a-zA-Z]{2,4}"; //NON-NLS + private static final String URL_REGEX = "(((((h|H)(t|T))|(f|F))(t|T)(p|P)(s|S?)\\:\\/\\/)|(w|W){3,3}\\.)[a-zA-Z0-9\\-\\.]+\\.([a-zA-Z]{2,5})(\\:[0-9]+)*(\\/($|[a-zA-Z0-9\\.\\,\\;\\?\\'\\\\+&%\\$#\\=~_\\-]+))*"; //NON-NLS /** * 12-19 digits, with possible single spaces or dashes in between, From d0051fff2717d841a4ea269cb174dfb06354a976 Mon Sep 17 00:00:00 2001 From: esaunders Date: Tue, 18 Sep 2018 17:44:59 -0400 Subject: [PATCH 3/5] Lowercase the file name before sending to Solr. --- .../src/org/sleuthkit/autopsy/keywordsearch/Ingester.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Ingester.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Ingester.java index a9cbe0c08b..c855edfed6 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Ingester.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Ingester.java @@ -93,7 +93,7 @@ class Ingester { * file, but the Solr server is probably fine. */ void indexMetaDataOnly(AbstractFile file) throws IngesterException { - indexChunk("", file.getName(), getContentFields(file)); + indexChunk("", file.getName().toLowerCase(), getContentFields(file)); } /** @@ -355,7 +355,7 @@ class Ingester { logger.log(Level.SEVERE, "Could not get data source id to properly index the file " + file.getId(), ex); //NON-NLS params.put(Server.Schema.IMAGE_ID.toString(), Long.toString(-1)); } - params.put(Server.Schema.FILE_NAME.toString(), file.getName()); + params.put(Server.Schema.FILE_NAME.toString(), file.getName().toLowerCase()); return params; } From d52868de86ffb7063e67e469bfba57f4f22c15f7 Mon Sep 17 00:00:00 2001 From: Brian Carrier Date: Fri, 21 Sep 2018 16:01:42 -0400 Subject: [PATCH 4/5] Added docs about right click action --- docs/doxygen/modAdvanced.dox | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/doxygen/modAdvanced.dox b/docs/doxygen/modAdvanced.dox index ef3b546662..769e304ca7 100644 --- a/docs/doxygen/modAdvanced.dox +++ b/docs/doxygen/modAdvanced.dox @@ -37,4 +37,15 @@ Autopsy will generate events as the application runs and modules may want to lis Preventing a user from modifying settings during ingest can be accomplished by listening for Ingest Job Events, and when there is an event setting the enabled status of those settings based off the opposite of IngestManager.getInstance().isIngestRunning(), which returns a boolean. +\subsection mod_dev_adv_right Right Click Options + +You can make a right-click menu that is displayed in the table. To do this, make a class that implements org.sleuthkit.autopsy.corecomponentinterfaces.ContextMenuActionsProvider and has the class as a ServiceProvider. Such as: + + @ServiceProvider(service = ContextMenuActionsProvider.class) + public class SubmitContextMenuActionsProvider implements ContextMenuActionsProvider { + +You can then implement a getActions() method that can decide if it should provide an action and returns the Actions that are relevant. + +For an example, refer to org.sleuthkit.autopsy.modules.fileextmismatch.FileExtMismatchContextMenuActionsProvider (github). + */ From af4f38764bcbe0e524c907afa7b4f292f8e85a75 Mon Sep 17 00:00:00 2001 From: Ann Priestman Date: Mon, 24 Sep 2018 09:44:58 -0400 Subject: [PATCH 5/5] Fix doxygen warnings --- .../datamodel/CorrelationAttributeInstance.java | 4 ++-- .../autopsy/centralrepository/datamodel/EamArtifactUtil.java | 2 -- .../sleuthkit/autopsy/centralrepository/datamodel/EamDb.java | 2 +- .../AllInterCaseCommonAttributeSearcher.java | 2 ++ .../AllIntraCaseCommonAttributeSearcher.java | 1 + .../SingleInterCaseCommonAttributeSearcher.java | 5 ++--- .../SingleIntraCaseCommonAttributeSearcher.java | 1 + .../autopsy/contentviewers/AnnotationsContentViewer.java | 1 - .../modules/embeddedfileextractor/SevenZipExtractor.java | 2 +- .../sleuthkit/autopsy/tabulardatareader/AbstractReader.java | 1 - 10 files changed, 10 insertions(+), 11 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeInstance.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeInstance.java index 261f8c4112..c26134c5b8 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeInstance.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeInstance.java @@ -283,7 +283,7 @@ public class CorrelationAttributeInstance implements Serializable { /** * - * @param id Unique ID for this Correlation Type + * @param typeId Unique ID for this Correlation Type * @param displayName Name of this type displayed in the UI. * @param dbTableName Central repository db table where data of this * type is stored. Must start with a lowercase letter and only contain @@ -387,7 +387,7 @@ public class CorrelationAttributeInstance implements Serializable { } /** - * @param id the typeId to set + * @param typeId the typeId to set */ public void setId(int typeId) { this.typeId = typeId; diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactUtil.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactUtil.java index 095c1a7139..2fb294cdfb 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactUtil.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactUtil.java @@ -55,8 +55,6 @@ public class EamArtifactUtil { * null. * * @param bbArtifact BlackboardArtifact to examine - * @param addInstanceDetails If true, add instance details from bbArtifact - * into the returned structure * @param checkEnabled If true, only create a CorrelationAttribute if it is * enabled * diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDb.java index d7b725109b..c7e385928d 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDb.java @@ -178,7 +178,7 @@ public interface EamDb { /** * Retrieves Case details based on Case ID * - * @param caseID unique identifier for a case + * @param caseId unique identifier for a case * * @return The retrieved case */ diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/AllInterCaseCommonAttributeSearcher.java b/Core/src/org/sleuthkit/autopsy/commonfilesearch/AllInterCaseCommonAttributeSearcher.java index a64d8b0758..6f8c224918 100644 --- a/Core/src/org/sleuthkit/autopsy/commonfilesearch/AllInterCaseCommonAttributeSearcher.java +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/AllInterCaseCommonAttributeSearcher.java @@ -40,6 +40,8 @@ public class AllInterCaseCommonAttributeSearcher extends InterCaseCommonAttribut * broadly categorized as media types * @param filterByDocMimeType match only on files whose mime types can be * broadly categorized as document types + * @param corAttrType attribute type + * @param percentageThreshold omit any matches with frequency above this threshold * * @throws EamDbException */ diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/AllIntraCaseCommonAttributeSearcher.java b/Core/src/org/sleuthkit/autopsy/commonfilesearch/AllIntraCaseCommonAttributeSearcher.java index a02e16cfd8..f7a954aee0 100644 --- a/Core/src/org/sleuthkit/autopsy/commonfilesearch/AllIntraCaseCommonAttributeSearcher.java +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/AllIntraCaseCommonAttributeSearcher.java @@ -39,6 +39,7 @@ final public class AllIntraCaseCommonAttributeSearcher extends IntraCaseCommonAt * broadly categorized as media types * @param filterByDocMimeType match only on files whose mime types can be * broadly categorized as document types + * @param percentageThreshold omit any matches with frequency above this threshold */ public AllIntraCaseCommonAttributeSearcher(Map dataSourceIdMap, boolean filterByMediaMimeType, boolean filterByDocMimeType, int percentageThreshold) { super(dataSourceIdMap, filterByMediaMimeType, filterByDocMimeType, percentageThreshold); diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/SingleInterCaseCommonAttributeSearcher.java b/Core/src/org/sleuthkit/autopsy/commonfilesearch/SingleInterCaseCommonAttributeSearcher.java index de8abfce20..c7c6f9fffd 100644 --- a/Core/src/org/sleuthkit/autopsy/commonfilesearch/SingleInterCaseCommonAttributeSearcher.java +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/SingleInterCaseCommonAttributeSearcher.java @@ -43,6 +43,8 @@ public class SingleInterCaseCommonAttributeSearcher extends InterCaseCommonAttri * @param correlationCaseId * @param filterByMediaMimeType * @param filterByDocMimeType + * @param corAttrType + * @param percentageThreshold * * @throws EamDbException */ @@ -58,9 +60,6 @@ public class SingleInterCaseCommonAttributeSearcher extends InterCaseCommonAttri * Collect metadata required to render the tree table where matches must * occur in the case with the given ID. * - * @param correlationCaseId id of case where matches must occur (no other - * matches will be shown) - * * @return business object needed to populate tree table with results * * @throws TskCoreException diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/SingleIntraCaseCommonAttributeSearcher.java b/Core/src/org/sleuthkit/autopsy/commonfilesearch/SingleIntraCaseCommonAttributeSearcher.java index cfbdf10c59..7e685d4f53 100644 --- a/Core/src/org/sleuthkit/autopsy/commonfilesearch/SingleIntraCaseCommonAttributeSearcher.java +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/SingleIntraCaseCommonAttributeSearcher.java @@ -43,6 +43,7 @@ final public class SingleIntraCaseCommonAttributeSearcher extends IntraCaseCommo * broadly categorized as media types * @param filterByDocMimeType match only on files whose mime types can be * broadly categorized as document types + * @param percentageThreshold omit any matches with frequency above this threshold */ public SingleIntraCaseCommonAttributeSearcher(Long dataSourceId, Map dataSourceIdMap, boolean filterByMediaMimeType, boolean filterByDocMimeType, int percentageThreshold) { super(dataSourceIdMap, filterByMediaMimeType, filterByDocMimeType, percentageThreshold); diff --git a/Core/src/org/sleuthkit/autopsy/contentviewers/AnnotationsContentViewer.java b/Core/src/org/sleuthkit/autopsy/contentviewers/AnnotationsContentViewer.java index 929fe1dc31..d88b5ac2fb 100755 --- a/Core/src/org/sleuthkit/autopsy/contentviewers/AnnotationsContentViewer.java +++ b/Core/src/org/sleuthkit/autopsy/contentviewers/AnnotationsContentViewer.java @@ -304,7 +304,6 @@ public class AnnotationsContentViewer extends javax.swing.JPanel implements Data * @param html The HTML text to add the table to. * @param attributeInstance The attribute instance whose information will be * used to populate the table. - * @param correlationType The correlation data type. */ @NbBundle.Messages({ "AnnotationsContentViewer.centralRepositoryEntryDataLabel.case=Case:", diff --git a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java index 798a05925e..4db0fdc9af 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java +++ b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java @@ -1050,7 +1050,7 @@ class SevenZipExtractor { * Updates the unpackedNode data in the tree after the archive has been * expanded to local disk. * - * @param EOR - ExtractOperationResult + * @param result - ExtractOperationResult * * @throws SevenZipException */ diff --git a/Core/src/org/sleuthkit/autopsy/tabulardatareader/AbstractReader.java b/Core/src/org/sleuthkit/autopsy/tabulardatareader/AbstractReader.java index 69e37cd825..b6413ea6e3 100755 --- a/Core/src/org/sleuthkit/autopsy/tabulardatareader/AbstractReader.java +++ b/Core/src/org/sleuthkit/autopsy/tabulardatareader/AbstractReader.java @@ -52,7 +52,6 @@ public abstract class AbstractReader implements AutoCloseable { * This function is common to all readers. * * @param file AbstractFile from the data source - * @param localDiskPath Local drive path to copy AbstractFile contents * @throws IOException Exception writing file contents * @throws NoCurrentCaseException Current case closed during file copying * @throws TskCoreException Exception finding files from abstract file