Merge branch 'develop' of https://github.com/sleuthkit/autopsy into 7084-FeedbackWhileLoadingArtifacts

Core/src/org/sleuthkit/autopsy/casemodule/Bundle.properties

@@ -146,13 +146,14 @@ AddImageWizardIngestConfigPanel.CANCEL_BUTTON.text=Cancel
 NewCaseVisualPanel1.CaseFolderOnCDriveError.text=Warning: Path to multi-user case folder is on \"C:\" drive
 NewCaseVisualPanel1.CaseFolderOnInternalDriveWindowsError.text=Warning: Path to case folder is on \"C:\" drive. Case folder is created on the target system
 NewCaseVisualPanel1.CaseFolderOnInternalDriveLinuxError.text=Warning: Path to case folder is on the target system. Create case folder in mounted drive.
+NewCaseVisualPanel1.uncPath.error=Error: UNC paths are not allowed for Single-User cases
 CollaborationMonitor.addingDataSourceStatus.msg={0} adding data source
 CollaborationMonitor.analyzingDataSourceStatus.msg={0} analyzing {1}
 MissingImageDialog.lbWarning.text=
 MissingImageDialog.lbWarning.toolTipText=
 NewCaseVisualPanel1.caseParentDirWarningLabel.text=
-NewCaseVisualPanel1.multiUserCaseRadioButton.text=Multi-user		
+NewCaseVisualPanel1.multiUserCaseRadioButton.text=Multi-User		
-NewCaseVisualPanel1.singleUserCaseRadioButton.text=Single-user
+NewCaseVisualPanel1.singleUserCaseRadioButton.text=Single-User
 NewCaseVisualPanel1.caseTypeLabel.text=Case Type: 
 SingleUserCaseConverter.BadDatabaseFileName=Database file does not exist!
 SingleUserCaseConverter.AlreadyMultiUser=Case is already multi-user!

Core/src/org/sleuthkit/autopsy/casemodule/ImageFilePanel.java

@@ -469,7 +469,7 @@ public class ImageFilePanel extends JPanel implements DocumentListener {
             return false;
         }
 
-        if (!PathValidator.isValidForMultiUserCase(path, Case.getCurrentCase().getCaseType())) {
+        if (!PathValidator.isValidForCaseType(path, Case.getCurrentCase().getCaseType())) {
             errorLabel.setVisible(true);
             errorLabel.setText(Bundle.ImageFilePanel_validatePanel_dataSourceOnCDriveError());
         }

Core/src/org/sleuthkit/autopsy/casemodule/LocalFilesPanel.java

@@ -290,7 +290,7 @@ final class LocalFilesPanel extends javax.swing.JPanel {
             final Case.CaseType currentCaseType = Case.getCurrentCaseThrows().getCaseType();
 
             for (String currentPath : pathsList) {
-                if (!PathValidator.isValidForMultiUserCase(currentPath, currentCaseType)) {
+                if (!PathValidator.isValidForCaseType(currentPath, currentCaseType)) {
                     errorLabel.setVisible(true);
                     errorLabel.setText(Bundle.LocalFilesPanel_pathValidation_dataSourceOnCDriveError());
                     return;

Core/src/org/sleuthkit/autopsy/casemodule/LogicalEvidenceFilePanel.java

@@ -191,7 +191,7 @@ final class LogicalEvidenceFilePanel extends javax.swing.JPanel implements Docum
         }
         // display warning if there is one (but don't disable "next" button)
         try {
-            if (!PathValidator.isValidForMultiUserCase(path, Case.getCurrentCaseThrows().getCaseType())) {
+            if (!PathValidator.isValidForCaseType(path, Case.getCurrentCaseThrows().getCaseType())) {
                 errorLabel.setVisible(true);
                 errorLabel.setText(Bundle.LogicalEvidenceFilePanel_pathValidation_dataSourceOnCDriveError());
                 return false;

Core/src/org/sleuthkit/autopsy/casemodule/NewCaseVisualPanel1.java

@@ -1,7 +1,7 @@
 /*
  * Autopsy Forensic Browser
  *
- * Copyright 2011-2018 Basis Technology Corp.
+ * Copyright 2011-2020 Basis Technology Corp.
  * Contact: carrier <at> sleuthkit <dot> org
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
@@ -28,7 +28,6 @@ import javax.swing.JPanel;
 import javax.swing.event.DocumentEvent;
 import javax.swing.event.DocumentListener;
 import org.sleuthkit.autopsy.casemodule.Case.CaseType;
-import org.sleuthkit.autopsy.core.UserPreferences;
 import org.sleuthkit.autopsy.coreutils.PathValidator;
 import org.sleuthkit.autopsy.coreutils.PlatformUtil;
 
@@ -149,13 +148,22 @@ final class NewCaseVisualPanel1 extends JPanel implements DocumentListener {
     private void validateSettings() {
         /**
          * Check the base case directory for the selected case type and show a
-         * warning if it is a dubious choice.
+         * warning if it is a dubious choice. For single user cases, disable 
+         * the "Next" button if path is invalid.
          */
         caseParentDirWarningLabel.setVisible(false);
         String parentDir = getCaseParentDir();
-        if (!PathValidator.isValidForMultiUserCase(parentDir, getCaseType())) {
+        if (!PathValidator.isValidForCaseType(parentDir, getCaseType())) {
-            caseParentDirWarningLabel.setVisible(true);
+            if (getCaseType() == CaseType.MULTI_USER_CASE) {
-            caseParentDirWarningLabel.setText(NbBundle.getMessage(this.getClass(), "NewCaseVisualPanel1.CaseFolderOnCDriveError.text"));
+                caseParentDirWarningLabel.setVisible(true);
+                caseParentDirWarningLabel.setText(NbBundle.getMessage(this.getClass(), "NewCaseVisualPanel1.CaseFolderOnCDriveError.text"));
+            } else {
+                // disable the "Next" button
+                caseParentDirWarningLabel.setVisible(true);
+                caseParentDirWarningLabel.setText(NbBundle.getMessage(this.getClass(), "NewCaseVisualPanel1.uncPath.error"));                
+                wizPanel.setIsFinish(false);
+                return;
+            }
         }
         
         /**

Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeUtil.java

@@ -49,6 +49,15 @@ public class CorrelationAttributeUtil {
     private static final Logger logger = Logger.getLogger(CorrelationAttributeUtil.class.getName());
     private static final List<String> domainsToSkip = Arrays.asList("localhost", "127.0.0.1");
 
+    // artifact ids that specifically have a TSK_DOMAIN attribute that should be handled by CR
+    private static Set<Integer> DOMAIN_ARTIFACT_TYPE_IDS = new HashSet<>(Arrays.asList(
+            ARTIFACT_TYPE.TSK_WEB_BOOKMARK.getTypeID(),
+            ARTIFACT_TYPE.TSK_WEB_COOKIE.getTypeID(),
+            ARTIFACT_TYPE.TSK_WEB_DOWNLOAD.getTypeID(),
+            ARTIFACT_TYPE.TSK_WEB_HISTORY.getTypeID(),
+            ARTIFACT_TYPE.TSK_WEB_CACHE.getTypeID()
+    ));
+
     /**
      * Gets a string that is expected to be the same string that is stored in
      * the correlation_types table in the central repository as the display name
@@ -68,40 +77,39 @@ public class CorrelationAttributeUtil {
     // Most notably, does not include KEYWORD HIT, CALLLOGS, MESSAGES, CONTACTS
     // TSK_INTERESTING_ARTIFACT_HIT (See JIRA-6129 for more details on the
     // interesting artifact hit).
-    
     // IMPORTANT: This set should be updated for new artifacts types that need to
     // be inserted into the CR.
-    private static final Set<Integer> SOURCE_TYPES_FOR_CR_INSERT = new HashSet<Integer>() {{
+    private static final Set<Integer> SOURCE_TYPES_FOR_CR_INSERT = new HashSet<Integer>() {
-        add(ARTIFACT_TYPE.TSK_WEB_BOOKMARK.getTypeID());
+        {
-        add(ARTIFACT_TYPE.TSK_WEB_COOKIE.getTypeID());
+            addAll(DOMAIN_ARTIFACT_TYPE_IDS);
-        add(ARTIFACT_TYPE.TSK_WEB_DOWNLOAD.getTypeID());
+
-        add(ARTIFACT_TYPE.TSK_WEB_HISTORY.getTypeID());
+            add(ARTIFACT_TYPE.TSK_DEVICE_ATTACHED.getTypeID());
-        add(ARTIFACT_TYPE.TSK_DEVICE_ATTACHED.getTypeID());
+            add(ARTIFACT_TYPE.TSK_WIFI_NETWORK.getTypeID());
-        add(ARTIFACT_TYPE.TSK_WIFI_NETWORK.getTypeID());
+            add(ARTIFACT_TYPE.TSK_WIFI_NETWORK_ADAPTER.getTypeID());
-        add(ARTIFACT_TYPE.TSK_WIFI_NETWORK_ADAPTER.getTypeID());
+            add(ARTIFACT_TYPE.TSK_BLUETOOTH_PAIRING.getTypeID());
-        add(ARTIFACT_TYPE.TSK_BLUETOOTH_PAIRING.getTypeID());
+            add(ARTIFACT_TYPE.TSK_BLUETOOTH_ADAPTER.getTypeID());
-        add(ARTIFACT_TYPE.TSK_BLUETOOTH_ADAPTER.getTypeID());
+            add(ARTIFACT_TYPE.TSK_DEVICE_INFO.getTypeID());
-        add(ARTIFACT_TYPE.TSK_DEVICE_INFO.getTypeID());
+            add(ARTIFACT_TYPE.TSK_SIM_ATTACHED.getTypeID());
-        add(ARTIFACT_TYPE.TSK_SIM_ATTACHED.getTypeID());
+            add(ARTIFACT_TYPE.TSK_WEB_FORM_ADDRESS.getTypeID());
-        add(ARTIFACT_TYPE.TSK_WEB_FORM_ADDRESS.getTypeID());
+            add(ARTIFACT_TYPE.TSK_ACCOUNT.getTypeID());
-        add(ARTIFACT_TYPE.TSK_ACCOUNT.getTypeID());
+        }
-    }};
+    };
 
     /**
      * Makes zero to many correlation attribute instances from the attributes of
      * artifacts that have correlatable data. The intention of this method is to
-     * use the results to save to the CR, not to correlate with them. If you 
+     * use the results to save to the CR, not to correlate with them. If you
-     * want to correlate, please use makeCorrAttrsForCorrelation. An artifact that can
+     * want to correlate, please use makeCorrAttrsForCorrelation. An artifact
-     * have correlatable data != An artifact that should be the source of data
+     * that can have correlatable data != An artifact that should be the source
-     * in the CR, so results may be un-necessarily incomplete.
+     * of data in the CR, so results may be un-necessarily incomplete.
-     * 
+     *
-     * @param artifact              An artifact.
+     * @param artifact An artifact.
      *
      * @return A list, possibly empty, of correlation attribute instances for
-     *          the artifact.
+     * the artifact.
      */
     public static List<CorrelationAttributeInstance> makeCorrAttrsToSave(BlackboardArtifact artifact) {
-        if(SOURCE_TYPES_FOR_CR_INSERT.contains(artifact.getArtifactTypeID())) {
+        if (SOURCE_TYPES_FOR_CR_INSERT.contains(artifact.getArtifactTypeID())) {
             // Restrict the correlation attributes to use for saving.
             // The artifacts which are suitable for saving are a subset of the
             // artifacts that are suitable for correlating.
@@ -114,10 +122,10 @@ public class CorrelationAttributeUtil {
     /**
      * Makes zero to many correlation attribute instances from the attributes of
      * artifacts that have correlatable data. The intention of this method is to
-     * use the results to correlate with, not to save. If you 
+     * use the results to correlate with, not to save. If you want to save,
-     * want to save, please use makeCorrAttrsToSave. An artifact that can
+     * please use makeCorrAttrsToSave. An artifact that can have correlatable
-     * have correlatable data != An artifact that should be the source of data
+     * data != An artifact that should be the source of data in the CR, so
-     * in the CR, so results may be too lenient.
+     * results may be too lenient.
      *
      * IMPORTANT: The correlation attribute instances are NOT added to the
      * central repository by this method.
@@ -130,10 +138,10 @@ public class CorrelationAttributeUtil {
      * whether receiving a null return value is an error or not, plus null
      * checking is easy to forget, while catching exceptions is enforced.
      *
-     * @param artifact              An artifact.
+     * @param artifact An artifact.
      *
      * @return A list, possibly empty, of correlation attribute instances for
-     *          the artifact.
+     * the artifact.
      */
     public static List<CorrelationAttributeInstance> makeCorrAttrsForCorrelation(BlackboardArtifact artifact) {
         List<CorrelationAttributeInstance> correlationAttrs = new ArrayList<>();
@@ -146,13 +154,10 @@ public class CorrelationAttributeUtil {
                     if (setNameAttr != null && CorrelationAttributeUtil.getEmailAddressAttrDisplayName().equals(setNameAttr.getValueString())) {
                         makeCorrAttrFromArtifactAttr(correlationAttrs, sourceArtifact, BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD, CorrelationAttributeInstance.EMAIL_TYPE_ID);
                     }
-                } else if (artifactTypeID == ARTIFACT_TYPE.TSK_WEB_BOOKMARK.getTypeID()
+                } else if (DOMAIN_ARTIFACT_TYPE_IDS.contains(artifactTypeID)) {
-                        || artifactTypeID == ARTIFACT_TYPE.TSK_WEB_COOKIE.getTypeID()
-                        || artifactTypeID == ARTIFACT_TYPE.TSK_WEB_DOWNLOAD.getTypeID()
-                        || artifactTypeID == ARTIFACT_TYPE.TSK_WEB_HISTORY.getTypeID()) {
                     BlackboardAttribute domainAttr = sourceArtifact.getAttribute(new BlackboardAttribute.Type(ATTRIBUTE_TYPE.TSK_DOMAIN));
                     if ((domainAttr != null)
-                            && ! domainsToSkip.contains(domainAttr.getValueString())) {
+                            && !domainsToSkip.contains(domainAttr.getValueString())) {
                         makeCorrAttrFromArtifactAttr(correlationAttrs, sourceArtifact, BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DOMAIN, CorrelationAttributeInstance.DOMAIN_TYPE_ID);
                     }
                 } else if (artifactTypeID == ARTIFACT_TYPE.TSK_DEVICE_ATTACHED.getTypeID()) {
@@ -192,12 +197,10 @@ public class CorrelationAttributeUtil {
         } catch (CorrelationAttributeNormalizationException ex) {
             logger.log(Level.WARNING, String.format("Error normalizing correlation attribute (%s)", artifact), ex); // NON-NLS
             return correlationAttrs;
-        } 
+        } catch (InvalidAccountIDException ex) {
-        catch (InvalidAccountIDException ex) {
             logger.log(Level.WARNING, String.format("Invalid account identifier (artifactID: %d)", artifact.getId())); // NON-NLS
             return correlationAttrs;
-        } 
+        } catch (CentralRepoException ex) {
-        catch (CentralRepoException ex) {
             logger.log(Level.SEVERE, String.format("Error querying central repository (%s)", artifact), ex); // NON-NLS
             return correlationAttrs;
         } catch (TskCoreException ex) {
@@ -259,11 +262,11 @@ public class CorrelationAttributeUtil {
      * @param artifact An artifact.
      *
      * @return The associated artifact if the input artifact is a
-     *         "meta-artifact", otherwise the input artifact.
+     * "meta-artifact", otherwise the input artifact.
      *
      * @throws NoCurrentCaseException If there is no open case.
-     * @throws TskCoreException       If there is an error querying thew case
+     * @throws TskCoreException If there is an error querying thew case
-     *                                database.
+     * database.
      */
     private static BlackboardArtifact getCorrAttrSourceArtifact(BlackboardArtifact artifact) throws NoCurrentCaseException, TskCoreException {
         BlackboardArtifact sourceArtifact = null;
@@ -287,7 +290,7 @@ public class CorrelationAttributeUtil {
      * repository by this method.
      *
      * @param corrAttrInstances A list of correlation attribute instances.
-     * @param acctArtifact      An account artifact.
+     * @param acctArtifact An account artifact.
      *
      * @return The correlation attribute instance.
      */
@@ -296,11 +299,11 @@ public class CorrelationAttributeUtil {
         // Get the account type from the artifact
         BlackboardAttribute accountTypeAttribute = acctArtifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_ACCOUNT_TYPE));
         String accountTypeStr = accountTypeAttribute.getValueString();
-        
+
         // @@TODO Vik-6136: CR currently does not know of custom account types.  
         // Ensure there is a predefined account type for this account.
         Account.Type predefinedAccountType = Account.Type.PREDEFINED_ACCOUNT_TYPES.stream().filter(type -> type.getTypeName().equalsIgnoreCase(accountTypeStr)).findAny().orElse(null);
-             
+
         // do not create any correlation attribute instance for a Device account
         if (Account.Type.DEVICE.getTypeName().equalsIgnoreCase(accountTypeStr) == false && predefinedAccountType != null) {
 
@@ -331,17 +334,14 @@ public class CorrelationAttributeUtil {
      * artifact. The correlation attribute instance is added to an input list.
      *
      * @param corrAttrInstances A list of correlation attribute instances.
-     * @param artifact          An artifact.
+     * @param artifact An artifact.
-     * @param artAttrType       The type of the atrribute of the artifact that
+     * @param artAttrType The type of the atrribute of the artifact that is to
-     *                          is to be made into a correlatin attribute
+     * be made into a correlatin attribute instance.
-     *                          instance.
+     * @param typeId The type ID for the desired correlation attribute instance.
-     * @param typeId            The type ID for the desired correlation
-     *                          attribute instance.
      *
      * @throws CentralRepoException If there is an error querying the central
-     *                              repository.
+     * repository.
-     * @throws TskCoreException     If there is an error querying the case
+     * @throws TskCoreException If there is an error querying the case database.
-     *                              database.
      */
     private static void makeCorrAttrFromArtifactAttr(List<CorrelationAttributeInstance> corrAttrInstances, BlackboardArtifact artifact, ATTRIBUTE_TYPE artAttrType, int typeId) throws CentralRepoException, TskCoreException {
         BlackboardAttribute attribute = artifact.getAttribute(new BlackboardAttribute.Type(artAttrType));
@@ -359,9 +359,9 @@ public class CorrelationAttributeUtil {
     /**
      * Makes a correlation attribute instance of a given type from an artifact.
      *
-     * @param artifact        The artifact.
+     * @param artifact The artifact.
      * @param correlationType the correlation attribute type.
-     * @param value           The correlation attribute value.
+     * @param value The correlation attribute value.
      *
      * TODO (Jira-6088): The methods in this low-level, utility class should
      * throw exceptions instead of logging them. The reason for this is that the
@@ -422,7 +422,7 @@ public class CorrelationAttributeUtil {
      * checking is easy to forget, while catching exceptions is enforced.
      *
      * @return The correlation attribute instance or null, if no such
-     *         correlation attribute instance was found or an error occurred.
+     * correlation attribute instance was found or an error occurred.
      */
     public static CorrelationAttributeInstance getCorrAttrForFile(AbstractFile file) {
 

Core/src/org/sleuthkit/autopsy/coreutils/PathValidator.java

@@ -1,7 +1,7 @@
 /*
  * Autopsy Forensic Browser
  *
- * Copyright 2013-2018 Basis Technology Corp.
+ * Copyright 2013-2020 Basis Technology Corp.
  * Contact: carrier <at> sleuthkit <dot> org
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
@@ -40,7 +40,7 @@ public final class PathValidator {
      *
      * @return - boolean true for valid path, false for invalid path
      */
-    public static boolean isValidForMultiUserCase(String path, Case.CaseType caseType) {
+    public static boolean isValidForCaseType(String path, Case.CaseType caseType) {
 
         if (caseType == Case.CaseType.MULTI_USER_CASE) {
             // check that path is not on "C:" drive
@@ -48,7 +48,10 @@ public final class PathValidator {
                 return false;
             }
         } else {
-            // single user case - no validation needed
+            // check that path is not a UNC path. Solr 8 does not allow UNC paths for indexes.
+            if (UNCPathUtilities.isUNC(path)) {
+                return false;
+            }
         }
 
         return true;
@@ -113,6 +116,19 @@ public final class PathValidator {
      */
     @Deprecated
     public static boolean isValid(String path, Case.CaseType caseType) {
-        return isValidForMultiUserCase(path, caseType);
+        return isValidForCaseType(path, caseType);
+    }
+    
+    /**
+     * Checks if the provided path is valid given the case type.
+     *
+     * @param path     - the path to validate
+     * @param caseType - the type of case which the path is being validated for
+     *
+     * @return - boolean true for valid path, false for invalid path
+     */
+    @Deprecated    
+    public static boolean isValidForMultiUserCase(String path, Case.CaseType caseType) {
+        return isValidForCaseType(path, caseType);
     }
 }

Core/src/org/sleuthkit/autopsy/datasourceprocessors/RawDSInputPanel.java

@@ -292,7 +292,7 @@ final class RawDSInputPanel extends JPanel implements DocumentListener {
         "RawDSInputPanel.noOpenCase.errMsg=Exception while getting open case."})
     private void warnIfPathIsInvalid(String path) {
         try {
-        if (!PathValidator.isValidForMultiUserCase(path, Case.getCurrentCaseThrows().getCaseType())) {
+        if (!PathValidator.isValidForCaseType(path, Case.getCurrentCaseThrows().getCaseType())) {
             errorLabel.setVisible(true);
             errorLabel.setText(Bundle.RawDSInputPanel_error_text());
         }

Experimental/src/org/sleuthkit/autopsy/experimental/volatilityDSP/MemoryDSInputPanel.java

@@ -407,7 +407,7 @@ final class MemoryDSInputPanel extends JPanel implements DocumentListener {
     })
     private void warnIfPathIsInvalid(String path) {
         try {
-            if (!PathValidator.isValidForMultiUserCase(path, Case.getCurrentCaseThrows().getCaseType())) {
+            if (!PathValidator.isValidForCaseType(path, Case.getCurrentCaseThrows().getCaseType())) {
                 errorLabel.setVisible(true);
                 errorLabel.setText(Bundle.MemoryDSInputPanel_errorMsg_dataSourcePathOnCdrive());
             }

KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Index.java

@@ -1,7 +1,7 @@
 /*
  * Autopsy Forensic Browser
  *
- * Copyright 2011-2017 Basis Technology Corp.
+ * Copyright 2011-2020 Basis Technology Corp.
  * Contact: carrier <at> sleuthkit <dot> org
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
@@ -21,7 +21,6 @@ package org.sleuthkit.autopsy.keywordsearch;
 import java.text.SimpleDateFormat;
 import java.util.Date;
 import org.apache.commons.lang.math.NumberUtils;
-import org.sleuthkit.autopsy.coreutils.UNCPathUtilities;
 
 /**
  * This class encapsulates KWS index data.
@@ -33,7 +32,6 @@ final class Index {
     private final String solrVersion;
     private final String indexName;
     private static final String DEFAULT_CORE_NAME = "text_index"; //NON-NLS
-    private final UNCPathUtilities uncPathUtilities = new UNCPathUtilities();
 
     /**
      * Constructs a representation of a text index.
@@ -47,7 +45,7 @@ final class Index {
      *                      need to be generated.
      */
     Index(String indexPath, String solrVersion, String schemaVersion, String coreName, String caseName) {
-        this.indexPath = uncPathUtilities.convertPathToUNC(indexPath);
+        this.indexPath = indexPath;
         this.solrVersion = solrVersion;
         this.schemaVersion = schemaVersion; 
         if (coreName == null || coreName.isEmpty()) {

KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexMetadata.java

@@ -1,7 +1,7 @@
 /*
  * Autopsy Forensic Browser
  *
- * Copyright 2011-2017 Basis Technology Corp.
+ * Copyright 2011-2020 Basis Technology Corp.
  * Contact: carrier <at> sleuthkit <dot> org
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
@@ -39,7 +39,6 @@ import javax.xml.transform.TransformerException;
 import javax.xml.transform.TransformerFactory;
 import javax.xml.transform.dom.DOMSource;
 import javax.xml.transform.stream.StreamResult;
-import org.sleuthkit.autopsy.coreutils.UNCPathUtilities;
 import org.sleuthkit.autopsy.coreutils.XMLUtil;
 import org.w3c.dom.Document;
 import org.w3c.dom.Element;
@@ -61,11 +60,10 @@ class IndexMetadata {
     private final static String SOLR_VERSION_ELEMENT_NAME = "SolrVersion"; //NON-NLS
     private final static String TEXT_INDEX_PATH_ELEMENT_NAME = "TextIndexPath"; //NON-NLS
     private List<Index> indexes = new ArrayList<>();
-    private final UNCPathUtilities uncPathUtilities = new UNCPathUtilities();
     
     IndexMetadata(String caseDirectory, Index index) throws TextIndexMetadataException {
         this.metadataFilePath = Paths.get(caseDirectory, METADATA_FILE_NAME);
-        this.caseDirectoryPath = Paths.get(uncPathUtilities.convertPathToUNC(caseDirectory));
+        this.caseDirectoryPath = Paths.get(caseDirectory);
         this.indexes.add(index);
         writeToFile();
     }
@@ -73,7 +71,7 @@ class IndexMetadata {
     IndexMetadata(String caseDirectory, List<Index> indexes) throws TextIndexMetadataException {
 
         this.metadataFilePath = Paths.get(caseDirectory, METADATA_FILE_NAME);
-        this.caseDirectoryPath = Paths.get(uncPathUtilities.convertPathToUNC(caseDirectory));
+        this.caseDirectoryPath = Paths.get(caseDirectory);
         this.indexes = indexes;
         writeToFile();
     }

KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Ingester.java

@@ -125,7 +125,55 @@ class Ingester {
     private Map<String, String> getContentFields(SleuthkitVisitableItem item) {
         return item.accept(SOLR_FIELDS_VISITOR);
     }
-
+    
+    /**
+     * Read and chunk the source text for indexing in Solr. Also performs
+     * language detection on the input text.
+     *
+     * @param <A>     The type of the Appendix provider that provides additional
+     *                text to append to the final chunk.
+     * @param <T>     A subclass of SleuthkitVisibleItem.
+     * @param Reader  The reader containing extracted text.
+     * @param source  The source from which text will be extracted, chunked, and
+     *                indexed.
+     * @param context The ingest job context that can be used to cancel this
+     *                process.
+     *
+     * @return True if indexing was completed, false otherwise.
+     *
+     * @throws org.sleuthkit.autopsy.keywordsearch.Ingester.IngesterException
+     */
+    // TODO (JIRA-3118): Cancelled text indexing does not propagate cancellation to clients 
+    < T extends SleuthkitVisitableItem> boolean indexText(Reader sourceReader, long sourceID, String sourceName, T source, IngestJobContext context) throws Ingester.IngesterException {
+        boolean doLanguageDetection = true;
+        return indexText(sourceReader, sourceID, sourceName, source, context, doLanguageDetection);
+    }
+    
+    /**
+     * Read and chunk the source text for indexing in Solr. Does NOT perform
+     * language detection on the input strings. Per JIRA-7100, it was determined 
+     * that language detection on extracted strings can take a really long time.
+     *
+     * @param <A>     The type of the Appendix provider that provides additional
+     *                text to append to the final chunk.
+     * @param <T>     A subclass of SleuthkitVisibleItem.
+     * @param Reader  The reader containing extracted text.
+     * @param source  The source from which text will be extracted, chunked, and
+     *                indexed.
+     * @param context The ingest job context that can be used to cancel this
+     *                process.
+     *
+     * @return True if indexing was completed, false otherwise.
+     *
+     * @throws org.sleuthkit.autopsy.keywordsearch.Ingester.IngesterException
+     */
+    // TODO (JIRA-3118): Cancelled text indexing does not propagate cancellation to clients 
+    < T extends SleuthkitVisitableItem> boolean indexStrings(Reader sourceReader, long sourceID, String sourceName, T source, IngestJobContext context) throws Ingester.IngesterException {
+        // Per JIRA-7100, it was determined that language detection on extracted strings can take a really long time.
+        boolean doLanguageDetection = false;
+        return indexText(sourceReader, sourceID, sourceName, source, context, doLanguageDetection);
+    }
+    
     /**
      * Read and chunk the source text for indexing in Solr.
      *
@@ -138,13 +186,14 @@ class Ingester {
      *                  and indexed.
      * @param context   The ingest job context that can be used to cancel this
      *                  process.
+     * @param doLanguageDetection A flag whether to perform language detection on the input text/strings.
      *
      * @return True if indexing was completed, false otherwise.
      *
      * @throws org.sleuthkit.autopsy.keywordsearch.Ingester.IngesterException
      */
     // TODO (JIRA-3118): Cancelled text indexing does not propagate cancellation to clients 
-    < T extends SleuthkitVisitableItem> boolean indexText(Reader sourceReader, long sourceID, String sourceName, T source, IngestJobContext context) throws Ingester.IngesterException {
+    private < T extends SleuthkitVisitableItem> boolean indexText(Reader sourceReader, long sourceID, String sourceName, T source, IngestJobContext context, boolean doLanguageDetection) throws Ingester.IngesterException {
         int numChunks = 0; //unknown until chunking is done
         
         Map<String, String> contentFields = Collections.unmodifiableMap(getContentFields(source));
@@ -162,8 +211,11 @@ class Ingester {
                 String chunkId = Server.getChunkIdString(sourceID, numChunks + 1);
                 fields.put(Server.Schema.ID.toString(), chunkId);
                 fields.put(Server.Schema.CHUNK_SIZE.toString(), String.valueOf(chunk.getBaseChunkLength()));
-                Optional<Language> language = languageSpecificContentIndexingHelper.detectLanguageIfNeeded(chunk);
+                Optional<Language> language = Optional.empty();
-                language.ifPresent(lang -> languageSpecificContentIndexingHelper.updateLanguageSpecificFields(fields, chunk, lang));
+                if (doLanguageDetection) {
+                    language = languageSpecificContentIndexingHelper.detectLanguageIfNeeded(chunk);
+                    language.ifPresent(lang -> languageSpecificContentIndexingHelper.updateLanguageSpecificFields(fields, chunk, lang));
+                }
                 try {
                     //add the chunk text to Solr index
                     indexChunk(chunk.toString(), chunk.geLowerCasedChunk(), sourceName, fields);

KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java

@@ -650,7 +650,7 @@ public final class KeywordSearchIngestModule implements FileIngestModule {
                 }
                 TextExtractor stringsExtractor = TextExtractorFactory.getStringsExtractor(aFile, stringsExtractionContext);
                 Reader extractedTextReader = stringsExtractor.getReader();
-                if (Ingester.getDefault().indexText(extractedTextReader, aFile.getId(), aFile.getName(), aFile, KeywordSearchIngestModule.this.context)) {
+                if (Ingester.getDefault().indexStrings(extractedTextReader, aFile.getId(), aFile.getName(), aFile, KeywordSearchIngestModule.this.context)) {
                     putIngestStatus(jobId, aFile.getId(), IngestStatus.STRINGS_INGESTED);
                     return true;
                 } else {

KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java

@@ -1,7 +1,7 @@
 /*
  * Autopsy Forensic Browser
  *
- * Copyright 2015-2019 Basis Technology Corp.
+ * Copyright 2015-2020 Basis Technology Corp.
  * Contact: carrier <at> sleuthkit <dot> org
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
@@ -125,7 +125,7 @@ public class SolrSearchService implements KeywordSearchService, AutopsyService {
                     // Try the StringsTextExtractor if Tika extractions fails.
                     TextExtractor stringsExtractor = TextExtractorFactory.getStringsExtractor(content, null);
                     Reader stringsExtractedTextReader = stringsExtractor.getReader();
-                    ingester.indexText(stringsExtractedTextReader, content.getId(), content.getName(), content, null);
+                    ingester.indexStrings(stringsExtractedTextReader, content.getId(), content.getName(), content, null);
                 } catch (Ingester.IngesterException | TextExtractor.InitReaderException ex1) {
                     throw new TskCoreException("Error indexing content", ex1);
                 }

RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Chromium.java

@@ -411,6 +411,7 @@ class Chromium extends Extract {
                 }
             }
             postArtifacts(bbartifacts);
+            bbartifacts.clear();
             dbFile.delete();
         }
     }

RecentActivity/src/org/sleuthkit/autopsy/recentactivity/DomainCategoryRunner.java

@@ -23,8 +23,10 @@ import java.net.URL;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.Comparator;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.logging.Level;
 import java.util.Set;
 import java.util.regex.Matcher;
@@ -76,8 +78,85 @@ class DomainCategoryRunner extends Extract {
     private static final String URL_REGEX_STR = String.format("^\\s*%s?%s?%s?", URL_REGEX_SCHEME, URL_REGEX_AUTHORITY, URL_REGEX_PATH);
     private static final Pattern URL_REGEX = Pattern.compile(URL_REGEX_STR);
 
+    private static int DATETIME_ACCESSED_TYPEID = ATTRIBUTE_TYPE.TSK_DATETIME_ACCESSED.getTypeID();
+    private static int URL_TYPEID = ATTRIBUTE_TYPE.TSK_URL.getTypeID();
+
     private static final Logger logger = Logger.getLogger(DomainCategoryRunner.class.getName());
 
+    /**
+     * Get seconds from epoch from the mapping for the attribute type id.
+     *
+     * @param attrMap A mapping of attribute type id to BlackboardAttribute for
+     * an artifact.
+     * @param attrTypeId The attribute type id to fetch.
+     * @return The time in seconds from epoch or 0 if cannot be found.
+     */
+    private static long getTimeOrZero(Map<Integer, BlackboardAttribute> attrMap, int attrTypeId) {
+        if (attrMap == null) {
+            return 0;
+        }
+
+        BlackboardAttribute attr = attrMap.get(attrTypeId);
+        return attr == null ? 0 : attr.getValueLong();
+    }
+
+    /**
+     * Get string for attribute type id or "" if cannot be determined.
+     *
+     * @param attrMap A mapping of attribute type id to BlackboardAttribute for
+     * an artifact.
+     * @param attrTypeId The attribute type id to fetch.
+     * @return The string value or "" if cannot be determined or null.
+     */
+    private static String getStringOrEmpty(Map<Integer, BlackboardAttribute> attrMap, int attrTypeId) {
+        if (attrMap == null) {
+            return "";
+        }
+
+        BlackboardAttribute attr = attrMap.get(attrTypeId);
+        String attrStr = attr == null ? "" : attr.getValueString();
+        return attrStr == null ? "" : attrStr;
+    }
+
+    /**
+     * Comparator for ensuring deterministic order of processing artifacts.
+     */
+    private static final Comparator<BlackboardArtifact> ARTIFACT_COMPARATOR = (a, b) -> {
+        // get attributes in map by type id
+        Map<Integer, BlackboardAttribute> attrMapA = null;
+        Map<Integer, BlackboardAttribute> attrMapB = null;
+
+        try {
+            attrMapA = a.getAttributes()
+                    .stream()
+                    .collect(Collectors.toMap(attr -> attr.getAttributeType().getTypeID(), attr -> attr, (attr1, attr2) -> attr1));
+
+            attrMapB = b.getAttributes()
+                    .stream()
+                    .collect(Collectors.toMap(attr -> attr.getAttributeType().getTypeID(), attr -> attr, (attr1, attr2) -> attr1));
+
+        } catch (TskCoreException ex) {
+            logger.log(Level.WARNING, "There was an error fetching attributes for artifacts", ex);
+            return 0;
+        }
+
+        // sort first on time
+        int timeCompare = Long.compare(getTimeOrZero(attrMapA, DATETIME_ACCESSED_TYPEID), getTimeOrZero(attrMapB, DATETIME_ACCESSED_TYPEID));
+        if (timeCompare != 0) {
+            // negate to push latest times to the front
+            return -timeCompare;
+        }
+
+        // sort next on url
+        int urlCompare = getStringOrEmpty(attrMapA, URL_TYPEID).compareToIgnoreCase(getStringOrEmpty(attrMapB, URL_TYPEID));
+        if (urlCompare != 0) {
+            return urlCompare;
+        }
+
+        // use id as last resort
+        return Long.compare(a.getId(), b.getId());
+    };
+
     private Content dataSource;
     private IngestJobContext context;
     private List<DomainCategorizer> domainProviders = Collections.emptyList();
@@ -272,11 +351,12 @@ class DomainCategoryRunner extends Extract {
         // only one suffix per ingest is captured so this tracks the suffixes seen.
         Set<String> hostSuffixesSeen = new HashSet<>();
         try {
-            Collection<BlackboardArtifact> listArtifacts = currentCase.getSleuthkitCase().getBlackboard().getArtifacts(
+            List<BlackboardArtifact> listArtifacts = currentCase.getSleuthkitCase().getBlackboard().getArtifacts(
                     Arrays.asList(new BlackboardArtifact.Type(ARTIFACT_TYPE.TSK_WEB_HISTORY)),
                     Arrays.asList(dataSource.getId()));
 
             logger.log(Level.INFO, "Processing {0} blackboard artifacts.", listArtifacts.size()); //NON-NLS
+            Collections.sort(listArtifacts, ARTIFACT_COMPARATOR);
 
             for (BlackboardArtifact artifact : listArtifacts) {
                 // make sure we haven't cancelled
@@ -361,11 +441,11 @@ class DomainCategoryRunner extends Extract {
         if (lookupList == null) {
             lookupList = Collections.emptyList();
         }
-        
+
         List<DomainCategorizer> foundProviders = lookupList.stream()
-                        .filter(provider -> provider != null)
+                .filter(provider -> provider != null)
-                        .sorted((a, b) -> a.getClass().getName().compareToIgnoreCase(b.getClass().getName()))
+                .sorted((a, b) -> a.getClass().getName().compareToIgnoreCase(b.getClass().getName()))
-                        .collect(Collectors.toList());
+                .collect(Collectors.toList());
 
         // add the default categorizer last as a last resort
         foundProviders.add(new DefaultDomainCategorizer());

RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java

@@ -247,7 +247,7 @@ class SearchEngineURLQueryAnalyzer extends Extract {
             }
         }
         try { //try to decode the url
-            String decoded = URLDecoder.decode(x, "UTF-8"); //NON-NLS
+            String decoded = URLDecoder.decode(x.replaceAll("%(?![0-9a-fA-F]{2})", "%25"), "UTF-8"); //NON-NLS
             return decoded;
         } catch (UnsupportedEncodingException exception) { //if it fails, return the encoded string
             logger.log(Level.FINE, "Error during URL decoding, returning undecoded value:"

thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/MboxParser.java

@@ -35,6 +35,7 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.logging.Level;
 import org.sleuthkit.autopsy.coreutils.Logger;
+import org.sleuthkit.autopsy.modules.filetypeid.FileTypeDetector;
 import org.apache.james.mime4j.dom.Message;
 import org.apache.james.mime4j.mboxiterator.CharBufferWrapper;
 import org.apache.james.mime4j.mboxiterator.MboxIterator;
@@ -42,6 +43,7 @@ import org.apache.tika.parser.txt.CharsetDetector;
 import org.apache.tika.parser.txt.CharsetMatch;
 import org.apache.commons.validator.routines.EmailValidator;
 import org.openide.util.NbBundle;
+import org.sleuthkit.datamodel.AbstractFile;
 
 /**
  * An Iterator for parsing mbox files. Wraps an instance of MBoxEmailIterator.
@@ -56,12 +58,25 @@ class MboxParser extends MimeJ4MessageParser implements Iterator<EmailMessage> {
         setLocalPath(localPath);
     }
 
-    static boolean isValidMimeTypeMbox(byte[] buffer) {
+    static boolean isValidMimeTypeMbox(byte[] buffer, AbstractFile abstractFile) {
         String mboxHeaderLine = new String(buffer);
         if (mboxHeaderLine.startsWith("From ")) {
-            String[] mboxLineValues = mboxHeaderLine.split(" ");
+            String mimeType = abstractFile.getMIMEType();
-            EmailValidator validator = EmailValidator.getInstance(true, true);
+        
-            return validator.isValid(mboxLineValues[1]);
+            // if it is not present, attempt to use the FileTypeDetector to determine
+            if (mimeType == null || mimeType.isEmpty()) {
+                FileTypeDetector fileTypeDetector = null;
+                try {
+                    fileTypeDetector = new FileTypeDetector();
+                } catch (FileTypeDetector.FileTypeDetectorInitException ex) {
+                    logger.log(Level.WARNING, String.format("Unable to create file type detector for determining MIME type for file %s with id of %d", abstractFile.getName(), abstractFile.getId()));
+                    return false;
+                }
+                mimeType = fileTypeDetector.getMIMEType(abstractFile);
+            } 
+            if (mimeType.equalsIgnoreCase("application/mbox")) {
+                return true;
+            }
         }
         return false; //NON-NLS
     }

thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java

@@ -130,7 +130,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
             if (abstractFile.getSize() > 64) {
                 int byteRead = abstractFile.read(t, 0, 64);
                 if (byteRead > 0) {
-                    isMbox = MboxParser.isValidMimeTypeMbox(t);
+                    isMbox = MboxParser.isValidMimeTypeMbox(t, abstractFile);
                     isEMLFile = EMLParser.isEMLFile(abstractFile, t);
                 }
             }