diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/Bundle.properties b/Core/src/org/sleuthkit/autopsy/casemodule/Bundle.properties index 987323a52b..9019887a68 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/casemodule/Bundle.properties @@ -146,13 +146,14 @@ AddImageWizardIngestConfigPanel.CANCEL_BUTTON.text=Cancel NewCaseVisualPanel1.CaseFolderOnCDriveError.text=Warning: Path to multi-user case folder is on \"C:\" drive NewCaseVisualPanel1.CaseFolderOnInternalDriveWindowsError.text=Warning: Path to case folder is on \"C:\" drive. Case folder is created on the target system NewCaseVisualPanel1.CaseFolderOnInternalDriveLinuxError.text=Warning: Path to case folder is on the target system. Create case folder in mounted drive. +NewCaseVisualPanel1.uncPath.error=Error: UNC paths are not allowed for Single-User cases CollaborationMonitor.addingDataSourceStatus.msg={0} adding data source CollaborationMonitor.analyzingDataSourceStatus.msg={0} analyzing {1} MissingImageDialog.lbWarning.text= MissingImageDialog.lbWarning.toolTipText= NewCaseVisualPanel1.caseParentDirWarningLabel.text= -NewCaseVisualPanel1.multiUserCaseRadioButton.text=Multi-user -NewCaseVisualPanel1.singleUserCaseRadioButton.text=Single-user +NewCaseVisualPanel1.multiUserCaseRadioButton.text=Multi-User +NewCaseVisualPanel1.singleUserCaseRadioButton.text=Single-User NewCaseVisualPanel1.caseTypeLabel.text=Case Type: SingleUserCaseConverter.BadDatabaseFileName=Database file does not exist! SingleUserCaseConverter.AlreadyMultiUser=Case is already multi-user! diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/ImageFilePanel.java b/Core/src/org/sleuthkit/autopsy/casemodule/ImageFilePanel.java index 76f328a825..4c6126fcb7 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/ImageFilePanel.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/ImageFilePanel.java @@ -469,7 +469,7 @@ public class ImageFilePanel extends JPanel implements DocumentListener { return false; } - if (!PathValidator.isValidForMultiUserCase(path, Case.getCurrentCase().getCaseType())) { + if (!PathValidator.isValidForCaseType(path, Case.getCurrentCase().getCaseType())) { errorLabel.setVisible(true); errorLabel.setText(Bundle.ImageFilePanel_validatePanel_dataSourceOnCDriveError()); } diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/LocalFilesPanel.java b/Core/src/org/sleuthkit/autopsy/casemodule/LocalFilesPanel.java index 6703b16dc5..d55dab831e 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/LocalFilesPanel.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/LocalFilesPanel.java @@ -290,7 +290,7 @@ final class LocalFilesPanel extends javax.swing.JPanel { final Case.CaseType currentCaseType = Case.getCurrentCaseThrows().getCaseType(); for (String currentPath : pathsList) { - if (!PathValidator.isValidForMultiUserCase(currentPath, currentCaseType)) { + if (!PathValidator.isValidForCaseType(currentPath, currentCaseType)) { errorLabel.setVisible(true); errorLabel.setText(Bundle.LocalFilesPanel_pathValidation_dataSourceOnCDriveError()); return; diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/LogicalEvidenceFilePanel.java b/Core/src/org/sleuthkit/autopsy/casemodule/LogicalEvidenceFilePanel.java index 103fd8fd6c..6d45be07b9 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/LogicalEvidenceFilePanel.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/LogicalEvidenceFilePanel.java @@ -191,7 +191,7 @@ final class LogicalEvidenceFilePanel extends javax.swing.JPanel implements Docum } // display warning if there is one (but don't disable "next" button) try { - if (!PathValidator.isValidForMultiUserCase(path, Case.getCurrentCaseThrows().getCaseType())) { + if (!PathValidator.isValidForCaseType(path, Case.getCurrentCaseThrows().getCaseType())) { errorLabel.setVisible(true); errorLabel.setText(Bundle.LogicalEvidenceFilePanel_pathValidation_dataSourceOnCDriveError()); return false; diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/NewCaseVisualPanel1.java b/Core/src/org/sleuthkit/autopsy/casemodule/NewCaseVisualPanel1.java index b9ac003d02..32812b56ce 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/NewCaseVisualPanel1.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/NewCaseVisualPanel1.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2011-2018 Basis Technology Corp. + * Copyright 2011-2020 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -28,7 +28,6 @@ import javax.swing.JPanel; import javax.swing.event.DocumentEvent; import javax.swing.event.DocumentListener; import org.sleuthkit.autopsy.casemodule.Case.CaseType; -import org.sleuthkit.autopsy.core.UserPreferences; import org.sleuthkit.autopsy.coreutils.PathValidator; import org.sleuthkit.autopsy.coreutils.PlatformUtil; @@ -149,13 +148,22 @@ final class NewCaseVisualPanel1 extends JPanel implements DocumentListener { private void validateSettings() { /** * Check the base case directory for the selected case type and show a - * warning if it is a dubious choice. + * warning if it is a dubious choice. For single user cases, disable + * the "Next" button if path is invalid. */ caseParentDirWarningLabel.setVisible(false); String parentDir = getCaseParentDir(); - if (!PathValidator.isValidForMultiUserCase(parentDir, getCaseType())) { - caseParentDirWarningLabel.setVisible(true); - caseParentDirWarningLabel.setText(NbBundle.getMessage(this.getClass(), "NewCaseVisualPanel1.CaseFolderOnCDriveError.text")); + if (!PathValidator.isValidForCaseType(parentDir, getCaseType())) { + if (getCaseType() == CaseType.MULTI_USER_CASE) { + caseParentDirWarningLabel.setVisible(true); + caseParentDirWarningLabel.setText(NbBundle.getMessage(this.getClass(), "NewCaseVisualPanel1.CaseFolderOnCDriveError.text")); + } else { + // disable the "Next" button + caseParentDirWarningLabel.setVisible(true); + caseParentDirWarningLabel.setText(NbBundle.getMessage(this.getClass(), "NewCaseVisualPanel1.uncPath.error")); + wizPanel.setIsFinish(false); + return; + } } /** diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeUtil.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeUtil.java index 88f76c393c..4867b4eb0a 100755 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeUtil.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeUtil.java @@ -49,6 +49,15 @@ public class CorrelationAttributeUtil { private static final Logger logger = Logger.getLogger(CorrelationAttributeUtil.class.getName()); private static final List domainsToSkip = Arrays.asList("localhost", "127.0.0.1"); + // artifact ids that specifically have a TSK_DOMAIN attribute that should be handled by CR + private static Set DOMAIN_ARTIFACT_TYPE_IDS = new HashSet<>(Arrays.asList( + ARTIFACT_TYPE.TSK_WEB_BOOKMARK.getTypeID(), + ARTIFACT_TYPE.TSK_WEB_COOKIE.getTypeID(), + ARTIFACT_TYPE.TSK_WEB_DOWNLOAD.getTypeID(), + ARTIFACT_TYPE.TSK_WEB_HISTORY.getTypeID(), + ARTIFACT_TYPE.TSK_WEB_CACHE.getTypeID() + )); + /** * Gets a string that is expected to be the same string that is stored in * the correlation_types table in the central repository as the display name @@ -68,40 +77,39 @@ public class CorrelationAttributeUtil { // Most notably, does not include KEYWORD HIT, CALLLOGS, MESSAGES, CONTACTS // TSK_INTERESTING_ARTIFACT_HIT (See JIRA-6129 for more details on the // interesting artifact hit). - // IMPORTANT: This set should be updated for new artifacts types that need to // be inserted into the CR. - private static final Set SOURCE_TYPES_FOR_CR_INSERT = new HashSet() {{ - add(ARTIFACT_TYPE.TSK_WEB_BOOKMARK.getTypeID()); - add(ARTIFACT_TYPE.TSK_WEB_COOKIE.getTypeID()); - add(ARTIFACT_TYPE.TSK_WEB_DOWNLOAD.getTypeID()); - add(ARTIFACT_TYPE.TSK_WEB_HISTORY.getTypeID()); - add(ARTIFACT_TYPE.TSK_DEVICE_ATTACHED.getTypeID()); - add(ARTIFACT_TYPE.TSK_WIFI_NETWORK.getTypeID()); - add(ARTIFACT_TYPE.TSK_WIFI_NETWORK_ADAPTER.getTypeID()); - add(ARTIFACT_TYPE.TSK_BLUETOOTH_PAIRING.getTypeID()); - add(ARTIFACT_TYPE.TSK_BLUETOOTH_ADAPTER.getTypeID()); - add(ARTIFACT_TYPE.TSK_DEVICE_INFO.getTypeID()); - add(ARTIFACT_TYPE.TSK_SIM_ATTACHED.getTypeID()); - add(ARTIFACT_TYPE.TSK_WEB_FORM_ADDRESS.getTypeID()); - add(ARTIFACT_TYPE.TSK_ACCOUNT.getTypeID()); - }}; + private static final Set SOURCE_TYPES_FOR_CR_INSERT = new HashSet() { + { + addAll(DOMAIN_ARTIFACT_TYPE_IDS); + + add(ARTIFACT_TYPE.TSK_DEVICE_ATTACHED.getTypeID()); + add(ARTIFACT_TYPE.TSK_WIFI_NETWORK.getTypeID()); + add(ARTIFACT_TYPE.TSK_WIFI_NETWORK_ADAPTER.getTypeID()); + add(ARTIFACT_TYPE.TSK_BLUETOOTH_PAIRING.getTypeID()); + add(ARTIFACT_TYPE.TSK_BLUETOOTH_ADAPTER.getTypeID()); + add(ARTIFACT_TYPE.TSK_DEVICE_INFO.getTypeID()); + add(ARTIFACT_TYPE.TSK_SIM_ATTACHED.getTypeID()); + add(ARTIFACT_TYPE.TSK_WEB_FORM_ADDRESS.getTypeID()); + add(ARTIFACT_TYPE.TSK_ACCOUNT.getTypeID()); + } + }; /** * Makes zero to many correlation attribute instances from the attributes of * artifacts that have correlatable data. The intention of this method is to - * use the results to save to the CR, not to correlate with them. If you - * want to correlate, please use makeCorrAttrsForCorrelation. An artifact that can - * have correlatable data != An artifact that should be the source of data - * in the CR, so results may be un-necessarily incomplete. - * - * @param artifact An artifact. + * use the results to save to the CR, not to correlate with them. If you + * want to correlate, please use makeCorrAttrsForCorrelation. An artifact + * that can have correlatable data != An artifact that should be the source + * of data in the CR, so results may be un-necessarily incomplete. + * + * @param artifact An artifact. * * @return A list, possibly empty, of correlation attribute instances for - * the artifact. + * the artifact. */ public static List makeCorrAttrsToSave(BlackboardArtifact artifact) { - if(SOURCE_TYPES_FOR_CR_INSERT.contains(artifact.getArtifactTypeID())) { + if (SOURCE_TYPES_FOR_CR_INSERT.contains(artifact.getArtifactTypeID())) { // Restrict the correlation attributes to use for saving. // The artifacts which are suitable for saving are a subset of the // artifacts that are suitable for correlating. @@ -114,10 +122,10 @@ public class CorrelationAttributeUtil { /** * Makes zero to many correlation attribute instances from the attributes of * artifacts that have correlatable data. The intention of this method is to - * use the results to correlate with, not to save. If you - * want to save, please use makeCorrAttrsToSave. An artifact that can - * have correlatable data != An artifact that should be the source of data - * in the CR, so results may be too lenient. + * use the results to correlate with, not to save. If you want to save, + * please use makeCorrAttrsToSave. An artifact that can have correlatable + * data != An artifact that should be the source of data in the CR, so + * results may be too lenient. * * IMPORTANT: The correlation attribute instances are NOT added to the * central repository by this method. @@ -130,10 +138,10 @@ public class CorrelationAttributeUtil { * whether receiving a null return value is an error or not, plus null * checking is easy to forget, while catching exceptions is enforced. * - * @param artifact An artifact. + * @param artifact An artifact. * * @return A list, possibly empty, of correlation attribute instances for - * the artifact. + * the artifact. */ public static List makeCorrAttrsForCorrelation(BlackboardArtifact artifact) { List correlationAttrs = new ArrayList<>(); @@ -146,13 +154,10 @@ public class CorrelationAttributeUtil { if (setNameAttr != null && CorrelationAttributeUtil.getEmailAddressAttrDisplayName().equals(setNameAttr.getValueString())) { makeCorrAttrFromArtifactAttr(correlationAttrs, sourceArtifact, BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD, CorrelationAttributeInstance.EMAIL_TYPE_ID); } - } else if (artifactTypeID == ARTIFACT_TYPE.TSK_WEB_BOOKMARK.getTypeID() - || artifactTypeID == ARTIFACT_TYPE.TSK_WEB_COOKIE.getTypeID() - || artifactTypeID == ARTIFACT_TYPE.TSK_WEB_DOWNLOAD.getTypeID() - || artifactTypeID == ARTIFACT_TYPE.TSK_WEB_HISTORY.getTypeID()) { + } else if (DOMAIN_ARTIFACT_TYPE_IDS.contains(artifactTypeID)) { BlackboardAttribute domainAttr = sourceArtifact.getAttribute(new BlackboardAttribute.Type(ATTRIBUTE_TYPE.TSK_DOMAIN)); if ((domainAttr != null) - && ! domainsToSkip.contains(domainAttr.getValueString())) { + && !domainsToSkip.contains(domainAttr.getValueString())) { makeCorrAttrFromArtifactAttr(correlationAttrs, sourceArtifact, BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DOMAIN, CorrelationAttributeInstance.DOMAIN_TYPE_ID); } } else if (artifactTypeID == ARTIFACT_TYPE.TSK_DEVICE_ATTACHED.getTypeID()) { @@ -192,12 +197,10 @@ public class CorrelationAttributeUtil { } catch (CorrelationAttributeNormalizationException ex) { logger.log(Level.WARNING, String.format("Error normalizing correlation attribute (%s)", artifact), ex); // NON-NLS return correlationAttrs; - } - catch (InvalidAccountIDException ex) { + } catch (InvalidAccountIDException ex) { logger.log(Level.WARNING, String.format("Invalid account identifier (artifactID: %d)", artifact.getId())); // NON-NLS return correlationAttrs; - } - catch (CentralRepoException ex) { + } catch (CentralRepoException ex) { logger.log(Level.SEVERE, String.format("Error querying central repository (%s)", artifact), ex); // NON-NLS return correlationAttrs; } catch (TskCoreException ex) { @@ -259,11 +262,11 @@ public class CorrelationAttributeUtil { * @param artifact An artifact. * * @return The associated artifact if the input artifact is a - * "meta-artifact", otherwise the input artifact. + * "meta-artifact", otherwise the input artifact. * * @throws NoCurrentCaseException If there is no open case. - * @throws TskCoreException If there is an error querying thew case - * database. + * @throws TskCoreException If there is an error querying thew case + * database. */ private static BlackboardArtifact getCorrAttrSourceArtifact(BlackboardArtifact artifact) throws NoCurrentCaseException, TskCoreException { BlackboardArtifact sourceArtifact = null; @@ -287,7 +290,7 @@ public class CorrelationAttributeUtil { * repository by this method. * * @param corrAttrInstances A list of correlation attribute instances. - * @param acctArtifact An account artifact. + * @param acctArtifact An account artifact. * * @return The correlation attribute instance. */ @@ -296,11 +299,11 @@ public class CorrelationAttributeUtil { // Get the account type from the artifact BlackboardAttribute accountTypeAttribute = acctArtifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_ACCOUNT_TYPE)); String accountTypeStr = accountTypeAttribute.getValueString(); - + // @@TODO Vik-6136: CR currently does not know of custom account types. // Ensure there is a predefined account type for this account. Account.Type predefinedAccountType = Account.Type.PREDEFINED_ACCOUNT_TYPES.stream().filter(type -> type.getTypeName().equalsIgnoreCase(accountTypeStr)).findAny().orElse(null); - + // do not create any correlation attribute instance for a Device account if (Account.Type.DEVICE.getTypeName().equalsIgnoreCase(accountTypeStr) == false && predefinedAccountType != null) { @@ -331,17 +334,14 @@ public class CorrelationAttributeUtil { * artifact. The correlation attribute instance is added to an input list. * * @param corrAttrInstances A list of correlation attribute instances. - * @param artifact An artifact. - * @param artAttrType The type of the atrribute of the artifact that - * is to be made into a correlatin attribute - * instance. - * @param typeId The type ID for the desired correlation - * attribute instance. + * @param artifact An artifact. + * @param artAttrType The type of the atrribute of the artifact that is to + * be made into a correlatin attribute instance. + * @param typeId The type ID for the desired correlation attribute instance. * * @throws CentralRepoException If there is an error querying the central - * repository. - * @throws TskCoreException If there is an error querying the case - * database. + * repository. + * @throws TskCoreException If there is an error querying the case database. */ private static void makeCorrAttrFromArtifactAttr(List corrAttrInstances, BlackboardArtifact artifact, ATTRIBUTE_TYPE artAttrType, int typeId) throws CentralRepoException, TskCoreException { BlackboardAttribute attribute = artifact.getAttribute(new BlackboardAttribute.Type(artAttrType)); @@ -359,9 +359,9 @@ public class CorrelationAttributeUtil { /** * Makes a correlation attribute instance of a given type from an artifact. * - * @param artifact The artifact. + * @param artifact The artifact. * @param correlationType the correlation attribute type. - * @param value The correlation attribute value. + * @param value The correlation attribute value. * * TODO (Jira-6088): The methods in this low-level, utility class should * throw exceptions instead of logging them. The reason for this is that the @@ -422,7 +422,7 @@ public class CorrelationAttributeUtil { * checking is easy to forget, while catching exceptions is enforced. * * @return The correlation attribute instance or null, if no such - * correlation attribute instance was found or an error occurred. + * correlation attribute instance was found or an error occurred. */ public static CorrelationAttributeInstance getCorrAttrForFile(AbstractFile file) { diff --git a/Core/src/org/sleuthkit/autopsy/coreutils/PathValidator.java b/Core/src/org/sleuthkit/autopsy/coreutils/PathValidator.java index e3b091174a..0e0a12a995 100644 --- a/Core/src/org/sleuthkit/autopsy/coreutils/PathValidator.java +++ b/Core/src/org/sleuthkit/autopsy/coreutils/PathValidator.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2013-2018 Basis Technology Corp. + * Copyright 2013-2020 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -40,7 +40,7 @@ public final class PathValidator { * * @return - boolean true for valid path, false for invalid path */ - public static boolean isValidForMultiUserCase(String path, Case.CaseType caseType) { + public static boolean isValidForCaseType(String path, Case.CaseType caseType) { if (caseType == Case.CaseType.MULTI_USER_CASE) { // check that path is not on "C:" drive @@ -48,7 +48,10 @@ public final class PathValidator { return false; } } else { - // single user case - no validation needed + // check that path is not a UNC path. Solr 8 does not allow UNC paths for indexes. + if (UNCPathUtilities.isUNC(path)) { + return false; + } } return true; @@ -113,6 +116,19 @@ public final class PathValidator { */ @Deprecated public static boolean isValid(String path, Case.CaseType caseType) { - return isValidForMultiUserCase(path, caseType); + return isValidForCaseType(path, caseType); + } + + /** + * Checks if the provided path is valid given the case type. + * + * @param path - the path to validate + * @param caseType - the type of case which the path is being validated for + * + * @return - boolean true for valid path, false for invalid path + */ + @Deprecated + public static boolean isValidForMultiUserCase(String path, Case.CaseType caseType) { + return isValidForCaseType(path, caseType); } } diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/RawDSInputPanel.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/RawDSInputPanel.java index 747653ee65..1a02f65060 100644 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/RawDSInputPanel.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/RawDSInputPanel.java @@ -292,7 +292,7 @@ final class RawDSInputPanel extends JPanel implements DocumentListener { "RawDSInputPanel.noOpenCase.errMsg=Exception while getting open case."}) private void warnIfPathIsInvalid(String path) { try { - if (!PathValidator.isValidForMultiUserCase(path, Case.getCurrentCaseThrows().getCaseType())) { + if (!PathValidator.isValidForCaseType(path, Case.getCurrentCaseThrows().getCaseType())) { errorLabel.setVisible(true); errorLabel.setText(Bundle.RawDSInputPanel_error_text()); } diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/volatilityDSP/MemoryDSInputPanel.java b/Experimental/src/org/sleuthkit/autopsy/experimental/volatilityDSP/MemoryDSInputPanel.java index c08acee193..3d41adf201 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/volatilityDSP/MemoryDSInputPanel.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/volatilityDSP/MemoryDSInputPanel.java @@ -407,7 +407,7 @@ final class MemoryDSInputPanel extends JPanel implements DocumentListener { }) private void warnIfPathIsInvalid(String path) { try { - if (!PathValidator.isValidForMultiUserCase(path, Case.getCurrentCaseThrows().getCaseType())) { + if (!PathValidator.isValidForCaseType(path, Case.getCurrentCaseThrows().getCaseType())) { errorLabel.setVisible(true); errorLabel.setText(Bundle.MemoryDSInputPanel_errorMsg_dataSourcePathOnCdrive()); } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Index.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Index.java index 85afdbff07..f884654bb8 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Index.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Index.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2011-2017 Basis Technology Corp. + * Copyright 2011-2020 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,7 +21,6 @@ package org.sleuthkit.autopsy.keywordsearch; import java.text.SimpleDateFormat; import java.util.Date; import org.apache.commons.lang.math.NumberUtils; -import org.sleuthkit.autopsy.coreutils.UNCPathUtilities; /** * This class encapsulates KWS index data. @@ -33,7 +32,6 @@ final class Index { private final String solrVersion; private final String indexName; private static final String DEFAULT_CORE_NAME = "text_index"; //NON-NLS - private final UNCPathUtilities uncPathUtilities = new UNCPathUtilities(); /** * Constructs a representation of a text index. @@ -47,7 +45,7 @@ final class Index { * need to be generated. */ Index(String indexPath, String solrVersion, String schemaVersion, String coreName, String caseName) { - this.indexPath = uncPathUtilities.convertPathToUNC(indexPath); + this.indexPath = indexPath; this.solrVersion = solrVersion; this.schemaVersion = schemaVersion; if (coreName == null || coreName.isEmpty()) { diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexMetadata.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexMetadata.java index 4594bb36b3..b43ed40798 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexMetadata.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexMetadata.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2011-2017 Basis Technology Corp. + * Copyright 2011-2020 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -39,7 +39,6 @@ import javax.xml.transform.TransformerException; import javax.xml.transform.TransformerFactory; import javax.xml.transform.dom.DOMSource; import javax.xml.transform.stream.StreamResult; -import org.sleuthkit.autopsy.coreutils.UNCPathUtilities; import org.sleuthkit.autopsy.coreutils.XMLUtil; import org.w3c.dom.Document; import org.w3c.dom.Element; @@ -61,11 +60,10 @@ class IndexMetadata { private final static String SOLR_VERSION_ELEMENT_NAME = "SolrVersion"; //NON-NLS private final static String TEXT_INDEX_PATH_ELEMENT_NAME = "TextIndexPath"; //NON-NLS private List indexes = new ArrayList<>(); - private final UNCPathUtilities uncPathUtilities = new UNCPathUtilities(); IndexMetadata(String caseDirectory, Index index) throws TextIndexMetadataException { this.metadataFilePath = Paths.get(caseDirectory, METADATA_FILE_NAME); - this.caseDirectoryPath = Paths.get(uncPathUtilities.convertPathToUNC(caseDirectory)); + this.caseDirectoryPath = Paths.get(caseDirectory); this.indexes.add(index); writeToFile(); } @@ -73,7 +71,7 @@ class IndexMetadata { IndexMetadata(String caseDirectory, List indexes) throws TextIndexMetadataException { this.metadataFilePath = Paths.get(caseDirectory, METADATA_FILE_NAME); - this.caseDirectoryPath = Paths.get(uncPathUtilities.convertPathToUNC(caseDirectory)); + this.caseDirectoryPath = Paths.get(caseDirectory); this.indexes = indexes; writeToFile(); } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Ingester.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Ingester.java index cea68452e1..feeea64957 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Ingester.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Ingester.java @@ -125,7 +125,55 @@ class Ingester { private Map getContentFields(SleuthkitVisitableItem item) { return item.accept(SOLR_FIELDS_VISITOR); } - + + /** + * Read and chunk the source text for indexing in Solr. Also performs + * language detection on the input text. + * + * @param The type of the Appendix provider that provides additional + * text to append to the final chunk. + * @param A subclass of SleuthkitVisibleItem. + * @param Reader The reader containing extracted text. + * @param source The source from which text will be extracted, chunked, and + * indexed. + * @param context The ingest job context that can be used to cancel this + * process. + * + * @return True if indexing was completed, false otherwise. + * + * @throws org.sleuthkit.autopsy.keywordsearch.Ingester.IngesterException + */ + // TODO (JIRA-3118): Cancelled text indexing does not propagate cancellation to clients + < T extends SleuthkitVisitableItem> boolean indexText(Reader sourceReader, long sourceID, String sourceName, T source, IngestJobContext context) throws Ingester.IngesterException { + boolean doLanguageDetection = true; + return indexText(sourceReader, sourceID, sourceName, source, context, doLanguageDetection); + } + + /** + * Read and chunk the source text for indexing in Solr. Does NOT perform + * language detection on the input strings. Per JIRA-7100, it was determined + * that language detection on extracted strings can take a really long time. + * + * @param The type of the Appendix provider that provides additional + * text to append to the final chunk. + * @param A subclass of SleuthkitVisibleItem. + * @param Reader The reader containing extracted text. + * @param source The source from which text will be extracted, chunked, and + * indexed. + * @param context The ingest job context that can be used to cancel this + * process. + * + * @return True if indexing was completed, false otherwise. + * + * @throws org.sleuthkit.autopsy.keywordsearch.Ingester.IngesterException + */ + // TODO (JIRA-3118): Cancelled text indexing does not propagate cancellation to clients + < T extends SleuthkitVisitableItem> boolean indexStrings(Reader sourceReader, long sourceID, String sourceName, T source, IngestJobContext context) throws Ingester.IngesterException { + // Per JIRA-7100, it was determined that language detection on extracted strings can take a really long time. + boolean doLanguageDetection = false; + return indexText(sourceReader, sourceID, sourceName, source, context, doLanguageDetection); + } + /** * Read and chunk the source text for indexing in Solr. * @@ -138,13 +186,14 @@ class Ingester { * and indexed. * @param context The ingest job context that can be used to cancel this * process. + * @param doLanguageDetection A flag whether to perform language detection on the input text/strings. * * @return True if indexing was completed, false otherwise. * * @throws org.sleuthkit.autopsy.keywordsearch.Ingester.IngesterException */ // TODO (JIRA-3118): Cancelled text indexing does not propagate cancellation to clients - < T extends SleuthkitVisitableItem> boolean indexText(Reader sourceReader, long sourceID, String sourceName, T source, IngestJobContext context) throws Ingester.IngesterException { + private < T extends SleuthkitVisitableItem> boolean indexText(Reader sourceReader, long sourceID, String sourceName, T source, IngestJobContext context, boolean doLanguageDetection) throws Ingester.IngesterException { int numChunks = 0; //unknown until chunking is done Map contentFields = Collections.unmodifiableMap(getContentFields(source)); @@ -162,8 +211,11 @@ class Ingester { String chunkId = Server.getChunkIdString(sourceID, numChunks + 1); fields.put(Server.Schema.ID.toString(), chunkId); fields.put(Server.Schema.CHUNK_SIZE.toString(), String.valueOf(chunk.getBaseChunkLength())); - Optional language = languageSpecificContentIndexingHelper.detectLanguageIfNeeded(chunk); - language.ifPresent(lang -> languageSpecificContentIndexingHelper.updateLanguageSpecificFields(fields, chunk, lang)); + Optional language = Optional.empty(); + if (doLanguageDetection) { + language = languageSpecificContentIndexingHelper.detectLanguageIfNeeded(chunk); + language.ifPresent(lang -> languageSpecificContentIndexingHelper.updateLanguageSpecificFields(fields, chunk, lang)); + } try { //add the chunk text to Solr index indexChunk(chunk.toString(), chunk.geLowerCasedChunk(), sourceName, fields); diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java index 2212e0e309..4ef6db2a24 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java @@ -650,7 +650,7 @@ public final class KeywordSearchIngestModule implements FileIngestModule { } TextExtractor stringsExtractor = TextExtractorFactory.getStringsExtractor(aFile, stringsExtractionContext); Reader extractedTextReader = stringsExtractor.getReader(); - if (Ingester.getDefault().indexText(extractedTextReader, aFile.getId(), aFile.getName(), aFile, KeywordSearchIngestModule.this.context)) { + if (Ingester.getDefault().indexStrings(extractedTextReader, aFile.getId(), aFile.getName(), aFile, KeywordSearchIngestModule.this.context)) { putIngestStatus(jobId, aFile.getId(), IngestStatus.STRINGS_INGESTED); return true; } else { diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java index ec82726695..df44b2347a 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2015-2019 Basis Technology Corp. + * Copyright 2015-2020 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -125,7 +125,7 @@ public class SolrSearchService implements KeywordSearchService, AutopsyService { // Try the StringsTextExtractor if Tika extractions fails. TextExtractor stringsExtractor = TextExtractorFactory.getStringsExtractor(content, null); Reader stringsExtractedTextReader = stringsExtractor.getReader(); - ingester.indexText(stringsExtractedTextReader, content.getId(), content.getName(), content, null); + ingester.indexStrings(stringsExtractedTextReader, content.getId(), content.getName(), content, null); } catch (Ingester.IngesterException | TextExtractor.InitReaderException ex1) { throw new TskCoreException("Error indexing content", ex1); } diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Chromium.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Chromium.java index f45770a24d..24a1936727 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Chromium.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Chromium.java @@ -411,6 +411,7 @@ class Chromium extends Extract { } } postArtifacts(bbartifacts); + bbartifacts.clear(); dbFile.delete(); } } diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/DomainCategoryRunner.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/DomainCategoryRunner.java index 8b74ce48ce..cb06a525c9 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/DomainCategoryRunner.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/DomainCategoryRunner.java @@ -23,8 +23,10 @@ import java.net.URL; import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.Comparator; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.logging.Level; import java.util.Set; import java.util.regex.Matcher; @@ -76,8 +78,85 @@ class DomainCategoryRunner extends Extract { private static final String URL_REGEX_STR = String.format("^\\s*%s?%s?%s?", URL_REGEX_SCHEME, URL_REGEX_AUTHORITY, URL_REGEX_PATH); private static final Pattern URL_REGEX = Pattern.compile(URL_REGEX_STR); + private static int DATETIME_ACCESSED_TYPEID = ATTRIBUTE_TYPE.TSK_DATETIME_ACCESSED.getTypeID(); + private static int URL_TYPEID = ATTRIBUTE_TYPE.TSK_URL.getTypeID(); + private static final Logger logger = Logger.getLogger(DomainCategoryRunner.class.getName()); + /** + * Get seconds from epoch from the mapping for the attribute type id. + * + * @param attrMap A mapping of attribute type id to BlackboardAttribute for + * an artifact. + * @param attrTypeId The attribute type id to fetch. + * @return The time in seconds from epoch or 0 if cannot be found. + */ + private static long getTimeOrZero(Map attrMap, int attrTypeId) { + if (attrMap == null) { + return 0; + } + + BlackboardAttribute attr = attrMap.get(attrTypeId); + return attr == null ? 0 : attr.getValueLong(); + } + + /** + * Get string for attribute type id or "" if cannot be determined. + * + * @param attrMap A mapping of attribute type id to BlackboardAttribute for + * an artifact. + * @param attrTypeId The attribute type id to fetch. + * @return The string value or "" if cannot be determined or null. + */ + private static String getStringOrEmpty(Map attrMap, int attrTypeId) { + if (attrMap == null) { + return ""; + } + + BlackboardAttribute attr = attrMap.get(attrTypeId); + String attrStr = attr == null ? "" : attr.getValueString(); + return attrStr == null ? "" : attrStr; + } + + /** + * Comparator for ensuring deterministic order of processing artifacts. + */ + private static final Comparator ARTIFACT_COMPARATOR = (a, b) -> { + // get attributes in map by type id + Map attrMapA = null; + Map attrMapB = null; + + try { + attrMapA = a.getAttributes() + .stream() + .collect(Collectors.toMap(attr -> attr.getAttributeType().getTypeID(), attr -> attr, (attr1, attr2) -> attr1)); + + attrMapB = b.getAttributes() + .stream() + .collect(Collectors.toMap(attr -> attr.getAttributeType().getTypeID(), attr -> attr, (attr1, attr2) -> attr1)); + + } catch (TskCoreException ex) { + logger.log(Level.WARNING, "There was an error fetching attributes for artifacts", ex); + return 0; + } + + // sort first on time + int timeCompare = Long.compare(getTimeOrZero(attrMapA, DATETIME_ACCESSED_TYPEID), getTimeOrZero(attrMapB, DATETIME_ACCESSED_TYPEID)); + if (timeCompare != 0) { + // negate to push latest times to the front + return -timeCompare; + } + + // sort next on url + int urlCompare = getStringOrEmpty(attrMapA, URL_TYPEID).compareToIgnoreCase(getStringOrEmpty(attrMapB, URL_TYPEID)); + if (urlCompare != 0) { + return urlCompare; + } + + // use id as last resort + return Long.compare(a.getId(), b.getId()); + }; + private Content dataSource; private IngestJobContext context; private List domainProviders = Collections.emptyList(); @@ -272,11 +351,12 @@ class DomainCategoryRunner extends Extract { // only one suffix per ingest is captured so this tracks the suffixes seen. Set hostSuffixesSeen = new HashSet<>(); try { - Collection listArtifacts = currentCase.getSleuthkitCase().getBlackboard().getArtifacts( + List listArtifacts = currentCase.getSleuthkitCase().getBlackboard().getArtifacts( Arrays.asList(new BlackboardArtifact.Type(ARTIFACT_TYPE.TSK_WEB_HISTORY)), Arrays.asList(dataSource.getId())); logger.log(Level.INFO, "Processing {0} blackboard artifacts.", listArtifacts.size()); //NON-NLS + Collections.sort(listArtifacts, ARTIFACT_COMPARATOR); for (BlackboardArtifact artifact : listArtifacts) { // make sure we haven't cancelled @@ -361,11 +441,11 @@ class DomainCategoryRunner extends Extract { if (lookupList == null) { lookupList = Collections.emptyList(); } - + List foundProviders = lookupList.stream() - .filter(provider -> provider != null) - .sorted((a, b) -> a.getClass().getName().compareToIgnoreCase(b.getClass().getName())) - .collect(Collectors.toList()); + .filter(provider -> provider != null) + .sorted((a, b) -> a.getClass().getName().compareToIgnoreCase(b.getClass().getName())) + .collect(Collectors.toList()); // add the default categorizer last as a last resort foundProviders.add(new DefaultDomainCategorizer()); diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java index 0dde950a05..4166120823 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java @@ -247,7 +247,7 @@ class SearchEngineURLQueryAnalyzer extends Extract { } } try { //try to decode the url - String decoded = URLDecoder.decode(x, "UTF-8"); //NON-NLS + String decoded = URLDecoder.decode(x.replaceAll("%(?![0-9a-fA-F]{2})", "%25"), "UTF-8"); //NON-NLS return decoded; } catch (UnsupportedEncodingException exception) { //if it fails, return the encoded string logger.log(Level.FINE, "Error during URL decoding, returning undecoded value:" diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/MboxParser.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/MboxParser.java index eb4efc7451..241bd3bcb0 100644 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/MboxParser.java +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/MboxParser.java @@ -35,6 +35,7 @@ import java.util.Iterator; import java.util.List; import java.util.logging.Level; import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.autopsy.modules.filetypeid.FileTypeDetector; import org.apache.james.mime4j.dom.Message; import org.apache.james.mime4j.mboxiterator.CharBufferWrapper; import org.apache.james.mime4j.mboxiterator.MboxIterator; @@ -42,6 +43,7 @@ import org.apache.tika.parser.txt.CharsetDetector; import org.apache.tika.parser.txt.CharsetMatch; import org.apache.commons.validator.routines.EmailValidator; import org.openide.util.NbBundle; +import org.sleuthkit.datamodel.AbstractFile; /** * An Iterator for parsing mbox files. Wraps an instance of MBoxEmailIterator. @@ -56,12 +58,25 @@ class MboxParser extends MimeJ4MessageParser implements Iterator { setLocalPath(localPath); } - static boolean isValidMimeTypeMbox(byte[] buffer) { + static boolean isValidMimeTypeMbox(byte[] buffer, AbstractFile abstractFile) { String mboxHeaderLine = new String(buffer); if (mboxHeaderLine.startsWith("From ")) { - String[] mboxLineValues = mboxHeaderLine.split(" "); - EmailValidator validator = EmailValidator.getInstance(true, true); - return validator.isValid(mboxLineValues[1]); + String mimeType = abstractFile.getMIMEType(); + + // if it is not present, attempt to use the FileTypeDetector to determine + if (mimeType == null || mimeType.isEmpty()) { + FileTypeDetector fileTypeDetector = null; + try { + fileTypeDetector = new FileTypeDetector(); + } catch (FileTypeDetector.FileTypeDetectorInitException ex) { + logger.log(Level.WARNING, String.format("Unable to create file type detector for determining MIME type for file %s with id of %d", abstractFile.getName(), abstractFile.getId())); + return false; + } + mimeType = fileTypeDetector.getMIMEType(abstractFile); + } + if (mimeType.equalsIgnoreCase("application/mbox")) { + return true; + } } return false; //NON-NLS } diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java index 12f3342dac..2cfa766906 100644 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java @@ -130,7 +130,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { if (abstractFile.getSize() > 64) { int byteRead = abstractFile.read(t, 0, 64); if (byteRead > 0) { - isMbox = MboxParser.isValidMimeTypeMbox(t); + isMbox = MboxParser.isValidMimeTypeMbox(t, abstractFile); isEMLFile = EMLParser.isEMLFile(abstractFile, t); } }