mstoeck3/autopsy-flatpak
1
0
Fork 0
mirror of https://github.com/overcuriousity/autopsy-flatpak.git synced 2025-07-17 10:17:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'develop' of https://github.com/sleuthkit/autopsy into 7084-FeedbackWhileLoadingArtifacts

Browse Source
This commit is contained in:
William Schaefer 2020-12-18 17:24:53 -05:00
commit 02e6c7e004
19 changed files with 270 additions and 101 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
Core/src/org/sleuthkit/autopsy/casemodule/Bundle.properties
View File

@ -146,13 +146,14 @@ AddImageWizardIngestConfigPanel.CANCEL_BUTTON.text=Cancel
NewCaseVisualPanel1.CaseFolderOnCDriveError.text=Warning: Path to multi-user case folder is on \"C:\" drive
NewCaseVisualPanel1.CaseFolderOnInternalDriveWindowsError.text=Warning: Path to case folder is on \"C:\" drive. Case folder is created on the target system
NewCaseVisualPanel1.CaseFolderOnInternalDriveLinuxError.text=Warning: Path to case folder is on the target system. Create case folder in mounted drive.
NewCaseVisualPanel1.uncPath.error=Error: UNC paths are not allowed for Single-User cases
CollaborationMonitor.addingDataSourceStatus.msg={0} adding data source
CollaborationMonitor.analyzingDataSourceStatus.msg={0} analyzing {1}
MissingImageDialog.lbWarning.text=
MissingImageDialog.lbWarning.toolTipText=
NewCaseVisualPanel1.caseParentDirWarningLabel.text=
NewCaseVisualPanel1.multiUserCaseRadioButton.text=Multi-user
NewCaseVisualPanel1.singleUserCaseRadioButton.text=Single-user
NewCaseVisualPanel1.multiUserCaseRadioButton.text=Multi-User
NewCaseVisualPanel1.singleUserCaseRadioButton.text=Single-User
NewCaseVisualPanel1.caseTypeLabel.text=Case Type:
SingleUserCaseConverter.BadDatabaseFileName=Database file does not exist!
SingleUserCaseConverter.AlreadyMultiUser=Case is already multi-user!

2
Core/src/org/sleuthkit/autopsy/casemodule/ImageFilePanel.java
View File

@ -469,7 +469,7 @@ public class ImageFilePanel extends JPanel implements DocumentListener {
return false;
}
if (!PathValidator.isValidForMultiUserCase(path, Case.getCurrentCase().getCaseType())) {
if (!PathValidator.isValidForCaseType(path, Case.getCurrentCase().getCaseType())) {
errorLabel.setVisible(true);
errorLabel.setText(Bundle.ImageFilePanel_validatePanel_dataSourceOnCDriveError());
}

2
Core/src/org/sleuthkit/autopsy/casemodule/LocalFilesPanel.java
View File

@ -290,7 +290,7 @@ final class LocalFilesPanel extends javax.swing.JPanel {
final Case.CaseType currentCaseType = Case.getCurrentCaseThrows().getCaseType();
for (String currentPath : pathsList) {
if (!PathValidator.isValidForMultiUserCase(currentPath, currentCaseType)) {
if (!PathValidator.isValidForCaseType(currentPath, currentCaseType)) {
errorLabel.setVisible(true);
errorLabel.setText(Bundle.LocalFilesPanel_pathValidation_dataSourceOnCDriveError());
return;

2
Core/src/org/sleuthkit/autopsy/casemodule/LogicalEvidenceFilePanel.java
View File

@ -191,7 +191,7 @@ final class LogicalEvidenceFilePanel extends javax.swing.JPanel implements Docum
}
// display warning if there is one (but don't disable "next" button)
try {
if (!PathValidator.isValidForMultiUserCase(path, Case.getCurrentCaseThrows().getCaseType())) {
if (!PathValidator.isValidForCaseType(path, Case.getCurrentCaseThrows().getCaseType())) {
errorLabel.setVisible(true);
errorLabel.setText(Bundle.LogicalEvidenceFilePanel_pathValidation_dataSourceOnCDriveError());
return false;

20
Core/src/org/sleuthkit/autopsy/casemodule/NewCaseVisualPanel1.java
View File

@ -1,7 +1,7 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2011-2018 Basis Technology Corp.
* Copyright 2011-2020 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
@ -28,7 +28,6 @@ import javax.swing.JPanel;
import javax.swing.event.DocumentEvent;
import javax.swing.event.DocumentListener;
import org.sleuthkit.autopsy.casemodule.Case.CaseType;
import org.sleuthkit.autopsy.core.UserPreferences;
import org.sleuthkit.autopsy.coreutils.PathValidator;
import org.sleuthkit.autopsy.coreutils.PlatformUtil;
@ -149,13 +148,22 @@ final class NewCaseVisualPanel1 extends JPanel implements DocumentListener {
private void validateSettings() {
/**
* Check the base case directory for the selected case type and show a
* warning if it is a dubious choice.
* warning if it is a dubious choice. For single user cases, disable
* the "Next" button if path is invalid.
*/
caseParentDirWarningLabel.setVisible(false);
String parentDir = getCaseParentDir();
if (!PathValidator.isValidForMultiUserCase(parentDir, getCaseType())) {
caseParentDirWarningLabel.setVisible(true);
caseParentDirWarningLabel.setText(NbBundle.getMessage(this.getClass(), "NewCaseVisualPanel1.CaseFolderOnCDriveError.text"));
if (!PathValidator.isValidForCaseType(parentDir, getCaseType())) {
if (getCaseType() == CaseType.MULTI_USER_CASE) {
caseParentDirWarningLabel.setVisible(true);
caseParentDirWarningLabel.setText(NbBundle.getMessage(this.getClass(), "NewCaseVisualPanel1.CaseFolderOnCDriveError.text"));
} else {
// disable the "Next" button
caseParentDirWarningLabel.setVisible(true);
caseParentDirWarningLabel.setText(NbBundle.getMessage(this.getClass(), "NewCaseVisualPanel1.uncPath.error"));
wizPanel.setIsFinish(false);
return;
}
}
/**

114
Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeUtil.java
View File

@ -49,6 +49,15 @@ public class CorrelationAttributeUtil {
private static final Logger logger = Logger.getLogger(CorrelationAttributeUtil.class.getName());
private static final List<String> domainsToSkip = Arrays.asList("localhost", "127.0.0.1");
// artifact ids that specifically have a TSK_DOMAIN attribute that should be handled by CR
private static Set<Integer> DOMAIN_ARTIFACT_TYPE_IDS = new HashSet<>(Arrays.asList(
ARTIFACT_TYPE.TSK_WEB_BOOKMARK.getTypeID(),
ARTIFACT_TYPE.TSK_WEB_COOKIE.getTypeID(),
ARTIFACT_TYPE.TSK_WEB_DOWNLOAD.getTypeID(),
ARTIFACT_TYPE.TSK_WEB_HISTORY.getTypeID(),
ARTIFACT_TYPE.TSK_WEB_CACHE.getTypeID()
));
/**
* Gets a string that is expected to be the same string that is stored in
* the correlation_types table in the central repository as the display name
@ -68,40 +77,39 @@ public class CorrelationAttributeUtil {
// Most notably, does not include KEYWORD HIT, CALLLOGS, MESSAGES, CONTACTS
// TSK_INTERESTING_ARTIFACT_HIT (See JIRA-6129 for more details on the
// interesting artifact hit).
// IMPORTANT: This set should be updated for new artifacts types that need to
// be inserted into the CR.
private static final Set<Integer> SOURCE_TYPES_FOR_CR_INSERT = new HashSet<Integer>() {{
add(ARTIFACT_TYPE.TSK_WEB_BOOKMARK.getTypeID());
add(ARTIFACT_TYPE.TSK_WEB_COOKIE.getTypeID());
add(ARTIFACT_TYPE.TSK_WEB_DOWNLOAD.getTypeID());
add(ARTIFACT_TYPE.TSK_WEB_HISTORY.getTypeID());
add(ARTIFACT_TYPE.TSK_DEVICE_ATTACHED.getTypeID());
add(ARTIFACT_TYPE.TSK_WIFI_NETWORK.getTypeID());
add(ARTIFACT_TYPE.TSK_WIFI_NETWORK_ADAPTER.getTypeID());
add(ARTIFACT_TYPE.TSK_BLUETOOTH_PAIRING.getTypeID());
add(ARTIFACT_TYPE.TSK_BLUETOOTH_ADAPTER.getTypeID());
add(ARTIFACT_TYPE.TSK_DEVICE_INFO.getTypeID());
add(ARTIFACT_TYPE.TSK_SIM_ATTACHED.getTypeID());
add(ARTIFACT_TYPE.TSK_WEB_FORM_ADDRESS.getTypeID());
add(ARTIFACT_TYPE.TSK_ACCOUNT.getTypeID());
}};
private static final Set<Integer> SOURCE_TYPES_FOR_CR_INSERT = new HashSet<Integer>() {
{
addAll(DOMAIN_ARTIFACT_TYPE_IDS);
add(ARTIFACT_TYPE.TSK_DEVICE_ATTACHED.getTypeID());
add(ARTIFACT_TYPE.TSK_WIFI_NETWORK.getTypeID());
add(ARTIFACT_TYPE.TSK_WIFI_NETWORK_ADAPTER.getTypeID());
add(ARTIFACT_TYPE.TSK_BLUETOOTH_PAIRING.getTypeID());
add(ARTIFACT_TYPE.TSK_BLUETOOTH_ADAPTER.getTypeID());
add(ARTIFACT_TYPE.TSK_DEVICE_INFO.getTypeID());
add(ARTIFACT_TYPE.TSK_SIM_ATTACHED.getTypeID());
add(ARTIFACT_TYPE.TSK_WEB_FORM_ADDRESS.getTypeID());
add(ARTIFACT_TYPE.TSK_ACCOUNT.getTypeID());
}
};
/**
* Makes zero to many correlation attribute instances from the attributes of
* artifacts that have correlatable data. The intention of this method is to
* use the results to save to the CR, not to correlate with them. If you
* want to correlate, please use makeCorrAttrsForCorrelation. An artifact that can
* have correlatable data != An artifact that should be the source of data
* in the CR, so results may be un-necessarily incomplete.
*
* @param artifact An artifact.
* use the results to save to the CR, not to correlate with them. If you
* want to correlate, please use makeCorrAttrsForCorrelation. An artifact
* that can have correlatable data != An artifact that should be the source
* of data in the CR, so results may be un-necessarily incomplete.
*
* @param artifact An artifact.
*
* @return A list, possibly empty, of correlation attribute instances for
* the artifact.
* the artifact.
*/
public static List<CorrelationAttributeInstance> makeCorrAttrsToSave(BlackboardArtifact artifact) {
if(SOURCE_TYPES_FOR_CR_INSERT.contains(artifact.getArtifactTypeID())) {
if (SOURCE_TYPES_FOR_CR_INSERT.contains(artifact.getArtifactTypeID())) {
// Restrict the correlation attributes to use for saving.
// The artifacts which are suitable for saving are a subset of the
// artifacts that are suitable for correlating.
@ -114,10 +122,10 @@ public class CorrelationAttributeUtil {
/**
* Makes zero to many correlation attribute instances from the attributes of
* artifacts that have correlatable data. The intention of this method is to
* use the results to correlate with, not to save. If you
* want to save, please use makeCorrAttrsToSave. An artifact that can
* have correlatable data != An artifact that should be the source of data
* in the CR, so results may be too lenient.
* use the results to correlate with, not to save. If you want to save,
* please use makeCorrAttrsToSave. An artifact that can have correlatable
* data != An artifact that should be the source of data in the CR, so
* results may be too lenient.
*
* IMPORTANT: The correlation attribute instances are NOT added to the
* central repository by this method.
@ -130,10 +138,10 @@ public class CorrelationAttributeUtil {
* whether receiving a null return value is an error or not, plus null
* checking is easy to forget, while catching exceptions is enforced.
*
* @param artifact An artifact.
* @param artifact An artifact.
*
* @return A list, possibly empty, of correlation attribute instances for
* the artifact.
* the artifact.
*/
public static List<CorrelationAttributeInstance> makeCorrAttrsForCorrelation(BlackboardArtifact artifact) {
List<CorrelationAttributeInstance> correlationAttrs = new ArrayList<>();
@ -146,13 +154,10 @@ public class CorrelationAttributeUtil {
if (setNameAttr != null && CorrelationAttributeUtil.getEmailAddressAttrDisplayName().equals(setNameAttr.getValueString())) {
makeCorrAttrFromArtifactAttr(correlationAttrs, sourceArtifact, BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD, CorrelationAttributeInstance.EMAIL_TYPE_ID);
}
} else if (artifactTypeID == ARTIFACT_TYPE.TSK_WEB_BOOKMARK.getTypeID()
|| artifactTypeID == ARTIFACT_TYPE.TSK_WEB_COOKIE.getTypeID()
|| artifactTypeID == ARTIFACT_TYPE.TSK_WEB_DOWNLOAD.getTypeID()
|| artifactTypeID == ARTIFACT_TYPE.TSK_WEB_HISTORY.getTypeID()) {
} else if (DOMAIN_ARTIFACT_TYPE_IDS.contains(artifactTypeID)) {
BlackboardAttribute domainAttr = sourceArtifact.getAttribute(new BlackboardAttribute.Type(ATTRIBUTE_TYPE.TSK_DOMAIN));
if ((domainAttr != null)
&& ! domainsToSkip.contains(domainAttr.getValueString())) {
&& !domainsToSkip.contains(domainAttr.getValueString())) {
makeCorrAttrFromArtifactAttr(correlationAttrs, sourceArtifact, BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DOMAIN, CorrelationAttributeInstance.DOMAIN_TYPE_ID);
}
} else if (artifactTypeID == ARTIFACT_TYPE.TSK_DEVICE_ATTACHED.getTypeID()) {
@ -192,12 +197,10 @@ public class CorrelationAttributeUtil {
} catch (CorrelationAttributeNormalizationException ex) {
logger.log(Level.WARNING, String.format("Error normalizing correlation attribute (%s)", artifact), ex); // NON-NLS
return correlationAttrs;
}
catch (InvalidAccountIDException ex) {
} catch (InvalidAccountIDException ex) {
logger.log(Level.WARNING, String.format("Invalid account identifier (artifactID: %d)", artifact.getId())); // NON-NLS
return correlationAttrs;
}
catch (CentralRepoException ex) {
} catch (CentralRepoException ex) {
logger.log(Level.SEVERE, String.format("Error querying central repository (%s)", artifact), ex); // NON-NLS
return correlationAttrs;
} catch (TskCoreException ex) {
@ -259,11 +262,11 @@ public class CorrelationAttributeUtil {
* @param artifact An artifact.
*
* @return The associated artifact if the input artifact is a
* "meta-artifact", otherwise the input artifact.
* "meta-artifact", otherwise the input artifact.
*
* @throws NoCurrentCaseException If there is no open case.
* @throws TskCoreException If there is an error querying thew case
* database.
* @throws TskCoreException If there is an error querying thew case
* database.
*/
private static BlackboardArtifact getCorrAttrSourceArtifact(BlackboardArtifact artifact) throws NoCurrentCaseException, TskCoreException {
BlackboardArtifact sourceArtifact = null;
@ -287,7 +290,7 @@ public class CorrelationAttributeUtil {
* repository by this method.
*
* @param corrAttrInstances A list of correlation attribute instances.
* @param acctArtifact An account artifact.
* @param acctArtifact An account artifact.
*
* @return The correlation attribute instance.
*/
@ -296,11 +299,11 @@ public class CorrelationAttributeUtil {
// Get the account type from the artifact
BlackboardAttribute accountTypeAttribute = acctArtifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_ACCOUNT_TYPE));
String accountTypeStr = accountTypeAttribute.getValueString();
// @@TODO Vik-6136: CR currently does not know of custom account types.
// Ensure there is a predefined account type for this account.
Account.Type predefinedAccountType = Account.Type.PREDEFINED_ACCOUNT_TYPES.stream().filter(type -> type.getTypeName().equalsIgnoreCase(accountTypeStr)).findAny().orElse(null);
// do not create any correlation attribute instance for a Device account
if (Account.Type.DEVICE.getTypeName().equalsIgnoreCase(accountTypeStr) == false && predefinedAccountType != null) {
@ -331,17 +334,14 @@ public class CorrelationAttributeUtil {
* artifact. The correlation attribute instance is added to an input list.
*
* @param corrAttrInstances A list of correlation attribute instances.
* @param artifact An artifact.
* @param artAttrType The type of the atrribute of the artifact that
* is to be made into a correlatin attribute
* instance.
* @param typeId The type ID for the desired correlation
* attribute instance.
* @param artifact An artifact.
* @param artAttrType The type of the atrribute of the artifact that is to
* be made into a correlatin attribute instance.
* @param typeId The type ID for the desired correlation attribute instance.
*
* @throws CentralRepoException If there is an error querying the central
* repository.
* @throws TskCoreException If there is an error querying the case
* database.
* repository.
* @throws TskCoreException If there is an error querying the case database.
*/
private static void makeCorrAttrFromArtifactAttr(List<CorrelationAttributeInstance> corrAttrInstances, BlackboardArtifact artifact, ATTRIBUTE_TYPE artAttrType, int typeId) throws CentralRepoException, TskCoreException {
BlackboardAttribute attribute = artifact.getAttribute(new BlackboardAttribute.Type(artAttrType));
@ -359,9 +359,9 @@ public class CorrelationAttributeUtil {
/**
* Makes a correlation attribute instance of a given type from an artifact.
*
* @param artifact The artifact.
* @param artifact The artifact.
* @param correlationType the correlation attribute type.
* @param value The correlation attribute value.
* @param value The correlation attribute value.
*
* TODO (Jira-6088): The methods in this low-level, utility class should
* throw exceptions instead of logging them. The reason for this is that the
@ -422,7 +422,7 @@ public class CorrelationAttributeUtil {
* checking is easy to forget, while catching exceptions is enforced.
*
* @return The correlation attribute instance or null, if no such
* correlation attribute instance was found or an error occurred.
* correlation attribute instance was found or an error occurred.
*/
public static CorrelationAttributeInstance getCorrAttrForFile(AbstractFile file) {

24
Core/src/org/sleuthkit/autopsy/coreutils/PathValidator.java
View File

@ -1,7 +1,7 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2013-2018 Basis Technology Corp.
* Copyright 2013-2020 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
@ -40,7 +40,7 @@ public final class PathValidator {
*
* @return - boolean true for valid path, false for invalid path
*/
public static boolean isValidForMultiUserCase(String path, Case.CaseType caseType) {
public static boolean isValidForCaseType(String path, Case.CaseType caseType) {
if (caseType == Case.CaseType.MULTI_USER_CASE) {
// check that path is not on "C:" drive
@ -48,7 +48,10 @@ public final class PathValidator {
return false;
}
} else {
// single user case - no validation needed
// check that path is not a UNC path. Solr 8 does not allow UNC paths for indexes.
if (UNCPathUtilities.isUNC(path)) {
return false;
}
}
return true;
@ -113,6 +116,19 @@ public final class PathValidator {
*/
@Deprecated
public static boolean isValid(String path, Case.CaseType caseType) {
return isValidForMultiUserCase(path, caseType);
return isValidForCaseType(path, caseType);
}
/**
* Checks if the provided path is valid given the case type.
*
* @param path - the path to validate
* @param caseType - the type of case which the path is being validated for
*
* @return - boolean true for valid path, false for invalid path
*/
@Deprecated
public static boolean isValidForMultiUserCase(String path, Case.CaseType caseType) {
return isValidForCaseType(path, caseType);
}
}

2
Core/src/org/sleuthkit/autopsy/datasourceprocessors/RawDSInputPanel.java
View File

@ -292,7 +292,7 @@ final class RawDSInputPanel extends JPanel implements DocumentListener {
"RawDSInputPanel.noOpenCase.errMsg=Exception while getting open case."})
private void warnIfPathIsInvalid(String path) {
try {
if (!PathValidator.isValidForMultiUserCase(path, Case.getCurrentCaseThrows().getCaseType())) {
if (!PathValidator.isValidForCaseType(path, Case.getCurrentCaseThrows().getCaseType())) {
errorLabel.setVisible(true);
errorLabel.setText(Bundle.RawDSInputPanel_error_text());
}

2
Experimental/src/org/sleuthkit/autopsy/experimental/volatilityDSP/MemoryDSInputPanel.java
View File

@ -407,7 +407,7 @@ final class MemoryDSInputPanel extends JPanel implements DocumentListener {
})
private void warnIfPathIsInvalid(String path) {
try {
if (!PathValidator.isValidForMultiUserCase(path, Case.getCurrentCaseThrows().getCaseType())) {
if (!PathValidator.isValidForCaseType(path, Case.getCurrentCaseThrows().getCaseType())) {
errorLabel.setVisible(true);
errorLabel.setText(Bundle.MemoryDSInputPanel_errorMsg_dataSourcePathOnCdrive());
}

6
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Index.java
View File

@ -1,7 +1,7 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2011-2017 Basis Technology Corp.
* Copyright 2011-2020 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
@ -21,7 +21,6 @@ package org.sleuthkit.autopsy.keywordsearch;
import java.text.SimpleDateFormat;
import java.util.Date;
import org.apache.commons.lang.math.NumberUtils;
import org.sleuthkit.autopsy.coreutils.UNCPathUtilities;
/**
* This class encapsulates KWS index data.
@ -33,7 +32,6 @@ final class Index {
private final String solrVersion;
private final String indexName;
private static final String DEFAULT_CORE_NAME = "text_index"; //NON-NLS
private final UNCPathUtilities uncPathUtilities = new UNCPathUtilities();
/**
* Constructs a representation of a text index.
@ -47,7 +45,7 @@ final class Index {
* need to be generated.
*/
Index(String indexPath, String solrVersion, String schemaVersion, String coreName, String caseName) {
this.indexPath = uncPathUtilities.convertPathToUNC(indexPath);
this.indexPath = indexPath;
this.solrVersion = solrVersion;
this.schemaVersion = schemaVersion;
if (coreName == null || coreName.isEmpty()) {

8
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexMetadata.java
View File

@ -1,7 +1,7 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2011-2017 Basis Technology Corp.
* Copyright 2011-2020 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
@ -39,7 +39,6 @@ import javax.xml.transform.TransformerException;
import javax.xml.transform.TransformerFactory;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;
import org.sleuthkit.autopsy.coreutils.UNCPathUtilities;
import org.sleuthkit.autopsy.coreutils.XMLUtil;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
@ -61,11 +60,10 @@ class IndexMetadata {
private final static String SOLR_VERSION_ELEMENT_NAME = "SolrVersion"; //NON-NLS
private final static String TEXT_INDEX_PATH_ELEMENT_NAME = "TextIndexPath"; //NON-NLS
private List<Index> indexes = new ArrayList<>();
private final UNCPathUtilities uncPathUtilities = new UNCPathUtilities();
IndexMetadata(String caseDirectory, Index index) throws TextIndexMetadataException {
this.metadataFilePath = Paths.get(caseDirectory, METADATA_FILE_NAME);
this.caseDirectoryPath = Paths.get(uncPathUtilities.convertPathToUNC(caseDirectory));
this.caseDirectoryPath = Paths.get(caseDirectory);
this.indexes.add(index);
writeToFile();
}
@ -73,7 +71,7 @@ class IndexMetadata {
IndexMetadata(String caseDirectory, List<Index> indexes) throws TextIndexMetadataException {
this.metadataFilePath = Paths.get(caseDirectory, METADATA_FILE_NAME);
this.caseDirectoryPath = Paths.get(uncPathUtilities.convertPathToUNC(caseDirectory));
this.caseDirectoryPath = Paths.get(caseDirectory);
this.indexes = indexes;
writeToFile();
}

60
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Ingester.java
View File

@ -125,7 +125,55 @@ class Ingester {
private Map<String, String> getContentFields(SleuthkitVisitableItem item) {
return item.accept(SOLR_FIELDS_VISITOR);
}
/**
* Read and chunk the source text for indexing in Solr. Also performs
* language detection on the input text.
*
* @param <A> The type of the Appendix provider that provides additional
* text to append to the final chunk.
* @param <T> A subclass of SleuthkitVisibleItem.
* @param Reader The reader containing extracted text.
* @param source The source from which text will be extracted, chunked, and
* indexed.
* @param context The ingest job context that can be used to cancel this
* process.
*
* @return True if indexing was completed, false otherwise.
*
* @throws org.sleuthkit.autopsy.keywordsearch.Ingester.IngesterException
*/
// TODO (JIRA-3118): Cancelled text indexing does not propagate cancellation to clients
< T extends SleuthkitVisitableItem> boolean indexText(Reader sourceReader, long sourceID, String sourceName, T source, IngestJobContext context) throws Ingester.IngesterException {
boolean doLanguageDetection = true;
return indexText(sourceReader, sourceID, sourceName, source, context, doLanguageDetection);
}
/**
* Read and chunk the source text for indexing in Solr. Does NOT perform
* language detection on the input strings. Per JIRA-7100, it was determined
* that language detection on extracted strings can take a really long time.
*
* @param <A> The type of the Appendix provider that provides additional
* text to append to the final chunk.
* @param <T> A subclass of SleuthkitVisibleItem.
* @param Reader The reader containing extracted text.
* @param source The source from which text will be extracted, chunked, and
* indexed.
* @param context The ingest job context that can be used to cancel this
* process.
*
* @return True if indexing was completed, false otherwise.
*
* @throws org.sleuthkit.autopsy.keywordsearch.Ingester.IngesterException
*/
// TODO (JIRA-3118): Cancelled text indexing does not propagate cancellation to clients
< T extends SleuthkitVisitableItem> boolean indexStrings(Reader sourceReader, long sourceID, String sourceName, T source, IngestJobContext context) throws Ingester.IngesterException {
// Per JIRA-7100, it was determined that language detection on extracted strings can take a really long time.
boolean doLanguageDetection = false;
return indexText(sourceReader, sourceID, sourceName, source, context, doLanguageDetection);
}
/**
* Read and chunk the source text for indexing in Solr.
*
@ -138,13 +186,14 @@ class Ingester {
* and indexed.
* @param context The ingest job context that can be used to cancel this
* process.
* @param doLanguageDetection A flag whether to perform language detection on the input text/strings.
*
* @return True if indexing was completed, false otherwise.
*
* @throws org.sleuthkit.autopsy.keywordsearch.Ingester.IngesterException
*/
// TODO (JIRA-3118): Cancelled text indexing does not propagate cancellation to clients
< T extends SleuthkitVisitableItem> boolean indexText(Reader sourceReader, long sourceID, String sourceName, T source, IngestJobContext context) throws Ingester.IngesterException {
private < T extends SleuthkitVisitableItem> boolean indexText(Reader sourceReader, long sourceID, String sourceName, T source, IngestJobContext context, boolean doLanguageDetection) throws Ingester.IngesterException {
int numChunks = 0; //unknown until chunking is done
Map<String, String> contentFields = Collections.unmodifiableMap(getContentFields(source));
@ -162,8 +211,11 @@ class Ingester {
String chunkId = Server.getChunkIdString(sourceID, numChunks + 1);
fields.put(Server.Schema.ID.toString(), chunkId);
fields.put(Server.Schema.CHUNK_SIZE.toString(), String.valueOf(chunk.getBaseChunkLength()));
Optional<Language> language = languageSpecificContentIndexingHelper.detectLanguageIfNeeded(chunk);
language.ifPresent(lang -> languageSpecificContentIndexingHelper.updateLanguageSpecificFields(fields, chunk, lang));
Optional<Language> language = Optional.empty();
if (doLanguageDetection) {
language = languageSpecificContentIndexingHelper.detectLanguageIfNeeded(chunk);
language.ifPresent(lang -> languageSpecificContentIndexingHelper.updateLanguageSpecificFields(fields, chunk, lang));
}
try {
//add the chunk text to Solr index
indexChunk(chunk.toString(), chunk.geLowerCasedChunk(), sourceName, fields);

2
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java
View File

@ -650,7 +650,7 @@ public final class KeywordSearchIngestModule implements FileIngestModule {
}
TextExtractor stringsExtractor = TextExtractorFactory.getStringsExtractor(aFile, stringsExtractionContext);
Reader extractedTextReader = stringsExtractor.getReader();
if (Ingester.getDefault().indexText(extractedTextReader, aFile.getId(), aFile.getName(), aFile, KeywordSearchIngestModule.this.context)) {
if (Ingester.getDefault().indexStrings(extractedTextReader, aFile.getId(), aFile.getName(), aFile, KeywordSearchIngestModule.this.context)) {
putIngestStatus(jobId, aFile.getId(), IngestStatus.STRINGS_INGESTED);
return true;
} else {

4
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java
View File

@ -1,7 +1,7 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2015-2019 Basis Technology Corp.
* Copyright 2015-2020 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
@ -125,7 +125,7 @@ public class SolrSearchService implements KeywordSearchService, AutopsyService {
// Try the StringsTextExtractor if Tika extractions fails.
TextExtractor stringsExtractor = TextExtractorFactory.getStringsExtractor(content, null);
Reader stringsExtractedTextReader = stringsExtractor.getReader();
ingester.indexText(stringsExtractedTextReader, content.getId(), content.getName(), content, null);
ingester.indexStrings(stringsExtractedTextReader, content.getId(), content.getName(), content, null);
} catch (Ingester.IngesterException | TextExtractor.InitReaderException ex1) {
throw new TskCoreException("Error indexing content", ex1);
}

1
RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Chromium.java
View File

@ -411,6 +411,7 @@ class Chromium extends Extract {
}
}
postArtifacts(bbartifacts);
bbartifacts.clear();
dbFile.delete();
}
}

90
RecentActivity/src/org/sleuthkit/autopsy/recentactivity/DomainCategoryRunner.java
View File

@ -23,8 +23,10 @@ import java.net.URL;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.logging.Level;
import java.util.Set;
import java.util.regex.Matcher;
@ -76,8 +78,85 @@ class DomainCategoryRunner extends Extract {
private static final String URL_REGEX_STR = String.format("^\\s*%s?%s?%s?", URL_REGEX_SCHEME, URL_REGEX_AUTHORITY, URL_REGEX_PATH);
private static final Pattern URL_REGEX = Pattern.compile(URL_REGEX_STR);
private static int DATETIME_ACCESSED_TYPEID = ATTRIBUTE_TYPE.TSK_DATETIME_ACCESSED.getTypeID();
private static int URL_TYPEID = ATTRIBUTE_TYPE.TSK_URL.getTypeID();
private static final Logger logger = Logger.getLogger(DomainCategoryRunner.class.getName());
/**
* Get seconds from epoch from the mapping for the attribute type id.
*
* @param attrMap A mapping of attribute type id to BlackboardAttribute for
* an artifact.
* @param attrTypeId The attribute type id to fetch.
* @return The time in seconds from epoch or 0 if cannot be found.
*/
private static long getTimeOrZero(Map<Integer, BlackboardAttribute> attrMap, int attrTypeId) {
if (attrMap == null) {
return 0;
}
BlackboardAttribute attr = attrMap.get(attrTypeId);
return attr == null ? 0 : attr.getValueLong();
}
/**
* Get string for attribute type id or "" if cannot be determined.
*
* @param attrMap A mapping of attribute type id to BlackboardAttribute for
* an artifact.
* @param attrTypeId The attribute type id to fetch.
* @return The string value or "" if cannot be determined or null.
*/
private static String getStringOrEmpty(Map<Integer, BlackboardAttribute> attrMap, int attrTypeId) {
if (attrMap == null) {
return "";
}
BlackboardAttribute attr = attrMap.get(attrTypeId);
String attrStr = attr == null ? "" : attr.getValueString();
return attrStr == null ? "" : attrStr;
}
/**
* Comparator for ensuring deterministic order of processing artifacts.
*/
private static final Comparator<BlackboardArtifact> ARTIFACT_COMPARATOR = (a, b) -> {
// get attributes in map by type id
Map<Integer, BlackboardAttribute> attrMapA = null;
Map<Integer, BlackboardAttribute> attrMapB = null;
try {
attrMapA = a.getAttributes()
.stream()
.collect(Collectors.toMap(attr -> attr.getAttributeType().getTypeID(), attr -> attr, (attr1, attr2) -> attr1));
attrMapB = b.getAttributes()
.stream()
.collect(Collectors.toMap(attr -> attr.getAttributeType().getTypeID(), attr -> attr, (attr1, attr2) -> attr1));
} catch (TskCoreException ex) {
logger.log(Level.WARNING, "There was an error fetching attributes for artifacts", ex);
return 0;
}
// sort first on time
int timeCompare = Long.compare(getTimeOrZero(attrMapA, DATETIME_ACCESSED_TYPEID), getTimeOrZero(attrMapB, DATETIME_ACCESSED_TYPEID));
if (timeCompare != 0) {
// negate to push latest times to the front
return -timeCompare;
}
// sort next on url
int urlCompare = getStringOrEmpty(attrMapA, URL_TYPEID).compareToIgnoreCase(getStringOrEmpty(attrMapB, URL_TYPEID));
if (urlCompare != 0) {
return urlCompare;
}
// use id as last resort
return Long.compare(a.getId(), b.getId());
};
private Content dataSource;
private IngestJobContext context;
private List<DomainCategorizer> domainProviders = Collections.emptyList();
@ -272,11 +351,12 @@ class DomainCategoryRunner extends Extract {
// only one suffix per ingest is captured so this tracks the suffixes seen.
Set<String> hostSuffixesSeen = new HashSet<>();
try {
Collection<BlackboardArtifact> listArtifacts = currentCase.getSleuthkitCase().getBlackboard().getArtifacts(
List<BlackboardArtifact> listArtifacts = currentCase.getSleuthkitCase().getBlackboard().getArtifacts(
Arrays.asList(new BlackboardArtifact.Type(ARTIFACT_TYPE.TSK_WEB_HISTORY)),
Arrays.asList(dataSource.getId()));
logger.log(Level.INFO, "Processing {0} blackboard artifacts.", listArtifacts.size()); //NON-NLS
Collections.sort(listArtifacts, ARTIFACT_COMPARATOR);
for (BlackboardArtifact artifact : listArtifacts) {
// make sure we haven't cancelled
@ -361,11 +441,11 @@ class DomainCategoryRunner extends Extract {
if (lookupList == null) {
lookupList = Collections.emptyList();
}
List<DomainCategorizer> foundProviders = lookupList.stream()
.filter(provider -> provider != null)
.sorted((a, b) -> a.getClass().getName().compareToIgnoreCase(b.getClass().getName()))
.collect(Collectors.toList());
.filter(provider -> provider != null)
.sorted((a, b) -> a.getClass().getName().compareToIgnoreCase(b.getClass().getName()))
.collect(Collectors.toList());
// add the default categorizer last as a last resort
foundProviders.add(new DefaultDomainCategorizer());

2
RecentActivity/src/org/sleuthkit/autopsy/recentactivity/SearchEngineURLQueryAnalyzer.java
View File

@ -247,7 +247,7 @@ class SearchEngineURLQueryAnalyzer extends Extract {
}
}
try { //try to decode the url
String decoded = URLDecoder.decode(x, "UTF-8"); //NON-NLS
String decoded = URLDecoder.decode(x.replaceAll("%(?![0-9a-fA-F]{2})", "%25"), "UTF-8"); //NON-NLS
return decoded;
} catch (UnsupportedEncodingException exception) { //if it fails, return the encoded string
logger.log(Level.FINE, "Error during URL decoding, returning undecoded value:"

23
thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/MboxParser.java
View File

@ -35,6 +35,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.logging.Level;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.modules.filetypeid.FileTypeDetector;
import org.apache.james.mime4j.dom.Message;
import org.apache.james.mime4j.mboxiterator.CharBufferWrapper;
import org.apache.james.mime4j.mboxiterator.MboxIterator;
@ -42,6 +43,7 @@ import org.apache.tika.parser.txt.CharsetDetector;
import org.apache.tika.parser.txt.CharsetMatch;
import org.apache.commons.validator.routines.EmailValidator;
import org.openide.util.NbBundle;
import org.sleuthkit.datamodel.AbstractFile;
/**
* An Iterator for parsing mbox files. Wraps an instance of MBoxEmailIterator.
@ -56,12 +58,25 @@ class MboxParser extends MimeJ4MessageParser implements Iterator<EmailMessage> {
setLocalPath(localPath);
}
static boolean isValidMimeTypeMbox(byte[] buffer) {
static boolean isValidMimeTypeMbox(byte[] buffer, AbstractFile abstractFile) {
String mboxHeaderLine = new String(buffer);
if (mboxHeaderLine.startsWith("From ")) {
String[] mboxLineValues = mboxHeaderLine.split(" ");
EmailValidator validator = EmailValidator.getInstance(true, true);
return validator.isValid(mboxLineValues[1]);
String mimeType = abstractFile.getMIMEType();
// if it is not present, attempt to use the FileTypeDetector to determine
if (mimeType == null || mimeType.isEmpty()) {
FileTypeDetector fileTypeDetector = null;
try {
fileTypeDetector = new FileTypeDetector();
} catch (FileTypeDetector.FileTypeDetectorInitException ex) {
logger.log(Level.WARNING, String.format("Unable to create file type detector for determining MIME type for file %s with id of %d", abstractFile.getName(), abstractFile.getId()));
return false;
}
mimeType = fileTypeDetector.getMIMEType(abstractFile);
}
if (mimeType.equalsIgnoreCase("application/mbox")) {
return true;
}
}
return false; //NON-NLS
}

2
thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java
View File

@ -130,7 +130,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule {
if (abstractFile.getSize() > 64) {
int byteRead = abstractFile.read(t, 0, 64);
if (byteRead > 0) {
isMbox = MboxParser.isValidMimeTypeMbox(t);
isMbox = MboxParser.isValidMimeTypeMbox(t, abstractFile);
isEMLFile = EMLParser.isEMLFile(abstractFile, t);
}
}