diff --git a/.gitattributes b/.gitattributes index cd5271c982..8749e5dadb 100644 --- a/.gitattributes +++ b/.gitattributes @@ -13,3 +13,9 @@ Doxyfile text *.py text diff=python *.pl text + +# ensure solr scripts that are bash scripts not ending with.sh are lf instead of crlf +/KeywordSearch/solr/bin/autopsy-solr eol=lf +/KeywordSearch/solr/bin/init.d/solr eol=lf +/KeywordSearch/solr/bin/post eol=lf +/KeywordSearch/solr/bin/solr eol=lf \ No newline at end of file diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/AddImageTask.java b/Core/src/org/sleuthkit/autopsy/casemodule/AddImageTask.java index 53320d9313..28ad688b1a 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/AddImageTask.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/AddImageTask.java @@ -32,6 +32,7 @@ import org.sleuthkit.autopsy.imagewriter.ImageWriterService; import org.sleuthkit.autopsy.imagewriter.ImageWriterSettings; import org.sleuthkit.datamodel.AddDataSourceCallbacks; import org.sleuthkit.datamodel.Content; +import org.sleuthkit.datamodel.Host; import org.sleuthkit.datamodel.Image; import org.sleuthkit.datamodel.SleuthkitJNI; import org.sleuthkit.datamodel.TskCoreException; @@ -106,7 +107,7 @@ class AddImageTask implements Runnable { try { synchronized (tskAddImageProcessLock) { if (!tskAddImageProcessStopped) { - tskAddImageProcess = currentCase.getSleuthkitCase().makeAddImageProcess(imageDetails.timeZone, true, imageDetails.ignoreFatOrphanFiles, imageWriterPath); + tskAddImageProcess = currentCase.getSleuthkitCase().makeAddImageProcess(imageDetails.timeZone, true, imageDetails.ignoreFatOrphanFiles, imageDetails.host, imageWriterPath); } else { return; } @@ -317,9 +318,10 @@ class AddImageTask implements Runnable { String md5; String sha1; String sha256; + Host host; ImageWriterSettings imageWriterSettings; - ImageDetails(String deviceId, Image image, int sectorSize, String timeZone, boolean ignoreFatOrphanFiles, String md5, String sha1, String sha256, ImageWriterSettings imageWriterSettings) { + ImageDetails(String deviceId, Image image, int sectorSize, String timeZone, boolean ignoreFatOrphanFiles, String md5, String sha1, String sha256, Host host, ImageWriterSettings imageWriterSettings) { this.deviceId = deviceId; this.image = image; this.sectorSize = sectorSize; @@ -328,6 +330,7 @@ class AddImageTask implements Runnable { this.md5 = md5; this.sha1 = sha1; this.sha256 = sha256; + this.host = host; this.imageWriterSettings = imageWriterSettings; } diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/AddLocalFilesTask.java b/Core/src/org/sleuthkit/autopsy/casemodule/AddLocalFilesTask.java index 185f696a97..6e4a71440f 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/AddLocalFilesTask.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/AddLocalFilesTask.java @@ -28,6 +28,7 @@ import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorProgress import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.Content; +import org.sleuthkit.datamodel.Host; import org.sleuthkit.datamodel.LocalFilesDataSource; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskDataException; @@ -42,6 +43,7 @@ class AddLocalFilesTask implements Runnable { private static final Logger LOGGER = Logger.getLogger(AddLocalFilesTask.class.getName()); private final String deviceId; private final String rootVirtualDirectoryName; + private final Host host; private final List localFilePaths; private final DataSourceProcessorProgressMonitor progress; private final DataSourceProcessorCallback callback; @@ -64,14 +66,16 @@ class AddLocalFilesTask implements Runnable { * form: LogicalFileSet[N] * @param localFilePaths A list of localFilePaths of local/logical * files and/or directories. + * @param host The host for this data source (may be null). * @param progressMonitor Progress monitor to report progress * during processing. * @param callback Callback to call when processing is done. */ - AddLocalFilesTask(String deviceId, String rootVirtualDirectoryName, List localFilePaths, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callback) { + AddLocalFilesTask(String deviceId, String rootVirtualDirectoryName, List localFilePaths, Host host, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callback) { this.deviceId = deviceId; this.rootVirtualDirectoryName = rootVirtualDirectoryName; this.localFilePaths = localFilePaths; + this.host = host; this.callback = callback; this.progress = progressMonitor; } @@ -88,7 +92,7 @@ class AddLocalFilesTask implements Runnable { try { progress.setIndeterminate(true); FileManager fileManager = Case.getCurrentCaseThrows().getServices().getFileManager(); - LocalFilesDataSource newDataSource = fileManager.addLocalFilesDataSource(deviceId, rootVirtualDirectoryName, "", localFilePaths, new ProgressUpdater()); + LocalFilesDataSource newDataSource = fileManager.addLocalFilesDataSource(deviceId, rootVirtualDirectoryName, "", host, localFilePaths, new ProgressUpdater()); newDataSources.add(newDataSource); } catch (TskDataException | TskCoreException | NoCurrentCaseException ex) { errors.add(ex.getMessage()); diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/ImageDSProcessor.java b/Core/src/org/sleuthkit/autopsy/casemodule/ImageDSProcessor.java index 207b83ed8a..2bb2cebbbd 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/ImageDSProcessor.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/ImageDSProcessor.java @@ -39,6 +39,7 @@ import org.sleuthkit.autopsy.datasourceprocessors.AutoIngestDataSourceProcessor; import org.sleuthkit.autopsy.ingest.IngestJobSettings; import org.sleuthkit.autopsy.ingest.IngestManager; import org.sleuthkit.autopsy.ingest.IngestStream; +import org.sleuthkit.datamodel.Host; import org.sleuthkit.datamodel.Image; import org.sleuthkit.datamodel.SleuthkitJNI; import org.sleuthkit.datamodel.TskCoreException; @@ -81,6 +82,7 @@ public class ImageDSProcessor implements DataSourceProcessor, AutoIngestDataSour private String md5; private String sha1; private String sha256; + private Host host = null; private boolean setDataSourceOptionsCalled; static { @@ -181,11 +183,32 @@ public class ImageDSProcessor implements DataSourceProcessor, AutoIngestDataSour */ @Override public void run(DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callback) { + run(null, progressMonitor, callback); + } + + /** + * Adds a data source to the case database using a background task in a + * separate thread and the settings provided by the selection and + * configuration panel. Returns as soon as the background task is started. + * The background task uses a callback object to signal task completion and + * return results. + * + * This method should not be called unless isPanelValid returns true. + * + * @param host Host for this data source. + * @param progressMonitor Progress monitor that will be used by the + * background task to report progress. + * @param callback Callback that will be used by the background task + * to return results. + */ + @Override + public void run(Host host, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callback) { ingestStream = new DefaultIngestStream(); readConfigSettings(); + this.host = host; try { image = SleuthkitJNI.addImageToDatabase(Case.getCurrentCase().getSleuthkitCase(), - new String[]{imagePath}, sectorSize, timeZone, md5, sha1, sha256, deviceId); + new String[]{imagePath}, sectorSize, timeZone, md5, sha1, sha256, deviceId, this.host); } catch (TskCoreException ex) { logger.log(Level.SEVERE, "Error adding data source with path " + imagePath + " to database", ex); final List errors = new ArrayList<>(); @@ -217,14 +240,47 @@ public class ImageDSProcessor implements DataSourceProcessor, AutoIngestDataSour @Override public void runWithIngestStream(IngestJobSettings settings, DataSourceProcessorProgressMonitor progress, DataSourceProcessorCallback callBack) { + runWithIngestStream(null, settings, progress, callBack); + } + + /** + * Adds a data source to the case database using a background task in a + * separate thread and the settings provided by the selection and + * configuration panel. Files found during ingest will be sent directly to the + * IngestStream provided. Returns as soon as the background task is started. + * The background task uses a callback object to signal task completion and + * return results. + * + * This method should not be called unless isPanelValid returns true, and + * should only be called for DSPs that support ingest streams. + * + * @param host The host for this data source. + * @param settings The ingest job settings. + * @param progress Progress monitor that will be used by the + * background task to report progress. + * @param callBack Callback that will be used by the background task + * to return results. + */ + @Override + public void runWithIngestStream(Host host, IngestJobSettings settings, DataSourceProcessorProgressMonitor progress, + DataSourceProcessorCallback callBack) { // Read the settings from the wizard readConfigSettings(); + this.host = host; + + // HOSTTODO - remove once passing in a host + try { + this.host = Case.getCurrentCase().getSleuthkitCase().getHostManager().getOrCreateHost("ImageDSProcessor Host"); + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, "Error creating/loading host", ex); + this.host = null; + } // Set up the data source before creating the ingest stream try { image = SleuthkitJNI.addImageToDatabase(Case.getCurrentCase().getSleuthkitCase(), - new String[]{imagePath}, sectorSize, timeZone, md5, sha1, sha256, deviceId); + new String[]{imagePath}, sectorSize, timeZone, md5, sha1, sha256, deviceId, this.host); } catch (TskCoreException ex) { logger.log(Level.SEVERE, "Error adding data source with path " + imagePath + " to database", ex); final List errors = new ArrayList<>(); @@ -370,7 +426,7 @@ public class ImageDSProcessor implements DataSourceProcessor, AutoIngestDataSour return; } - AddImageTask.ImageDetails imageDetails = new AddImageTask.ImageDetails(deviceId, image, sectorSize, timeZone, ignoreFatOrphanFiles, md5, sha1, sha256, null); + AddImageTask.ImageDetails imageDetails = new AddImageTask.ImageDetails(deviceId, image, sectorSize, timeZone, ignoreFatOrphanFiles, md5, sha1, sha256, host, null); addImageTask = new AddImageTask(imageDetails, progressMonitor, new StreamingAddDataSourceCallbacks(ingestStream), @@ -405,6 +461,7 @@ public class ImageDSProcessor implements DataSourceProcessor, AutoIngestDataSour imagePath = null; timeZone = null; ignoreFatOrphanFiles = false; + host = null; configPanel.reset(); setDataSourceOptionsCalled = false; } @@ -428,7 +485,6 @@ public class ImageDSProcessor implements DataSourceProcessor, AutoIngestDataSour try { // verify that the image has a file system that TSK can process - Case currentCase = Case.getCurrentCaseThrows(); if (!DataSourceUtils.imageHasFileSystem(dataSourcePath)) { // image does not have a file system that TSK can process return 0; @@ -521,6 +577,7 @@ public class ImageDSProcessor implements DataSourceProcessor, AutoIngestDataSour this.sectorSize = 0; this.timeZone = Calendar.getInstance().getTimeZone().getID(); this.ignoreFatOrphanFiles = ignoreFatOrphanFiles; + this.host = null; setDataSourceOptionsCalled = true; } diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/LocalDiskDSProcessor.java b/Core/src/org/sleuthkit/autopsy/casemodule/LocalDiskDSProcessor.java index 0551d15466..209cdf56b7 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/LocalDiskDSProcessor.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/LocalDiskDSProcessor.java @@ -31,6 +31,7 @@ import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorProgress import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessor; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.imagewriter.ImageWriterSettings; +import org.sleuthkit.datamodel.Host; import org.sleuthkit.datamodel.Image; import org.sleuthkit.datamodel.SleuthkitJNI; import org.sleuthkit.datamodel.TskCoreException; @@ -56,6 +57,7 @@ public class LocalDiskDSProcessor implements DataSourceProcessor { private String drivePath; private int sectorSize; private String timeZone; + private Host host; private ImageWriterSettings imageWriterSettings; private boolean ignoreFatOrphanFiles; private boolean setDataSourceOptionsCalled; @@ -135,6 +137,26 @@ public class LocalDiskDSProcessor implements DataSourceProcessor { */ @Override public void run(DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callback) { + run(null, progressMonitor, callback); + } + + /** + * Adds a data source to the case database using a background task in a + * separate thread and the settings provided by the selection and + * configuration panel. Returns as soon as the background task is started. + * The background task uses a callback object to signal task completion and + * return results. + * + * This method should not be called unless isPanelValid returns true. + * + * @param host Host for this data source. + * @param progressMonitor Progress monitor that will be used by the + * background task to report progress. + * @param callback Callback that will be used by the background task + * to return results. + */ + @Override + public void run(Host host, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callback) { if (!setDataSourceOptionsCalled) { deviceId = UUID.randomUUID().toString(); drivePath = configPanel.getContentPath(); @@ -147,12 +169,21 @@ public class LocalDiskDSProcessor implements DataSourceProcessor { imageWriterSettings = null; } } + + this.host = host; + // HOSTTODO - set to value from config panel + try { + this.host = Case.getCurrentCase().getSleuthkitCase().getHostManager().getOrCreateHost("LocalDiskDSProcessor Host"); + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, "Error creating/loading host", ex); + this.host = null; + } Image image; try { image = SleuthkitJNI.addImageToDatabase(Case.getCurrentCase().getSleuthkitCase(), new String[]{drivePath}, sectorSize, - timeZone, null, null, null, deviceId); + timeZone, null, null, null, deviceId, this.host); } catch (TskCoreException ex) { logger.log(Level.SEVERE, "Error adding local disk with path " + drivePath + " to database", ex); final List errors = new ArrayList<>(); @@ -162,7 +193,7 @@ public class LocalDiskDSProcessor implements DataSourceProcessor { } addDiskTask = new AddImageTask( - new AddImageTask.ImageDetails(deviceId, image, sectorSize, timeZone, ignoreFatOrphanFiles, null, null, null, imageWriterSettings), + new AddImageTask.ImageDetails(deviceId, image, sectorSize, timeZone, ignoreFatOrphanFiles, null, null, null, this.host, imageWriterSettings), progressMonitor, new StreamingAddDataSourceCallbacks(new DefaultIngestStream()), new StreamingAddImageTaskCallback(new DefaultIngestStream(), callback)); @@ -230,7 +261,7 @@ public class LocalDiskDSProcessor implements DataSourceProcessor { return; } - addDiskTask = new AddImageTask(new AddImageTask.ImageDetails(deviceId, image, sectorSize, timeZone, ignoreFatOrphanFiles, null, null, null, imageWriterSettings), + addDiskTask = new AddImageTask(new AddImageTask.ImageDetails(deviceId, image, sectorSize, timeZone, ignoreFatOrphanFiles, null, null, null, null, imageWriterSettings), progressMonitor, new StreamingAddDataSourceCallbacks(new DefaultIngestStream()), new StreamingAddImageTaskCallback(new DefaultIngestStream(), callback)); diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/LocalFilesDSProcessor.java b/Core/src/org/sleuthkit/autopsy/casemodule/LocalFilesDSProcessor.java index 4d5e3eac36..5a633d0191 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/LocalFilesDSProcessor.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/LocalFilesDSProcessor.java @@ -42,6 +42,8 @@ import org.sleuthkit.autopsy.coreutils.ExecUtil; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.PlatformUtil; import org.sleuthkit.autopsy.datasourceprocessors.AutoIngestDataSourceProcessor; +import org.sleuthkit.datamodel.Host; +import org.sleuthkit.datamodel.TskCoreException; /** * A local/logical files/logical evidence file(.lo1)/or directories data source @@ -137,7 +139,7 @@ public class LocalFilesDSProcessor implements DataSourceProcessor, AutoIngestDat return configPanel.validatePanel(); } - /** + /** * Adds a data source to the case database using a background task in a * separate thread and the settings provided by the selection and * configuration panel. Returns as soon as the background task is started. @@ -153,7 +155,36 @@ public class LocalFilesDSProcessor implements DataSourceProcessor, AutoIngestDat */ @Override public void run(DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callback) { + run(null, progressMonitor, callback); + } + + /** + * Adds a data source to the case database using a background task in a + * separate thread and the settings provided by the selection and + * configuration panel. Returns as soon as the background task is started. + * The background task uses a callback object to signal task completion and + * return results. + * + * This method should not be called unless isPanelValid returns true. + * + * @param host Host for this data source. + * @param progressMonitor Progress monitor that will be used by the + * background task to report progress. + * @param callback Callback that will be used by the background task + * to return results. + */ + @Override + public void run(Host host, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callback) { if (!setDataSourceOptionsCalled) { + + // HOSTTODO - use passed in value + try { + host = Case.getCurrentCase().getSleuthkitCase().getHostManager().getOrCreateHost("LocalFilesDSProcessor Host"); + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, "Error creating/loading host", ex); + host = null; + } + localFilePaths = configPanel.getContentPaths(); if (configPanel.subTypeIsLogicalEvidencePanel()) { try { @@ -171,7 +202,7 @@ public class LocalFilesDSProcessor implements DataSourceProcessor, AutoIngestDat } } } - run(UUID.randomUUID().toString(), configPanel.getFileSetName(), localFilePaths, progressMonitor, callback); + run(UUID.randomUUID().toString(), configPanel.getFileSetName(), localFilePaths, host, progressMonitor, callback); } /** @@ -284,6 +315,34 @@ public class LocalFilesDSProcessor implements DataSourceProcessor, AutoIngestDat return executablePath; } + + /** + * Adds a data source to the case database using a background task in a + * separate thread and the given settings instead of those provided by the + * selection and configuration panel. Returns as soon as the background task + * is started and uses the callback object to signal task completion and + * return results. + * + * @param deviceId An ASCII-printable identifier for the + * device associated with the data source + * that is intended to be unique across + * multiple cases (e.g., a UUID). + * @param rootVirtualDirectoryName The name to give to the virtual directory + * that will serve as the root for the + * local/logical files and/or directories + * that compose the data source. Pass the + * empty string to get a default name of the + * form: LogicalFileSet[N] + * @param localFilePaths A list of local/logical file and/or + * directory localFilePaths. + * @param host The host for this data source. + * @param progressMonitor Progress monitor for reporting progress + * during processing. + * @param callback Callback to call when processing is done. + */ + void run(String deviceId, String rootVirtualDirectoryName, List localFilePaths, Host host, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callback) { + new Thread(new AddLocalFilesTask(deviceId, rootVirtualDirectoryName, localFilePaths, host, progressMonitor, callback)).start(); + } /** * Adds a data source to the case database using a background task in a @@ -309,7 +368,7 @@ public class LocalFilesDSProcessor implements DataSourceProcessor, AutoIngestDat * @param callback Callback to call when processing is done. */ public void run(String deviceId, String rootVirtualDirectoryName, List localFilePaths, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callback) { - new Thread(new AddLocalFilesTask(deviceId, rootVirtualDirectoryName, localFilePaths, progressMonitor, callback)).start(); + run(deviceId, rootVirtualDirectoryName, localFilePaths, null, progressMonitor, callback); } /** diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/services/FileManager.java b/Core/src/org/sleuthkit/autopsy/casemodule/services/FileManager.java index ece1c00736..1335d24b26 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/services/FileManager.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/services/FileManager.java @@ -36,6 +36,7 @@ import org.sleuthkit.autopsy.ingest.ModuleContentEvent; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.DerivedFile; +import org.sleuthkit.datamodel.Host; import org.sleuthkit.datamodel.LayoutFile; import org.sleuthkit.datamodel.LocalDirectory; import org.sleuthkit.datamodel.SleuthkitCase; @@ -490,6 +491,40 @@ public class FileManager implements Closeable { * directory that does not exist or cannot be read. */ public synchronized LocalFilesDataSource addLocalFilesDataSource(String deviceId, String rootVirtualDirectoryName, String timeZone, List localFilePaths, FileAddProgressUpdater progressUpdater) throws TskCoreException, TskDataException { + return addLocalFilesDataSource(deviceId, rootVirtualDirectoryName, timeZone, null, localFilePaths, progressUpdater); + } + + /** + * Adds a set of local/logical files and/or directories to the case database + * as data source. + * + * @param deviceId An ASCII-printable identifier for the + * device associated with the data source + * that is intended to be unique across + * multiple cases (e.g., a UUID). + * @param rootVirtualDirectoryName The name to give to the virtual directory + * that will serve as the root for the + * local/logical files and/or directories + * that compose the data source. Pass the + * empty string to get a default name of the + * form: LogicalFileSet[N] + * @param timeZone The time zone used to process the data + * source, may be the empty string. + * @param host The host for this data source (may be null). + * @param localFilePaths A list of local/logical file and/or + * directory localFilePaths. + * @param progressUpdater Called after each file/directory is added + * to the case database. + * + * @return A local files data source object. + * + * @throws TskCoreException If there is a problem completing a database + * operation. + * @throws TskDataException if any of the local file paths is for a file or + * directory that does not exist or cannot be read. + */ + public synchronized LocalFilesDataSource addLocalFilesDataSource(String deviceId, String rootVirtualDirectoryName, String timeZone, Host host, + List localFilePaths, FileAddProgressUpdater progressUpdater) throws TskCoreException, TskDataException { if (null == caseDb) { throw new TskCoreException("File manager has been closed"); } @@ -506,7 +541,7 @@ public class FileManager implements Closeable { * children to the case database. */ trans = caseDb.beginTransaction(); - LocalFilesDataSource dataSource = caseDb.addLocalFilesDataSource(deviceId, rootDirectoryName, timeZone, trans); + LocalFilesDataSource dataSource = caseDb.addLocalFilesDataSource(deviceId, rootDirectoryName, timeZone, host, trans); List filesAdded = new ArrayList<>(); for (java.io.File localFile : localFiles) { AbstractFile fileAdded = addLocalFile(trans, dataSource, localFile, TskData.EncodingType.NONE, progressUpdater); diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/services/TagNameDefinition.java b/Core/src/org/sleuthkit/autopsy/casemodule/services/TagNameDefinition.java index 17cff6d900..ff07662e64 100755 --- a/Core/src/org/sleuthkit/autopsy/casemodule/services/TagNameDefinition.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/services/TagNameDefinition.java @@ -61,19 +61,19 @@ final public class TagNameDefinition implements Comparable { private final TskData.FileKnown knownStatus; private static final List STANDARD_TAGS_DEFINITIONS = new ArrayList<>(); - private static final List OLD_CATEGORY_TAG_NAMES = new ArrayList<>(); + private static final List PROJECT_VIC_NAMES_NO_LONGER_USED = new ArrayList<>(); static { STANDARD_TAGS_DEFINITIONS.add(new TagNameDefinition(Bundle.TagNameDefinition_predefTagNames_bookmark_text(), "", TagName.HTML_COLOR.NONE, TskData.FileKnown.UNKNOWN)); STANDARD_TAGS_DEFINITIONS.add(new TagNameDefinition(Bundle.TagNameDefinition_predefTagNames_followUp_text(), "", TagName.HTML_COLOR.NONE, TskData.FileKnown.UNKNOWN)); STANDARD_TAGS_DEFINITIONS.add(new TagNameDefinition(Bundle.TagNameDefinition_predefTagNames_notableItem_text(), "", TagName.HTML_COLOR.NONE, TskData.FileKnown.BAD)); - OLD_CATEGORY_TAG_NAMES.add("CAT-1: Child Exploitation (Illegal)"); - OLD_CATEGORY_TAG_NAMES.add("CAT-2: Child Exploitation (Non-Illegal/Age Difficult)"); - OLD_CATEGORY_TAG_NAMES.add("CAT-3: CGI/Animation (Child Exploitive)"); - OLD_CATEGORY_TAG_NAMES.add("CAT-4: Exemplar/Comparison (Internal Use Only)"); - OLD_CATEGORY_TAG_NAMES.add("CAT-5: Non-pertinent"); - OLD_CATEGORY_TAG_NAMES.add("CAT-0: Uncategorized"); + PROJECT_VIC_NAMES_NO_LONGER_USED.add("CAT-1: Child Exploitation (Illegal)"); + PROJECT_VIC_NAMES_NO_LONGER_USED.add("CAT-2: Child Exploitation (Non-Illegal/Age Difficult)"); + PROJECT_VIC_NAMES_NO_LONGER_USED.add("CAT-3: CGI/Animation (Child Exploitive)"); + PROJECT_VIC_NAMES_NO_LONGER_USED.add("CAT-4: Exemplar/Comparison (Internal Use Only)"); + PROJECT_VIC_NAMES_NO_LONGER_USED.add("CAT-5: Non-pertinent"); + PROJECT_VIC_NAMES_NO_LONGER_USED.add("CAT-0: Uncategorized"); } /** @@ -259,7 +259,7 @@ final public class TagNameDefinition implements Comparable { */ static synchronized Set getTagNameDefinitions() { if (needsVersionUpdate()) { - updateTagDefinitions(); + updatePropertyFile(); } String tagsProperty = ModuleSettings.getConfigSetting(TAGS_SETTINGS_NAME, TAG_NAMES_SETTING_KEY); @@ -311,7 +311,7 @@ final public class TagNameDefinition implements Comparable { /** * Updates the Tag Definition file to the current format. */ - private static void updateTagDefinitions() { + private static void updatePropertyFile() { Integer version = getPropertyFileVersion(); List definitions = new ArrayList<>(); @@ -355,18 +355,18 @@ final public class TagNameDefinition implements Comparable { } // Remove the standard and Project VIC tags from the list - List tagStrings = new ArrayList<>(); + List tagStringsToKeep = new ArrayList<>(); List standardTags = getStandardTagNames(); for (TagNameDefinition def : definitions) { if (!standardTags.contains(def.getDisplayName()) - && !OLD_CATEGORY_TAG_NAMES.contains(def.getDisplayName())) { - tagStrings.add(def.toSettingsFormat()); + && !PROJECT_VIC_NAMES_NO_LONGER_USED.contains(def.getDisplayName())) { + tagStringsToKeep.add(def.toSettingsFormat()); } } // Write out the version and the new tag list. ModuleSettings.setConfigSetting(TAGS_SETTINGS_NAME, TAG_SETTING_VERSION_KEY, Integer.toString(TAG_SETTINGS_VERSION)); - ModuleSettings.setConfigSetting(TAGS_SETTINGS_NAME, TAG_NAMES_SETTING_KEY, String.join(";", tagStrings)); + ModuleSettings.setConfigSetting(TAGS_SETTINGS_NAME, TAG_NAMES_SETTING_KEY, String.join(";", tagStringsToKeep)); } /** diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/services/TagSetDefinition.java b/Core/src/org/sleuthkit/autopsy/casemodule/services/TagSetDefinition.java index ad11becdff..6303e2736e 100755 --- a/Core/src/org/sleuthkit/autopsy/casemodule/services/TagSetDefinition.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/services/TagSetDefinition.java @@ -88,7 +88,7 @@ final public class TagSetDefinition { } /** - * Returns a list of the defined TagSet objects. + * Returns a list of configured TagSets (from the user's config folder) * * @return A list of TagSetDefinition objects or empty list if none were * found. diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/services/TagsManager.java b/Core/src/org/sleuthkit/autopsy/casemodule/services/TagsManager.java index 1a9b2e792d..4bb813a779 100755 --- a/Core/src/org/sleuthkit/autopsy/casemodule/services/TagsManager.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/services/TagsManager.java @@ -55,7 +55,9 @@ public class TagsManager implements Closeable { private static final Logger LOGGER = Logger.getLogger(TagsManager.class.getName()); private final SleuthkitCase caseDb; - private static String DEFAULT_TAG_SET_NAME = "Project VIC"; + // NOTE: This name is also hard coded in Image Gallery and Projet Vic module. + // They need to stay in sync + private static String PROJECT_VIC_TAG_SET_NAME = "Project VIC"; private static final Object lock = new Object(); @@ -196,7 +198,7 @@ public class TagsManager implements Closeable { try { List tagSetList = Case.getCurrentCaseThrows().getSleuthkitCase().getTaggingManager().getTagSets(); for (TagSet tagSet : tagSetList) { - if (tagSet.getName().equals(DEFAULT_TAG_SET_NAME)) { + if (tagSet.getName().equals(PROJECT_VIC_TAG_SET_NAME)) { for (TagName tagName : tagSet.getTagNames()) { tagList.add(tagName.getDisplayName()); } @@ -237,7 +239,7 @@ public class TagsManager implements Closeable { } /** - * Creates a new TagSetDefinition file. + * Creates a new TagSetDefinition file that will be used for future cases * * @param tagSetDef The tag set definition. * @@ -258,23 +260,26 @@ public class TagsManager implements Closeable { TagsManager(SleuthkitCase caseDb) { this.caseDb = caseDb; - // Add standard tags and the Project VIC default tag set and tags. + // Add standard tags and any configured tag sets. TaggingManager taggingMgr = caseDb.getTaggingManager(); try { - List setList = taggingMgr.getTagSets(); - if (setList.isEmpty()) { + List tagSetsInCase = taggingMgr.getTagSets(); + if (tagSetsInCase.isEmpty()) { + + // add the standard tag names for (TagNameDefinition def : TagNameDefinition.getStandardTagNameDefinitions()) { caseDb.addOrUpdateTagName(def.getDisplayName(), def.getDescription(), def.getColor(), def.getKnownStatus()); } - //Assume new case and add tag sets + + //Assume new case and add all tag sets for (TagSetDefinition setDef : TagSetDefinition.readTagSetDefinitions()) { - List tagNameList = new ArrayList<>(); + List tagNamesInSet = new ArrayList<>(); for (TagNameDefinition tagNameDef : setDef.getTagNameDefinitions()) { - tagNameList.add(caseDb.addOrUpdateTagName(tagNameDef.getDisplayName(), tagNameDef.getDescription(), tagNameDef.getColor(), tagNameDef.getKnownStatus())); + tagNamesInSet.add(caseDb.addOrUpdateTagName(tagNameDef.getDisplayName(), tagNameDef.getDescription(), tagNameDef.getColor(), tagNameDef.getKnownStatus())); } - if (!tagNameList.isEmpty()) { - taggingMgr.addTagSet(setDef.getName(), tagNameList); + if (!tagNamesInSet.isEmpty()) { + taggingMgr.addTagSet(setDef.getName(), tagNamesInSet); } } } diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeNormalizer.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeNormalizer.java index 51b9b80f84..f7d423c85d 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeNormalizer.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeNormalizer.java @@ -27,6 +27,7 @@ import java.util.Set; import org.apache.commons.lang.StringUtils; import org.apache.commons.validator.routines.DomainValidator; import org.apache.commons.validator.routines.EmailValidator; +import org.sleuthkit.autopsy.coreutils.NetworkUtils; /** * Provides functions for normalizing data by attribute type before insertion or @@ -144,11 +145,11 @@ final public class CorrelationAttributeNormalizer { private static String normalizeDomain(String data) throws CorrelationAttributeNormalizationException { DomainValidator validator = DomainValidator.getInstance(true); if (validator.isValid(data)) { - return data.toLowerCase(); + return NetworkUtils.extractDomain(data.toLowerCase()); } else { final String validIpAddressRegex = "^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$"; if (data.matches(validIpAddressRegex)) { - return data; + return NetworkUtils.extractDomain(data); } else { throw new CorrelationAttributeNormalizationException(String.format("Data was expected to be a valid domain: %s", data)); } diff --git a/Core/src/org/sleuthkit/autopsy/core/UserPreferences.java b/Core/src/org/sleuthkit/autopsy/core/UserPreferences.java index 0d3fd0198a..537c25c827 100644 --- a/Core/src/org/sleuthkit/autopsy/core/UserPreferences.java +++ b/Core/src/org/sleuthkit/autopsy/core/UserPreferences.java @@ -28,6 +28,7 @@ import org.openide.util.NbPreferences; import org.python.icu.util.TimeZone; import org.sleuthkit.autopsy.machinesettings.UserMachinePreferences; import org.sleuthkit.autopsy.coreutils.ModuleSettings; +import org.sleuthkit.autopsy.coreutils.PlatformUtil; import org.sleuthkit.autopsy.coreutils.TextConverterException; import org.sleuthkit.autopsy.coreutils.Version; import org.sleuthkit.datamodel.CaseDbConnectionInfo; @@ -84,7 +85,8 @@ public final class UserPreferences { private static final boolean DISPLAY_TRANSLATED_NAMES_DEFAULT = true; public static final String EXTERNAL_HEX_EDITOR_PATH = "ExternalHexEditorPath"; public static final String SOLR_MAX_JVM_SIZE = "SolrMaxJVMSize"; - private static final int DEFAULT_SOLR_HEAP_SIZE_MB = 2048; + private static final int DEFAULT_SOLR_HEAP_SIZE_MB_64BIT_PLATFORM = 2048; + private static final int DEFAULT_SOLR_HEAP_SIZE_MB_32BIT_PLATFORM = 512; public static final String RESULTS_TABLE_PAGE_SIZE = "ResultsTablePageSize"; private static final String GEO_TILE_OPTION = "GeolocationTileOption"; private static final String GEO_OSM_TILE_ZIP_PATH = "GeolocationOsmZipPath"; @@ -534,12 +536,17 @@ public final class UserPreferences { } /** - * Get the maximum JVM heap size (in MB) for the embedded Solr server. + * Get the maximum JVM heap size (in MB) for the embedded Solr server. The returned value + * depends on the platform (64bit vs 32bit). * - * @return Saved value or default (2 GB) + * @return Saved value or default (2 GB for 64bit platforms, 512MB for 32bit) */ public static int getMaxSolrVMSize() { - return preferences.getInt(SOLR_MAX_JVM_SIZE, DEFAULT_SOLR_HEAP_SIZE_MB); + if (PlatformUtil.is64BitJVM()) { + return preferences.getInt(SOLR_MAX_JVM_SIZE, DEFAULT_SOLR_HEAP_SIZE_MB_64BIT_PLATFORM); + } else { + return preferences.getInt(SOLR_MAX_JVM_SIZE, DEFAULT_SOLR_HEAP_SIZE_MB_32BIT_PLATFORM); + } } /** diff --git a/Core/src/org/sleuthkit/autopsy/corecomponentinterfaces/DataSourceProcessor.java b/Core/src/org/sleuthkit/autopsy/corecomponentinterfaces/DataSourceProcessor.java index de51cf8add..a0e7db335c 100644 --- a/Core/src/org/sleuthkit/autopsy/corecomponentinterfaces/DataSourceProcessor.java +++ b/Core/src/org/sleuthkit/autopsy/corecomponentinterfaces/DataSourceProcessor.java @@ -20,6 +20,7 @@ package org.sleuthkit.autopsy.corecomponentinterfaces; import javax.swing.JPanel; import org.sleuthkit.autopsy.ingest.IngestJobSettings; +import org.sleuthkit.datamodel.Host; /** * Interface implemented by classes that add data sources of a particular type @@ -109,6 +110,25 @@ public interface DataSourceProcessor { */ void run(DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callback); + /** + * Adds a data source to the case database using a background task in a + * separate thread and the settings provided by the selection and + * configuration panel. Returns as soon as the background task is started. + * The background task uses a callback object to signal task completion and + * return results. + * + * This method should not be called unless isPanelValid returns true. + * + * @param host Host for the data source. + * @param progressMonitor Progress monitor that will be used by the + * background task to report progress. + * @param callback Callback that will be used by the background task + * to return results. + */ + default void run(Host host, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callback) { + run(progressMonitor, callback); + } + /** * Adds a data source to the case database using a background task in a * separate thread and the settings provided by the selection and @@ -131,6 +151,30 @@ public interface DataSourceProcessor { DataSourceProcessorCallback callBack) { throw new UnsupportedOperationException("Streaming ingest not supported for this data source processor"); } + + /** + * Adds a data source to the case database using a background task in a + * separate thread and the settings provided by the selection and + * configuration panel. Files found during ingest will be sent directly to + * the IngestStream provided. Returns as soon as the background task is + * started. The background task uses a callback object to signal task + * completion and return results. + * + * This method should not be called unless isPanelValid returns true, and + * should only be called for DSPs that support ingest streams. The ingest + * settings must be complete before calling this method. + * + * @param host Host for this data source. + * @param settings The ingest job settings. + * @param progress Progress monitor that will be used by the background task + * to report progress. + * @param callBack Callback that will be used by the background task to + * return results. + */ + default void runWithIngestStream(Host host, IngestJobSettings settings, DataSourceProcessorProgressMonitor progress, + DataSourceProcessorCallback callBack) { + runWithIngestStream(settings, progress, callBack); + } /** * Check if this DSP supports ingest streams. diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/DhsImageCategory.java b/Core/src/org/sleuthkit/autopsy/datamodel/DhsImageCategory.java index 50974519c7..4d7f87389b 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/DhsImageCategory.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/DhsImageCategory.java @@ -30,6 +30,7 @@ import org.openide.util.NbBundle; /** * Enum to represent the six categories in the DHS image categorization scheme. + * NOTE: This appears to not be used anywhere anymore after the ImageGallery refactoring */ @NbBundle.Messages({ "Category.one=CAT-1: Child Exploitation (Illegal)", diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/FileTypes.java b/Core/src/org/sleuthkit/autopsy/datamodel/FileTypes.java index 563f9c8cee..1a7abab0f9 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/FileTypes.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/FileTypes.java @@ -389,11 +389,6 @@ public final class FileTypes implements AutopsyVisitableItem { return content.newArtifact(artifactTypeID); } - @Override - public AnalysisResultAdded newAnalysisResult(BlackboardArtifact.Type artifactType, Score score, String conclusion, String configuration, String justification, Collection attributesList) throws TskCoreException { - return content.newAnalysisResult(artifactType, score, conclusion, configuration, justification, attributesList); - } - @Override public BlackboardArtifact newArtifact(BlackboardArtifact.ARTIFACT_TYPE type) throws TskCoreException { return content.newArtifact(type); @@ -434,15 +429,6 @@ public final class FileTypes implements AutopsyVisitableItem { return content.getAllArtifacts(); } - @Override - public List getAllAnalysisResults() throws TskCoreException { - return content.getAllAnalysisResults(); - } - - public List getAnalysisResults(BlackboardArtifact.Type artifactType) throws TskCoreException { - return content.getAnalysisResults(artifactType); - } - @Override public Set getHashSetNames() throws TskCoreException { return content.getHashSetNames(); @@ -468,9 +454,24 @@ public final class FileTypes implements AutopsyVisitableItem { return content.getAllArtifactsCount(); } + @Override + public AnalysisResultAdded newAnalysisResult(BlackboardArtifact.Type type, Score score, String string, String string1, String string2, Collection clctn) throws TskCoreException { + return content.newAnalysisResult(type, score, string, string1, string2, clctn); + } + @Override public Score getAggregateScore() throws TskCoreException { return content.getAggregateScore(); } + + @Override + public List getAnalysisResults(BlackboardArtifact.Type type) throws TskCoreException { + return content.getAnalysisResults(type); + } + + @Override + public List getAllAnalysisResults() throws TskCoreException { + return content.getAllAnalysisResults(); + } } } diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/AddRawImageTask.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/AddRawImageTask.java index 27e1b75344..eccfd5950a 100644 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/AddRawImageTask.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/AddRawImageTask.java @@ -32,6 +32,7 @@ import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorCallback import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorProgressMonitor; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.datamodel.Content; +import org.sleuthkit.datamodel.Host; import org.sleuthkit.datamodel.Image; import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.TskCoreException; @@ -47,6 +48,7 @@ final class AddRawImageTask implements Runnable { private final String imageFilePath; private final String timeZone; private final long chunkSize; + private final Host host; private final DataSourceProcessorProgressMonitor progressMonitor; private final DataSourceProcessorCallback callback; private boolean criticalErrorOccurred; @@ -68,11 +70,12 @@ final class AddRawImageTask implements Runnable { * progressMonitor during processing. * @param callback Callback to call when processing is done. */ - AddRawImageTask(String deviceId, String imageFilePath, String timeZone, long chunkSize, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callback) { + AddRawImageTask(String deviceId, String imageFilePath, String timeZone, long chunkSize, Host host, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callback) { this.deviceId = deviceId; this.imageFilePath = imageFilePath; this.timeZone = timeZone; this.chunkSize = chunkSize; + this.host = host; this.callback = callback; this.progressMonitor = progressMonitor; } @@ -150,7 +153,7 @@ final class AddRawImageTask implements Runnable { /* * Get Image that will be added to case */ - Image dataSource = caseDatabase.addImageInfo(0, imageFilePaths, timeZone); //TODO: change hard coded deviceId. + Image dataSource = caseDatabase.addImageInfo(0, imageFilePaths, timeZone, host); //TODO: change hard coded deviceId. dataSources.add(dataSource); List fileRanges = new ArrayList<>(); diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/RawDSProcessor.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/RawDSProcessor.java index 790c4ae11e..8760ac8da5 100644 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/RawDSProcessor.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/RawDSProcessor.java @@ -29,10 +29,13 @@ import javax.swing.filechooser.FileFilter; import org.openide.util.NbBundle.Messages; import org.openide.util.lookup.ServiceProvider; import org.openide.util.lookup.ServiceProviders; +import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.GeneralFilter; import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorProgressMonitor; import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorCallback; import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessor; +import org.sleuthkit.datamodel.Host; +import org.sleuthkit.datamodel.TskCoreException; /** * A Raw data source processor that implements the DataSourceProcessor service @@ -135,8 +138,37 @@ public class RawDSProcessor implements DataSourceProcessor, AutoIngestDataSource */ @Override public void run(DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callback) { + run(null, progressMonitor, callback); + } + + /** + * Adds a data source to the case database using a background task in a + * separate thread and the settings provided by the selection and + * configuration panel. Returns as soon as the background task is started. + * The background task uses a callback object to signal task completion and + * return results. + * + * This method should not be called unless isPanelValid returns true. + * + * @param host Host for the data source. + * @param progressMonitor Progress monitor that will be used by the + * background task to report progress. + * @param callback Callback that will be used by the background task + * to return results. + */ + @Override + public void run(Host host, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callback) { configPanel.storeSettings(); - run(UUID.randomUUID().toString(), configPanel.getImageFilePath(), configPanel.getTimeZone(), configPanel.getChunkSize(), progressMonitor, callback); + + // HOSTTODO - use passed in value + try { + host = Case.getCurrentCase().getSleuthkitCase().getHostManager().getOrCreateHost("RawDSProcessor Host"); + } catch (TskCoreException ex) { + // It's not worth adding a logger for temporary code + //logger.log(Level.SEVERE, "Error creating/loading host", ex); + host = null; + } + run(UUID.randomUUID().toString(), configPanel.getImageFilePath(), configPanel.getTimeZone(), configPanel.getChunkSize(), host, progressMonitor, callback); } /** @@ -157,12 +189,13 @@ public class RawDSProcessor implements DataSourceProcessor, AutoIngestDataSource * @param chunkSize The maximum size of each chunk of the raw * data source as it is divided up into virtual * unallocated space files. + * @param host The host for this data source. * @param progressMonitor Progress monitor for reporting progress * during processing. * @param callback Callback to call when processing is done. */ - private void run(String deviceId, String imageFilePath, String timeZone, long chunkSize, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callback) { - AddRawImageTask addImageTask = new AddRawImageTask(deviceId, imageFilePath, timeZone, chunkSize, progressMonitor, callback); + private void run(String deviceId, String imageFilePath, String timeZone, long chunkSize, Host host, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callback) { + AddRawImageTask addImageTask = new AddRawImageTask(deviceId, imageFilePath, timeZone, chunkSize, host, progressMonitor, callback); new Thread(addImageTask).start(); } @@ -206,7 +239,7 @@ public class RawDSProcessor implements DataSourceProcessor, AutoIngestDataSource @Override public void process(String deviceId, Path dataSourcePath, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callBack) { - run(deviceId, dataSourcePath.toString(), Calendar.getInstance().getTimeZone().getID(), DEFAULT_CHUNK_SIZE, progressMonitor, callBack); + run(deviceId, dataSourcePath.toString(), Calendar.getInstance().getTimeZone().getID(), DEFAULT_CHUNK_SIZE, null, progressMonitor, callBack); } } diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYDataSourceProcessor.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYDataSourceProcessor.java index 217f7e9823..a764e9f5f6 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYDataSourceProcessor.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYDataSourceProcessor.java @@ -49,6 +49,7 @@ import org.sleuthkit.autopsy.datasourceprocessors.AutoIngestDataSourceProcessor; import org.sleuthkit.autopsy.datasourceprocessors.AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.Blackboard.BlackboardException; +import org.sleuthkit.datamodel.Host; import org.sleuthkit.datamodel.LocalFilesDataSource; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskDataException; @@ -193,12 +194,40 @@ public class XRYDataSourceProcessor implements DataSourceProcessor, AutoIngestDa * in isPanelValid(). */ @Override + public void run(DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callback) { + run(null, progressMonitor, callback); + } + + /** + * Processes the XRY folder that the examiner selected. The heavy lifting is + * done off of the EDT, so this function will return while the + * path is still being processed. + * + * This function assumes the calling thread has sufficient privileges to + * read the folder and its child content, which should have been validated + * in isPanelValid(). + * + * @param host Host for the data source. + * @param progressMonitor Progress monitor that will be used by the + * background task to report progress. + * @param callback Callback that will be used by the background task + * to return results. + */ + @Override @NbBundle.Messages({ "XRYDataSourceProcessor.noCurrentCase=No case is open." }) - public void run(DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callback) { + public void run(Host host, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callback) { progressMonitor.setIndeterminate(true); + // HOSTTODO - use passed in value + try { + host = Case.getCurrentCase().getSleuthkitCase().getHostManager().getOrCreateHost("XRYDSProcessor Host"); + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, "Error creating/loading host", ex); + host = null; + } + String selectedFilePath = configPanel.getSelectedFilePath(); File selectedFile = new File(selectedFilePath); Path selectedPath = selectedFile.toPath(); @@ -209,7 +238,7 @@ public class XRYDataSourceProcessor implements DataSourceProcessor, AutoIngestDa String uniqueUUID = UUID.randomUUID().toString(); //Move heavy lifting to a background task. swingWorker = new XRYReportProcessorSwingWorker(xryFolder, progressMonitor, - callback, currentCase, uniqueUUID); + callback, currentCase, uniqueUUID, host); swingWorker.execute(); } catch (NoCurrentCaseException ex) { logger.log(Level.WARNING, "[XRY DSP] No case is currently open.", ex); @@ -241,7 +270,7 @@ public class XRYDataSourceProcessor implements DataSourceProcessor, AutoIngestDa Case currentCase = Case.getCurrentCaseThrows(); //Move heavy lifting to a background task. swingWorker = new XRYReportProcessorSwingWorker(xryFolder, progressMonitor, - callBack, currentCase, deviceId); + callBack, currentCase, deviceId, null); swingWorker.execute(); } catch (NoCurrentCaseException ex) { logger.log(Level.WARNING, "[XRY DSP] No case is currently open.", ex); @@ -275,17 +304,19 @@ public class XRYDataSourceProcessor implements DataSourceProcessor, AutoIngestDa private final Case currentCase; private final XRYFolder xryFolder; private final String uniqueUUID; + private final Host host; public XRYReportProcessorSwingWorker(XRYFolder folder, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callback, - Case currentCase, String uniqueUUID) { + Case currentCase, String uniqueUUID, Host host) { this.xryFolder = folder; this.progressMonitor = progressMonitor; this.callback = callback; this.currentCase = currentCase; this.uniqueUUID = uniqueUUID; + this.host = host; } @Override @@ -306,6 +337,7 @@ public class XRYDataSourceProcessor implements DataSourceProcessor, AutoIngestDa uniqueUUID, "XRY Text Export", //Name "", //Timezone + host, filePaths, new ProgressMonitorAdapter(progressMonitor)); diff --git a/Core/src/org/sleuthkit/autopsy/discovery/ui/ArtifactsListPanel.java b/Core/src/org/sleuthkit/autopsy/discovery/ui/ArtifactsListPanel.java index ae416d0e7f..d1328ef68c 100644 --- a/Core/src/org/sleuthkit/autopsy/discovery/ui/ArtifactsListPanel.java +++ b/Core/src/org/sleuthkit/autopsy/discovery/ui/ArtifactsListPanel.java @@ -18,6 +18,7 @@ */ package org.sleuthkit.autopsy.discovery.ui; +import java.awt.Dimension; import java.awt.Point; import java.util.ArrayList; import java.util.List; @@ -63,6 +64,7 @@ final class ArtifactsListPanel extends AbstractArtifactListPanel { for (int i = 0; i < tableModel.getColumnCount(); ++i) { artifactsTable.getColumnModel().getColumn(i).setCellRenderer(renderer); } + setMinimumSize(new Dimension(125, 20)); artifactsTable.getRowSorter().toggleSortOrder(0); artifactsTable.getRowSorter().toggleSortOrder(0); } diff --git a/Core/src/org/sleuthkit/autopsy/discovery/ui/DomainArtifactsTabPanel.form b/Core/src/org/sleuthkit/autopsy/discovery/ui/DomainArtifactsTabPanel.form index bdef0da96f..870763813c 100644 --- a/Core/src/org/sleuthkit/autopsy/discovery/ui/DomainArtifactsTabPanel.form +++ b/Core/src/org/sleuthkit/autopsy/discovery/ui/DomainArtifactsTabPanel.form @@ -4,8 +4,11 @@ - - + + + + + diff --git a/Core/src/org/sleuthkit/autopsy/discovery/ui/DomainArtifactsTabPanel.java b/Core/src/org/sleuthkit/autopsy/discovery/ui/DomainArtifactsTabPanel.java index e4ce23b865..bbb1a8a634 100644 --- a/Core/src/org/sleuthkit/autopsy/discovery/ui/DomainArtifactsTabPanel.java +++ b/Core/src/org/sleuthkit/autopsy/discovery/ui/DomainArtifactsTabPanel.java @@ -20,8 +20,12 @@ package org.sleuthkit.autopsy.discovery.ui; import org.sleuthkit.autopsy.contentviewers.artifactviewers.GeneralPurposeArtifactViewer; import com.google.common.eventbus.Subscribe; +import java.awt.Dimension; +import java.beans.PropertyChangeEvent; +import java.beans.PropertyChangeListener; import java.util.logging.Level; import javax.swing.JPanel; +import javax.swing.JSplitPane; import javax.swing.SwingUtilities; import javax.swing.event.ListSelectionEvent; import javax.swing.event.ListSelectionListener; @@ -41,13 +45,18 @@ final class DomainArtifactsTabPanel extends JPanel { private final ArtifactsListPanel listPanel; private final BlackboardArtifact.ARTIFACT_TYPE artifactType; private AbstractArtifactDetailsPanel rightPanel = null; + private int dividerLocation = 300; + private final PropertyChangeListener dividerListener; private ArtifactRetrievalStatus status = ArtifactRetrievalStatus.UNPOPULATED; private final ListSelectionListener listener = new ListSelectionListener() { @Override public void valueChanged(ListSelectionEvent event) { if (!event.getValueIsAdjusting()) { + mainSplitPane.removePropertyChangeListener(dividerListener); rightPanel.setArtifact(listPanel.getSelectedArtifact()); + mainSplitPane.setDividerLocation(dividerLocation); + mainSplitPane.addPropertyChangeListener(JSplitPane.DIVIDER_LOCATION_PROPERTY, dividerListener); } } }; @@ -60,12 +69,27 @@ final class DomainArtifactsTabPanel extends JPanel { @ThreadConfined(type = ThreadConfined.ThreadType.AWT) DomainArtifactsTabPanel(BlackboardArtifact.ARTIFACT_TYPE type) { initComponents(); + dividerListener = new PropertyChangeListener() { + @Override + public void propertyChange(PropertyChangeEvent evt) { + if (evt.getPropertyName().equalsIgnoreCase(JSplitPane.DIVIDER_LOCATION_PROPERTY) + && evt.getNewValue() instanceof Integer + && evt.getOldValue() instanceof Integer + && (JSplitPane.UNDEFINED_CONDITION != (int) evt.getNewValue())) { + dividerLocation = (int) evt.getNewValue(); + } + } + }; this.artifactType = type; listPanel = new ArtifactsListPanel(artifactType); + listPanel.setPreferredSize(new Dimension(100, 20)); listPanel.addMouseListener(new ArtifactMenuMouseAdapter(listPanel)); + mainSplitPane.setLeftComponent(listPanel); add(mainSplitPane); setRightComponent(); + mainSplitPane.addPropertyChangeListener(JSplitPane.DIVIDER_LOCATION_PROPERTY, dividerListener); + dividerLocation = mainSplitPane.getDividerLocation(); listPanel.addSelectionListener(listener); } @@ -121,6 +145,7 @@ final class DomainArtifactsTabPanel extends JPanel { @ThreadConfined(type = ThreadConfined.ThreadType.AWT) void setStatus(ArtifactRetrievalStatus status) { this.status = status; + mainSplitPane.removePropertyChangeListener(dividerListener); if (status == ArtifactRetrievalStatus.UNPOPULATED) { listPanel.clearList(); removeAll(); @@ -132,6 +157,8 @@ final class DomainArtifactsTabPanel extends JPanel { removeAll(); add(new LoadingPanel(artifactType.getDisplayName())); } + mainSplitPane.setDividerLocation(dividerLocation); + mainSplitPane.addPropertyChangeListener(JSplitPane.DIVIDER_LOCATION_PROPERTY, dividerListener); } /** @@ -144,6 +171,7 @@ final class DomainArtifactsTabPanel extends JPanel { void handleArtifactSearchResultEvent(DiscoveryEventUtils.ArtifactSearchResultEvent artifactresultEvent) { if (artifactType == artifactresultEvent.getArtifactType() && status == ArtifactRetrievalStatus.POPULATING) { SwingUtilities.invokeLater(() -> { + mainSplitPane.removePropertyChangeListener(dividerListener); listPanel.removeSelectionListener(listener); listPanel.addArtifacts(artifactresultEvent.getListOfArtifacts()); status = ArtifactRetrievalStatus.POPULATED; @@ -152,6 +180,8 @@ final class DomainArtifactsTabPanel extends JPanel { listPanel.selectFirst(); removeAll(); add(mainSplitPane); + mainSplitPane.setDividerLocation(dividerLocation); + mainSplitPane.addPropertyChangeListener(JSplitPane.DIVIDER_LOCATION_PROPERTY, dividerListener); if (artifactresultEvent.shouldGrabFocus()) { focusList(); } @@ -188,8 +218,9 @@ final class DomainArtifactsTabPanel extends JPanel { mainSplitPane = new javax.swing.JSplitPane(); - mainSplitPane.setDividerLocation(350); - mainSplitPane.setResizeWeight(0.1); + mainSplitPane.setDividerLocation(dividerLocation); + mainSplitPane.setResizeWeight(0.2); + mainSplitPane.setLastDividerLocation(250); setCursor(new java.awt.Cursor(java.awt.Cursor.DEFAULT_CURSOR)); setMinimumSize(new java.awt.Dimension(0, 0)); diff --git a/Core/src/org/sleuthkit/autopsy/discovery/ui/MiniTimelineArtifactListPanel.java b/Core/src/org/sleuthkit/autopsy/discovery/ui/MiniTimelineArtifactListPanel.java index 87a59edcf4..65362f72ff 100644 --- a/Core/src/org/sleuthkit/autopsy/discovery/ui/MiniTimelineArtifactListPanel.java +++ b/Core/src/org/sleuthkit/autopsy/discovery/ui/MiniTimelineArtifactListPanel.java @@ -18,6 +18,7 @@ */ package org.sleuthkit.autopsy.discovery.ui; +import java.awt.Dimension; import java.awt.Point; import java.util.ArrayList; import java.util.List; @@ -39,7 +40,7 @@ import org.sleuthkit.datamodel.TskCoreException; /** * Panel to display list of artifacts types and descriptions. */ -class MiniTimelineArtifactListPanel extends AbstractArtifactListPanel { +final class MiniTimelineArtifactListPanel extends AbstractArtifactListPanel { private static final long serialVersionUID = 1L; private final TypeDescriptionTableModel tableModel; @@ -61,6 +62,7 @@ class MiniTimelineArtifactListPanel extends AbstractArtifactListPanel { for (int i = 0; i < tableModel.getColumnCount(); ++i) { artifactsTable.getColumnModel().getColumn(i).setCellRenderer(renderer); } + setMinimumSize(new Dimension(125, 20)); artifactsTable.getRowSorter().toggleSortOrder(0); artifactsTable.getRowSorter().toggleSortOrder(0); } diff --git a/Core/src/org/sleuthkit/autopsy/discovery/ui/MiniTimelineDateListPanel.java b/Core/src/org/sleuthkit/autopsy/discovery/ui/MiniTimelineDateListPanel.java index 28c68c6ce6..8ede83b6d5 100644 --- a/Core/src/org/sleuthkit/autopsy/discovery/ui/MiniTimelineDateListPanel.java +++ b/Core/src/org/sleuthkit/autopsy/discovery/ui/MiniTimelineDateListPanel.java @@ -18,6 +18,7 @@ */ package org.sleuthkit.autopsy.discovery.ui; +import java.awt.Dimension; import java.util.ArrayList; import java.util.List; import javax.swing.JPanel; @@ -33,7 +34,7 @@ import org.sleuthkit.datamodel.BlackboardArtifact; /** * Panel to display list of dates and counts. */ -class MiniTimelineDateListPanel extends JPanel { +final class MiniTimelineDateListPanel extends JPanel { private static final long serialVersionUID = 1L; private final DateCountTableModel tableModel = new DateCountTableModel(); @@ -49,6 +50,7 @@ class MiniTimelineDateListPanel extends JPanel { for (int i = 0; i < tableModel.getColumnCount(); ++i) { jTable1.getColumnModel().getColumn(i).setCellRenderer(renderer); } + setMinimumSize(new Dimension(125, 20)); jTable1.getRowSorter().toggleSortOrder(0); jTable1.getRowSorter().toggleSortOrder(0); } diff --git a/Core/src/org/sleuthkit/autopsy/discovery/ui/MiniTimelinePanel.form b/Core/src/org/sleuthkit/autopsy/discovery/ui/MiniTimelinePanel.form index 0f0598a148..0c3b13bbad 100644 --- a/Core/src/org/sleuthkit/autopsy/discovery/ui/MiniTimelinePanel.form +++ b/Core/src/org/sleuthkit/autopsy/discovery/ui/MiniTimelinePanel.form @@ -4,8 +4,10 @@ - - + + + + @@ -16,7 +18,9 @@ - + + + diff --git a/Core/src/org/sleuthkit/autopsy/discovery/ui/MiniTimelinePanel.java b/Core/src/org/sleuthkit/autopsy/discovery/ui/MiniTimelinePanel.java index 87c99ad724..df8523fcfa 100644 --- a/Core/src/org/sleuthkit/autopsy/discovery/ui/MiniTimelinePanel.java +++ b/Core/src/org/sleuthkit/autopsy/discovery/ui/MiniTimelinePanel.java @@ -19,6 +19,10 @@ package org.sleuthkit.autopsy.discovery.ui; import com.google.common.eventbus.Subscribe; +import java.awt.Dimension; +import java.beans.PropertyChangeEvent; +import java.beans.PropertyChangeListener; +import javax.swing.JSplitPane; import javax.swing.SwingUtilities; import javax.swing.event.ListSelectionEvent; import javax.swing.event.ListSelectionListener; @@ -42,6 +46,9 @@ final class MiniTimelinePanel extends javax.swing.JPanel { private String selectedDomain = null; private final ListSelectionListener artifactListener; private final ListSelectionListener dateListener; + private int leftSplitPaneDividerLocation = 125; + private int mainSplitPaneDividerLocation = 300; + private final PropertyChangeListener dividerListener; @NbBundle.Messages({"MiniTimelinePanel.loadingPanel.details=the Timeline view"}) /** @@ -62,22 +69,51 @@ final class MiniTimelinePanel extends javax.swing.JPanel { } else { rightPanel = new GeneralPurposeArtifactViewer(); } + leftSplitPane.removePropertyChangeListener(dividerListener); + mainSplitPane.removePropertyChangeListener(dividerListener); mainSplitPane.setRightComponent(rightPanel.getComponent()); rightPanel.setArtifact(artifact); + mainSplitPane.setDividerLocation(mainSplitPaneDividerLocation); + leftSplitPane.setDividerLocation(leftSplitPaneDividerLocation); + mainSplitPane.setDividerLocation(mainSplitPaneDividerLocation); + leftSplitPane.setDividerLocation(leftSplitPaneDividerLocation); + leftSplitPane.addPropertyChangeListener(JSplitPane.DIVIDER_LOCATION_PROPERTY, dividerListener); + mainSplitPane.addPropertyChangeListener(JSplitPane.DIVIDER_LOCATION_PROPERTY, dividerListener); validate(); repaint(); } } }; + dividerListener = new PropertyChangeListener() { + @Override + public void propertyChange(PropertyChangeEvent evt) { + if (evt.getPropertyName().equalsIgnoreCase(JSplitPane.DIVIDER_LOCATION_PROPERTY) + && evt.getNewValue() instanceof Integer + && evt.getOldValue() instanceof Integer + && (JSplitPane.UNDEFINED_CONDITION != (int) evt.getNewValue())) { + if (evt.getSource().equals(leftSplitPane)) { + leftSplitPaneDividerLocation = (int) evt.getNewValue(); + } else if (evt.getSource().equals(mainSplitPane)) { + mainSplitPaneDividerLocation = (int) evt.getNewValue(); + } + } + } + }; dateListener = new ListSelectionListener() { @Override public void valueChanged(ListSelectionEvent event) { if (!event.getValueIsAdjusting()) { artifactListPanel.removeSelectionListener(artifactListener); + leftSplitPane.removePropertyChangeListener(dividerListener); + mainSplitPane.removePropertyChangeListener(dividerListener); artifactListPanel.clearList(); artifactListPanel.addArtifacts(dateListPanel.getArtifactsForSelectedDate()); artifactListPanel.addSelectionListener(artifactListener); artifactListPanel.selectFirst(); + mainSplitPane.setDividerLocation(mainSplitPaneDividerLocation); + leftSplitPane.setDividerLocation(leftSplitPaneDividerLocation); + leftSplitPane.addPropertyChangeListener(JSplitPane.DIVIDER_LOCATION_PROPERTY, dividerListener); + mainSplitPane.addPropertyChangeListener(JSplitPane.DIVIDER_LOCATION_PROPERTY, dividerListener); validate(); repaint(); } @@ -85,10 +121,16 @@ final class MiniTimelinePanel extends javax.swing.JPanel { }; dateListPanel.addSelectionListener(dateListener); artifactListPanel.addSelectionListener(artifactListener); + dateListPanel.setPreferredSize(new Dimension(100, 20)); leftSplitPane.setLeftComponent(dateListPanel); + artifactListPanel.setPreferredSize(new Dimension(100, 20)); leftSplitPane.setRightComponent(artifactListPanel); mainSplitPane.setRightComponent(rightPanel.getComponent()); add(mainSplitPane); + leftSplitPane.addPropertyChangeListener(JSplitPane.DIVIDER_LOCATION_PROPERTY, dividerListener); + mainSplitPane.addPropertyChangeListener(JSplitPane.DIVIDER_LOCATION_PROPERTY, dividerListener); + leftSplitPaneDividerLocation = leftSplitPane.getDividerLocation(); + mainSplitPaneDividerLocation = mainSplitPane.getDividerLocation(); } /** @@ -145,6 +187,8 @@ final class MiniTimelinePanel extends javax.swing.JPanel { if (miniTimelineResultEvent.getDomain().equals(selectedDomain)) { dateListPanel.removeListSelectionListener(dateListener); artifactListPanel.removeSelectionListener(artifactListener); + leftSplitPane.removePropertyChangeListener(dividerListener); + mainSplitPane.removePropertyChangeListener(dividerListener); dateListPanel.addArtifacts(miniTimelineResultEvent.getResultList()); status = DomainArtifactsTabPanel.ArtifactRetrievalStatus.POPULATED; setEnabled(!dateListPanel.isEmpty()); @@ -156,6 +200,10 @@ final class MiniTimelinePanel extends javax.swing.JPanel { } removeAll(); add(mainSplitPane); + mainSplitPane.setDividerLocation(mainSplitPaneDividerLocation); + leftSplitPane.setDividerLocation(leftSplitPaneDividerLocation); + leftSplitPane.addPropertyChangeListener(JSplitPane.DIVIDER_LOCATION_PROPERTY, dividerListener); + mainSplitPane.addPropertyChangeListener(JSplitPane.DIVIDER_LOCATION_PROPERTY, dividerListener); revalidate(); repaint(); } @@ -174,12 +222,12 @@ final class MiniTimelinePanel extends javax.swing.JPanel { mainSplitPane = new javax.swing.JSplitPane(); leftSplitPane = new javax.swing.JSplitPane(); - mainSplitPane.setDividerLocation(400); - mainSplitPane.setResizeWeight(0.1); + mainSplitPane.setDividerLocation(mainSplitPaneDividerLocation); + mainSplitPane.setResizeWeight(0.2); mainSplitPane.setToolTipText(""); mainSplitPane.setMinimumSize(new java.awt.Dimension(0, 0)); - leftSplitPane.setDividerLocation(198); + leftSplitPane.setDividerLocation(leftSplitPaneDividerLocation); leftSplitPane.setResizeWeight(0.5); leftSplitPane.setMinimumSize(new java.awt.Dimension(0, 0)); mainSplitPane.setLeftComponent(leftSplitPane); diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobSettings.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobSettings.java index 0dbbaebb57..918c30760f 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobSettings.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobSettings.java @@ -477,6 +477,9 @@ public final class IngestJobSettings { case "Exif Parser": //NON-NLS moduleNames.add("Picture Analyzer"); //NON-NLS break; + case "Drone Analyzer": + moduleNames.add("DJI Drone Analyzer"); + break; default: moduleNames.add(name); } diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java index c8cad5bf9e..a2297fafbf 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java @@ -48,6 +48,7 @@ import org.sleuthkit.datamodel.Blackboard; import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardAttribute; import org.sleuthkit.datamodel.Content; +import org.sleuthkit.datamodel.Host; import org.sleuthkit.datamodel.LocalFilesDataSource; import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.TskCoreException; @@ -71,6 +72,7 @@ final class AddLogicalImageTask implements Runnable { private final String timeZone; private final File src; private final File dest; + private final Host host; private final DataSourceProcessorCallback callback; private final DataSourceProcessorProgressMonitor progressMonitor; private final Blackboard blackboard; @@ -87,7 +89,7 @@ final class AddLogicalImageTask implements Runnable { AddLogicalImageTask(String deviceId, String timeZone, - File src, File dest, + File src, File dest, Host host, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callback ) throws NoCurrentCaseException { @@ -95,6 +97,7 @@ final class AddLogicalImageTask implements Runnable { this.timeZone = timeZone; this.src = src; this.dest = dest; + this.host = host; this.progressMonitor = progressMonitor; this.callback = callback; this.currentCase = Case.getCurrentCase(); @@ -229,7 +232,7 @@ final class AddLogicalImageTask implements Runnable { try { progressMonitor.setProgressText(Bundle.AddLogicalImageTask_addingExtractedFiles()); - interestingFileMap = addExtractedFiles(dest, resultsPath, newDataSources); + interestingFileMap = addExtractedFiles(dest, resultsPath, host, newDataSources); progressMonitor.setProgressText(Bundle.AddLogicalImageTask_doneAddingExtractedFiles()); } catch (IOException | TskCoreException ex) { errorList.add(ex.getMessage()); @@ -248,7 +251,7 @@ final class AddLogicalImageTask implements Runnable { callback.done(DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS, errorList, emptyDataSources); return; } - addMultipleImagesTask = new AddMultipleImagesTask(deviceId, imagePaths, timeZone , progressMonitor); + addMultipleImagesTask = new AddMultipleImagesTask(deviceId, imagePaths, timeZone, host, progressMonitor); } addMultipleImagesTask.run(); if (addMultipleImagesTask.getResult() == DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS) { @@ -474,14 +477,14 @@ final class AddLogicalImageTask implements Runnable { @Messages({ "# {0} - file number", "# {1} - total files", "AddLogicalImageTask.addingExtractedFile=Adding extracted files ({0}/{1})" }) - private Map> addExtractedFiles(File src, Path resultsPath, List newDataSources) throws TskCoreException, IOException { + private Map> addExtractedFiles(File src, Path resultsPath, Host host, List newDataSources) throws TskCoreException, IOException { SleuthkitCase skCase = Case.getCurrentCase().getSleuthkitCase(); SleuthkitCase.CaseDbTransaction trans = null; Map> interestingFileMap = new HashMap<>(); try { trans = skCase.beginTransaction(); - LocalFilesDataSource localFilesDataSource = skCase.addLocalFilesDataSource(deviceId, this.src.getName(), timeZone, trans); + LocalFilesDataSource localFilesDataSource = skCase.addLocalFilesDataSource(deviceId, this.src.getName(), timeZone, host, trans); LocalFileImporter fileImporter = new LocalFileImporter(skCase, trans); try (BufferedReader br = new BufferedReader(new InputStreamReader( diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddMultipleImagesTask.java b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddMultipleImagesTask.java index f5a2f2bd8c..0f73f1de6b 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddMultipleImagesTask.java +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddMultipleImagesTask.java @@ -30,6 +30,7 @@ import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorProgress import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.DefaultAddDataSourceCallbacks; +import org.sleuthkit.datamodel.Host; import org.sleuthkit.datamodel.Image; import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.SleuthkitJNI; @@ -53,6 +54,7 @@ class AddMultipleImagesTask implements Runnable { private final String deviceId; private final List imageFilePaths; private final String timeZone; + private final Host host; private final long chunkSize = TWO_GB; private final DataSourceProcessorProgressMonitor progressMonitor; private final Case currentCase; @@ -85,6 +87,7 @@ class AddMultipleImagesTask implements Runnable { * @param timeZone The time zone to use when processing dates and * times for the image, obtained from * java.util.TimeZone.getID. + * @param host Host for this data source (may be null). * @param progressMonitor Progress monitor for reporting progress during * processing. * @@ -94,11 +97,12 @@ class AddMultipleImagesTask implements Runnable { "# {0} - file", "AddMultipleImagesTask.addingFileAsLogicalFile=Adding: {0} as an unallocated space file.", "# {0} - deviceId", "# {1} - exceptionMessage", "AddMultipleImagesTask.errorAddingImgWithoutFileSystem=Error adding images without file systems for device {0}: {1}",}) - AddMultipleImagesTask(String deviceId, List imageFilePaths, String timeZone, + AddMultipleImagesTask(String deviceId, List imageFilePaths, String timeZone, Host host, DataSourceProcessorProgressMonitor progressMonitor) throws NoCurrentCaseException { this.deviceId = deviceId; this.imageFilePaths = imageFilePaths; this.timeZone = timeZone; + this.host = host; this.progressMonitor = progressMonitor; currentCase = Case.getCurrentCaseThrows(); this.criticalErrorOccurred = false; @@ -124,7 +128,7 @@ class AddMultipleImagesTask implements Runnable { for (String imageFilePath : imageFilePaths) { try { currentImage = SleuthkitJNI.addImageToDatabase(currentCase.getSleuthkitCase(), new String[]{imageFilePath}, - 0, timeZone, "", "", "", deviceId); + 0, timeZone, "", "", "", deviceId, host); } catch (TskCoreException ex) { LOGGER.log(Level.SEVERE, "Error adding image " + imageFilePath + " to database", ex); errorMessages.add(Bundle.AddMultipleImagesTask_imageError(imageFilePath)); @@ -134,7 +138,7 @@ class AddMultipleImagesTask implements Runnable { synchronized (tskAddImageProcessLock) { if (!tskAddImageProcessStopped) { - addImageProcess = currentCase.getSleuthkitCase().makeAddImageProcess(timeZone, false, false, ""); + addImageProcess = currentCase.getSleuthkitCase().makeAddImageProcess(timeZone, false, false, host, ""); } else { return; } diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/LogicalImagerDSProcessor.java b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/LogicalImagerDSProcessor.java index 5540c1ac80..67498064f0 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/LogicalImagerDSProcessor.java +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/LogicalImagerDSProcessor.java @@ -38,6 +38,8 @@ import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorCallback import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorProgressMonitor; import org.sleuthkit.autopsy.coreutils.TimeStampUtils; import org.sleuthkit.datamodel.Content; +import org.sleuthkit.datamodel.Host; +import org.sleuthkit.datamodel.TskCoreException; /** * A Logical Imager data source processor that implements the @@ -128,6 +130,26 @@ public final class LogicalImagerDSProcessor implements DataSourceProcessor { * @param callback Callback that will be used by the background task * to return results. */ + @Override + public void run(DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callback) { + run(null, progressMonitor, callback); + } + + /** + * Adds a data source to the case database using a background task in a + * separate thread and the settings provided by the selection and + * configuration panel. Returns as soon as the background task is started. + * The background task uses a callback object to signal task completion and + * return results. + * + * This method should not be called unless isPanelValid returns true. + * + * @param host Host for the data source. + * @param progressMonitor Progress monitor that will be used by the + * background task to report progress. + * @param callback Callback that will be used by the background task + * to return results. + */ @Messages({ "# {0} - imageDirPath", "LogicalImagerDSProcessor.imageDirPathNotFound={0} not found.\nUSB drive has been ejected.", "# {0} - directory", "LogicalImagerDSProcessor.failToCreateDirectory=Failed to create directory {0}", @@ -137,9 +159,18 @@ public final class LogicalImagerDSProcessor implements DataSourceProcessor { "LogicalImagerDSProcessor.noCurrentCase=No current case", }) @Override - public void run(DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callback) { + public void run(Host host, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callback) { configPanel.storeSettings(); + // HOSTTODO - set to value from config panel + try { + host = Case.getCurrentCase().getSleuthkitCase().getHostManager().getOrCreateHost("LogicalImagerDSProcessor Host"); + } catch (TskCoreException ex) { + // It's not worth adding a logger for temporary code + //logger.log(Level.SEVERE, "Error creating/loading host", ex); + host = null; + } + Path imageDirPath = configPanel.getImageDirPath(); List errorList = new ArrayList<>(); List emptyDataSources = new ArrayList<>(); @@ -187,7 +218,7 @@ public final class LogicalImagerDSProcessor implements DataSourceProcessor { try { String deviceId = UUID.randomUUID().toString(); String timeZone = Calendar.getInstance().getTimeZone().getID(); - run(deviceId, timeZone, src, dest, + run(deviceId, timeZone, src, dest, host, progressMonitor, callback); } catch (NoCurrentCaseException ex) { String msg = Bundle.LogicalImagerDSProcessor_noCurrentCase(); @@ -211,15 +242,16 @@ public final class LogicalImagerDSProcessor implements DataSourceProcessor { * java.util.TimeZone.getID. * @param src The source directory of image. * @param dest The destination directory to copy the source. + * @param host The host for this data source. * @param progressMonitor Progress monitor for reporting progress during * processing. * @param callback Callback to call when processing is done. */ private void run(String deviceId, String timeZone, - File src, File dest, + File src, File dest, Host host, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callback ) throws NoCurrentCaseException { - addLogicalImageTask = new AddLogicalImageTask(deviceId, timeZone, src, dest, + addLogicalImageTask = new AddLogicalImageTask(deviceId, timeZone, src, dest, host, progressMonitor, callback); Thread thread = new Thread(addLogicalImageTask); thread.start(); diff --git a/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/HashDbIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/HashDbIngestModule.java index 74e019d8cb..19b9ff28b7 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/HashDbIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/HashDbIngestModule.java @@ -41,8 +41,6 @@ import org.sleuthkit.autopsy.ingest.IngestModuleReferenceCounter; import org.sleuthkit.autopsy.ingest.IngestServices; import org.sleuthkit.autopsy.modules.hashdatabase.HashDbManager.HashDb; import org.sleuthkit.datamodel.AbstractFile; -import org.sleuthkit.datamodel.AnalysisResult; -import org.sleuthkit.datamodel.AnalysisResultAdded; import org.sleuthkit.datamodel.Blackboard; import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE; @@ -50,7 +48,6 @@ import org.sleuthkit.datamodel.BlackboardAttribute; import org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE; import org.sleuthkit.datamodel.HashHitInfo; import org.sleuthkit.datamodel.HashUtility; -import org.sleuthkit.datamodel.Score; import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskData; @@ -519,7 +516,7 @@ public class HashDbIngestModule implements FileIngestModule { private void postHashSetHitToBlackboard(AbstractFile abstractFile, String md5Hash, String hashSetName, String comment, boolean showInboxMessage) { try { String moduleName = HashLookupModuleFactory.getModuleName(); - //BlackboardArtifact badFile = abstractFile.newArtifact(ARTIFACT_TYPE.TSK_HASHSET_HIT); + BlackboardArtifact badFile = abstractFile.newArtifact(ARTIFACT_TYPE.TSK_HASHSET_HIT); Collection attributes = new ArrayList<>(); //TODO Revisit usage of deprecated constructor as per TSK-583 //BlackboardAttribute att2 = new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_SET_NAME.getTypeID(), MODULE_NAME, "Known Bad", hashSetName); @@ -527,22 +524,14 @@ public class HashDbIngestModule implements FileIngestModule { attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_HASH_MD5, moduleName, md5Hash)); attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_COMMENT, moduleName, comment)); - SleuthkitCase.CaseDbTransaction trans = this.skCase.beginTransaction(); - - AnalysisResultAdded resultAdded = blackboard.newAnalysisResult(new BlackboardArtifact.Type(ARTIFACT_TYPE.TSK_HASHSET_HIT), abstractFile.getId(), abstractFile.getDataSourceObjectId(), new Score(Score.Significance.MEDIUM, Score.Confidence.HIGH), moduleName, comment, hashSetName, attributes, trans); - AnalysisResult badFile = resultAdded.getAnalysisResult(); - trans.commit(); + badFile.addAttributes(attributes); try { /* * post the artifact which will index the artifact for keyword * search, and fire an event to notify UI of this new artifact */ - - - blackboard.postArtifact(badFile, moduleName); - } catch (Blackboard.BlackboardException ex) { logger.log(Level.SEVERE, "Unable to index blackboard artifact " + badFile.getArtifactID(), ex); //NON-NLS MessageNotifyUtil.Notify.error( @@ -585,7 +574,7 @@ public class HashDbIngestModule implements FileIngestModule { abstractFile.getName() + md5Hash, badFile)); } - } catch (TskException | Blackboard.BlackboardException ex) { + } catch (TskException ex) { logger.log(Level.WARNING, "Error creating blackboard artifact", ex); //NON-NLS } } diff --git a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/ALeappAnalyzerIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/ALeappAnalyzerIngestModule.java index acb0acbc7d..3409098d95 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/ALeappAnalyzerIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/ALeappAnalyzerIngestModule.java @@ -92,6 +92,10 @@ public class ALeappAnalyzerIngestModule implements DataSourceIngestModule { public void startUp(IngestJobContext context) throws IngestModuleException { this.context = context; + if (false == PlatformUtil.is64BitOS()) { + throw new IngestModuleException(NbBundle.getMessage(this.getClass(), "AleappAnalyzerIngestModule.not.64.bit.os")); + } + if (false == PlatformUtil.isWindowsOS()) { throw new IngestModuleException(Bundle.ALeappAnalyzerIngestModule_requires_windows()); } diff --git a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/Bundle.properties b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/Bundle.properties index 4236fad631..5dfed94624 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/Bundle.properties @@ -2,7 +2,9 @@ ILeappAnalyzerIngestModule.init.exception.msg=Unable to find {0}. ILeappAnalyzerIngestModule.processing.file=Processing file {0} ILeappAnalyzerIngestModule.parsing.file=Parsing file {0} ILeappAnalyzerIngestModule.processing.filesystem=Processing filesystem +IleappAnalyzerIngestModule.not.64.bit.os=iLeapp will not run on a 32bit operating system ALeappAnalyzerIngestModule.init.exception.msg=Unable to find {0}. ALeappAnalyzerIngestModule.processing.file=Processing file {0} ALeappAnalyzerIngestModule.parsing.file=Parsing file {0} -ALeappAnalyzerIngestModule.processing.filesystem=Processing filesystem \ No newline at end of file +ALeappAnalyzerIngestModule.processing.filesystem=Processing filesystem +AleappAnalyzerIngestModule.not.64.bit.os=aLeapp will not run on a 32bit operating system \ No newline at end of file diff --git a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/Bundle.properties-MERGED index b23a428107..123986b7d6 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/Bundle.properties-MERGED @@ -22,10 +22,12 @@ ILeappAnalyzerIngestModule.init.exception.msg=Unable to find {0}. ILeappAnalyzerIngestModule.processing.file=Processing file {0} ILeappAnalyzerIngestModule.parsing.file=Parsing file {0} ILeappAnalyzerIngestModule.processing.filesystem=Processing filesystem +IleappAnalyzerIngestModule.not.64.bit.os=iLeapp will not run on 32bit operating system ALeappAnalyzerIngestModule.init.exception.msg=Unable to find {0}. ALeappAnalyzerIngestModule.processing.file=Processing file {0} ALeappAnalyzerIngestModule.parsing.file=Parsing file {0} ALeappAnalyzerIngestModule.processing.filesystem=Processing filesystem +AleappAnalyzerIngestModule.not.64.bit.os=aLeapp will not run on 32bit operating system ILeappAnalyzerIngestModule.report.name=iLeapp Html Report ILeappAnalyzerIngestModule.requires.windows=iLeapp module requires windows. ILeappAnalyzerIngestModule.running.iLeapp=Running iLeapp diff --git a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/ILeappAnalyzerIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/ILeappAnalyzerIngestModule.java index 70e811905c..339be316a9 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/ILeappAnalyzerIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/ILeappAnalyzerIngestModule.java @@ -92,6 +92,10 @@ public class ILeappAnalyzerIngestModule implements DataSourceIngestModule { public void startUp(IngestJobContext context) throws IngestModuleException { this.context = context; + if (false == PlatformUtil.is64BitOS()) { + throw new IngestModuleException(NbBundle.getMessage(this.getClass(), "IleappAnalyzerIngestModule.not.64.bit.os")); + } + if (false == PlatformUtil.isWindowsOS()) { throw new IngestModuleException(Bundle.ILeappAnalyzerIngestModule_requires_windows()); } diff --git a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/LeappFileProcessor.java b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/LeappFileProcessor.java index c6fdb6ae1e..5318ddf85b 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/LeappFileProcessor.java +++ b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/LeappFileProcessor.java @@ -59,6 +59,7 @@ import org.sleuthkit.autopsy.casemodule.Case; import static org.sleuthkit.autopsy.casemodule.Case.getCurrentCase; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; import org.sleuthkit.autopsy.casemodule.services.FileManager; +import org.sleuthkit.autopsy.coreutils.NetworkUtils; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.PlatformUtil; import org.sleuthkit.autopsy.ingest.IngestModule.IngestModuleException; @@ -379,7 +380,9 @@ public final class LeappFileProcessor { return Collections.emptyList(); } - BlackboardAttribute attr = (value == null) ? null : getAttribute(colAttr.getAttributeType(), value, fileName); + String formattedValue = formatValueBasedOnAttrType(colAttr, value); + + BlackboardAttribute attr = (value == null) ? null : getAttribute(colAttr.getAttributeType(), formattedValue, fileName); if (attr == null) { logger.log(Level.WARNING, String.format("Blackboard attribute could not be parsed column %s at line %d in file %s. Omitting row.", colAttr.getColumnName(), lineNum, fileName)); return Collections.emptyList(); @@ -394,6 +397,21 @@ public final class LeappFileProcessor { return attrsToRet; } + /** + * Check type of attribute and possibly format string based on it. + * + * @param colAttr Column Attribute information + * @param value string to be formatted + * @return formatted string based on attribute type if no attribute type found then return original string + */ + private String formatValueBasedOnAttrType(TsvColumn colAttr, String value) { + if (colAttr.getAttributeType().getTypeName().equals("TSK_DOMAIN")) { + return NetworkUtils.extractDomain(value); + } + + return value; + } + /** * The format of time stamps in tsv. */ diff --git a/Core/src/org/sleuthkit/autopsy/report/modules/portablecase/PortableCaseReportModule.java b/Core/src/org/sleuthkit/autopsy/report/modules/portablecase/PortableCaseReportModule.java index 164c2e0faf..cc2adf377a 100644 --- a/Core/src/org/sleuthkit/autopsy/report/modules/portablecase/PortableCaseReportModule.java +++ b/Core/src/org/sleuthkit/autopsy/report/modules/portablecase/PortableCaseReportModule.java @@ -71,6 +71,7 @@ import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.ContentTag; import org.sleuthkit.datamodel.DataSource; import org.sleuthkit.datamodel.FileSystem; +import org.sleuthkit.datamodel.Host; import org.sleuthkit.datamodel.Image; import org.sleuthkit.datamodel.LocalFilesDataSource; import org.sleuthkit.datamodel.Pool; @@ -1079,12 +1080,20 @@ public class PortableCaseReportModule implements ReportModule { BlackboardArtifact artifactToCopy = (BlackboardArtifact) content; newContent = copyArtifact(parentId, artifactToCopy); } else { + + // Get or create the host (if needed) before beginning transaction. + Host newHost = null; + if (content instanceof DataSource) { + Host oldHost = ((DataSource)content).getHost(); + newHost = portableSkCase.getHostManager().getOrCreateHost(oldHost.getName()); + } + CaseDbTransaction trans = portableSkCase.beginTransaction(); try { if (content instanceof Image) { Image image = (Image) content; newContent = portableSkCase.addImage(image.getType(), image.getSsize(), image.getSize(), image.getName(), - new ArrayList<>(), image.getTimeZone(), image.getMd5(), image.getSha1(), image.getSha256(), image.getDeviceId(), trans); + new ArrayList<>(), image.getTimeZone(), image.getMd5(), image.getSha1(), image.getSha256(), image.getDeviceId(), newHost, trans); } else if (content instanceof VolumeSystem) { VolumeSystem vs = (VolumeSystem) content; newContent = portableSkCase.addVolumeSystem(parentId, vs.getType(), vs.getOffset(), vs.getBlockSize(), trans); @@ -1108,7 +1117,7 @@ public class PortableCaseReportModule implements ReportModule { if (abstractFile instanceof LocalFilesDataSource) { LocalFilesDataSource localFilesDS = (LocalFilesDataSource) abstractFile; - newContent = portableSkCase.addLocalFilesDataSource(localFilesDS.getDeviceId(), localFilesDS.getName(), localFilesDS.getTimeZone(), trans); + newContent = portableSkCase.addLocalFilesDataSource(localFilesDS.getDeviceId(), localFilesDS.getName(), localFilesDS.getTimeZone(), newHost, trans); } else { if (abstractFile.isDir()) { newContent = portableSkCase.addLocalDirectory(parentId, abstractFile.getName(), trans); diff --git a/CoreLibs/ivy.xml b/CoreLibs/ivy.xml index 1c83e48a09..7457154839 100644 --- a/CoreLibs/ivy.xml +++ b/CoreLibs/ivy.xml @@ -14,7 +14,7 @@ - + diff --git a/CoreLibs/nbproject/project.properties b/CoreLibs/nbproject/project.properties index fb53508bb4..3d8ebb951a 100644 --- a/CoreLibs/nbproject/project.properties +++ b/CoreLibs/nbproject/project.properties @@ -42,8 +42,8 @@ file.reference.javassist-3.12.1.GA.jar=release/modules/ext/javassist-3.12.1.GA.j file.reference.jfxtras-common-8.0-r4.jar=release/modules/ext/jfxtras-common-8.0-r4.jar file.reference.jfxtras-controls-8.0-r4.jar=release/modules/ext/jfxtras-controls-8.0-r4.jar file.reference.jfxtras-fxml-8.0-r4.jar=release/modules/ext/jfxtras-fxml-8.0-r4.jar -file.reference.jna-5.6.0.jar=release/modules/ext/jna-5.6.0.jar -file.reference.jna-platform-5.6.0.jar=release/modules/ext/jna-platform-5.6.0.jar +file.reference.jna-5.7.0.jar=release/modules/ext/jna-5.7.0.jar +file.reference.jna-platform-5.7.0.jar=release/modules/ext/jna-platform-5.7.0.jar file.reference.joda-time-2.4.jar=release/modules/ext/joda-time-2.4.jar file.reference.jsr305-1.3.9.jar=release/modules/ext/jsr305-1.3.9.jar file.reference.LGoodDatePicker-10.3.1.jar=release/modules/ext/LGoodDatePicker-10.3.1.jar diff --git a/CoreLibs/nbproject/project.xml b/CoreLibs/nbproject/project.xml index f697a317ee..24219c333d 100644 --- a/CoreLibs/nbproject/project.xml +++ b/CoreLibs/nbproject/project.xml @@ -923,8 +923,8 @@ release/modules/ext/commons-compress-1.18.jar - ext/jna-platform-5.6.0.jar - release\modules\ext\jna-platform-5.6.0.jar + ext/jna-platform-5.7.0.jar + release\modules\ext\jna-platform-5.7.0.jar ext/opencv-248.jar @@ -951,8 +951,8 @@ release/modules/ext/imageio-bmp-3.2.jar - ext/jna-5.6.0.jar - release\modules\ext\jna-5.6.0.jar + ext/jna-5.7.0.jar + release\modules\ext\jna-5.7.0.jar ext/commons-lang-2.6.jar diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/volatilityDSP/AddMemoryImageTask.java b/Experimental/src/org/sleuthkit/autopsy/experimental/volatilityDSP/AddMemoryImageTask.java index 7262d9f53e..d48121f21d 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/volatilityDSP/AddMemoryImageTask.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/volatilityDSP/AddMemoryImageTask.java @@ -34,6 +34,7 @@ import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.datamodel.Content; +import org.sleuthkit.datamodel.Host; /* * A runnable that adds a memory image data source to a case database. @@ -44,6 +45,7 @@ final class AddMemoryImageTask implements Runnable { private final String deviceId; private final String memoryImagePath; private final String timeZone; + private final Host host; private final List pluginsToRun; private final DataSourceProcessorProgressMonitor progressMonitor; private final DataSourceProcessorCallback callback; @@ -63,16 +65,18 @@ final class AddMemoryImageTask implements Runnable { * @param timeZone The time zone to use when processing dates and * times for the image, obtained from * java.util.TimeZone.getID. + * @param host The host for this data source (may be null). * @param progressMonitor Progress monitor for reporting progressMonitor * during processing. * @param callback Callback to call when processing is done. */ - AddMemoryImageTask(String deviceId, String memoryImagePath, String profile, List pluginsToRun, String timeZone, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callback) { + AddMemoryImageTask(String deviceId, String memoryImagePath, String profile, List pluginsToRun, String timeZone, Host host, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callback) { this.deviceId = deviceId; this.memoryImagePath = memoryImagePath; this.profile = profile; this.pluginsToRun = pluginsToRun; this.timeZone = timeZone; + this.host = host; this.callback = callback; this.progressMonitor = progressMonitor; } @@ -128,7 +132,7 @@ final class AddMemoryImageTask implements Runnable { /** * Attempts to add the input memory image to the case as a data source. * - * @return The Image object representation of the memeory image file data + * @return The Image object representation of the memory image file data * source. * * @throws NoCurrentCaseException If there is no current case. @@ -163,7 +167,7 @@ final class AddMemoryImageTask implements Runnable { * will need to be changed when a Device abstraction is added to the * SleuthKit data model. */ - Image dataSource = caseDatabase.addImageInfo(0, new ArrayList<>(Arrays.asList(memoryImagePath)), timeZone); + Image dataSource = caseDatabase.addImageInfo(0, new ArrayList<>(Arrays.asList(memoryImagePath)), timeZone, host); return dataSource; } diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/volatilityDSP/MemoryDSProcessor.java b/Experimental/src/org/sleuthkit/autopsy/experimental/volatilityDSP/MemoryDSProcessor.java index 5dbdca94ce..a0ca322fac 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/volatilityDSP/MemoryDSProcessor.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/volatilityDSP/MemoryDSProcessor.java @@ -23,9 +23,12 @@ import java.util.List; import javax.swing.JPanel; import org.openide.util.NbBundle.Messages; import org.openide.util.lookup.ServiceProvider; +import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorProgressMonitor; import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorCallback; import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessor; +import org.sleuthkit.datamodel.Host; +import org.sleuthkit.datamodel.TskCoreException; /** * A memory image data source processor that implements the DataSourceProcessor @@ -116,8 +119,37 @@ public class MemoryDSProcessor implements DataSourceProcessor { */ @Override public void run(DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callback) { + run(null, progressMonitor, callback); + } + + /** + * Adds a data source to the case database using a background task in a + * separate thread and the settings provided by the selection and + * configuration panel. Returns as soon as the background task is started. + * The background task uses a callback object to signal task completion and + * return results. + * + * This method should not be called unless isPanelValid returns true. + * + * @param host Host for the data source. + * @param progressMonitor Progress monitor that will be used by the + * background task to report progress. + * @param callback Callback that will be used by the background task + * to return results. + */ + @Override + public void run(Host host, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callback) { configPanel.storeSettings(); - run(UUID.randomUUID().toString(), configPanel.getImageFilePath(), configPanel.getProfile(), configPanel.getPluginsToRun(), configPanel.getTimeZone(), progressMonitor, callback); + + // HOSTTODO - replace with a call to configPanel().getHost() + try { + host = Case.getCurrentCase().getSleuthkitCase().getHostManager().getOrCreateHost("MemoryDSProcessor Host"); + } catch (TskCoreException ex) { + // It's not worth adding a logger for temporary code + //logger.log(Level.SEVERE, "Error creating/loading host", ex); + host = null; + } + run(UUID.randomUUID().toString(), configPanel.getImageFilePath(), configPanel.getProfile(), configPanel.getPluginsToRun(), configPanel.getTimeZone(), host, progressMonitor, callback); } /** @@ -136,12 +168,13 @@ public class MemoryDSProcessor implements DataSourceProcessor { * @param timeZone The time zone to use when processing dates and * times for the image, obtained from * java.util.TimeZone.getID. + * @param host The host for this data source (may be null) * @param progressMonitor Progress monitor for reporting progress during * processing. * @param callback Callback to call when processing is done. */ - private void run(String deviceId, String memoryImagePath, String profile, List pluginsToRun, String timeZone, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callback) { - addImageTask = new AddMemoryImageTask(deviceId, memoryImagePath, profile, pluginsToRun, timeZone, progressMonitor, callback); + private void run(String deviceId, String memoryImagePath, String profile, List pluginsToRun, String timeZone, Host host, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callback) { + addImageTask = new AddMemoryImageTask(deviceId, memoryImagePath, profile, pluginsToRun, timeZone, host, progressMonitor, callback); new Thread(addImageTask).start(); } diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java index 95509c4c16..2f13a48fd2 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java @@ -95,8 +95,7 @@ public final class ImageGalleryController { private static final Logger logger = Logger.getLogger(ImageGalleryController.class.getName()); private static final Set INGEST_JOB_EVENTS_OF_INTEREST = EnumSet.of(IngestManager.IngestJobEvent.DATA_SOURCE_ANALYSIS_STARTED, IngestManager.IngestJobEvent.DATA_SOURCE_ANALYSIS_COMPLETED); private static final Set INGEST_MODULE_EVENTS_OF_INTEREST = EnumSet.of(IngestManager.IngestModuleEvent.DATA_ADDED, IngestManager.IngestModuleEvent.FILE_DONE); - - private static String DEFAULT_TAG_SET_NAME = "Project VIC"; + /* * The file limit for image gallery. If the selected data source (or all * data sources, if that option is selected) has more than this many files @@ -738,7 +737,7 @@ public final class ImageGalleryController { List tagSetList = getCaseDatabase().getTaggingManager().getTagSets(); if (tagSetList != null && !tagSetList.isEmpty()) { for (TagSet set : tagSetList) { - if (set.getName().equals(getCategoryTagSetName())) { + if (set.getName().equals(ImageGalleryService.PROJECT_VIC_TAG_SET_NAME)) { return set; } } @@ -749,14 +748,6 @@ public final class ImageGalleryController { } } - /** - * Returns the name of the category tag set. - * - * @return Tagset name - */ - static String getCategoryTagSetName() { - return DEFAULT_TAG_SET_NAME; - } /** * A listener for ingest module application events. @@ -839,8 +830,11 @@ public final class ImageGalleryController { Content newDataSource = (Content) event.getNewValue(); if (isListeningEnabled()) { try { - drawableDB.insertOrUpdateDataSource(newDataSource.getId(), DrawableDB.DrawableDbBuildStatusEnum.UNKNOWN); - } catch (SQLException ex) { + // If the data source already exists and has a status other than UNKNOWN, don’t overwrite it. + if(drawableDB.getDataSourceDbBuildStatus(newDataSource.getId()) == DrawableDB.DrawableDbBuildStatusEnum.UNKNOWN) { + drawableDB.insertOrUpdateDataSource(newDataSource.getId(), DrawableDB.DrawableDbBuildStatusEnum.UNKNOWN); + } + } catch (SQLException | TskCoreException ex) { logger.log(Level.SEVERE, String.format("Error updating datasources table (data source object ID = %d, status = %s)", newDataSource.getId(), DrawableDB.DrawableDbBuildStatusEnum.UNKNOWN.toString()), ex); //NON-NLS } } diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryService.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryService.java index 176b6e1a44..4a8664eb6f 100755 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryService.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryService.java @@ -44,20 +44,32 @@ import org.sleuthkit.datamodel.TskData; }) public class ImageGalleryService implements AutopsyService { - private static final String CATEGORY_ONE_NAME = "Child Exploitation (Illegal)"; - private static final String CATEGORY_TWO_NAME = "Child Exploitation (Non-Illegal/Age Difficult)"; - private static final String CATEGORY_THREE_NAME = "CGI/Animation (Child Exploitive)"; - private static final String CATEGORY_FOUR_NAME = "Exemplar/Comparison (Internal Use Only)"; - private static final String CATEGORY_FIVE_NAME = "Non-pertinent"; - - private static final List DEFAULT_CATEGORY_DEFINITION = new ArrayList<>(); + /* Image Gallery has its own definition of Project VIC tag names because + * these will be used if the Project Vic module is not installed. These will + * get added when a case is opened if the tag set is not already defined. + * + * The following list of names must be kept in sync with the CountryManager + * code in the ProjectVic module. + * + * Autopsy Core Tag code and TSK DataModel upgrade code also have a + * references to the "Projet VIC" set name. Be careful changing any of these names. + */ + static String PROJECT_VIC_TAG_SET_NAME = "Project VIC"; + private static final String PV_US_CAT0 = "Non-Pertinent"; + private static final String PV_US_CAT1 = "Child Abuse Material - (CAM)"; + private static final String PV_US_CAT2 = "Child Exploitive (Non-CAM) Age Difficult"; + private static final String PV_US_CAT3 = "CGI/Animation - Child Exploitive"; + private static final String PV_US_CAT4 = "Comparison Images"; + + private static final List PROJECT_VIC_US_CATEGORIES = new ArrayList<>(); static { - DEFAULT_CATEGORY_DEFINITION.add(new TagNameDefinition(CATEGORY_ONE_NAME, "", TagName.HTML_COLOR.RED, TskData.FileKnown.BAD)); - DEFAULT_CATEGORY_DEFINITION.add(new TagNameDefinition(CATEGORY_TWO_NAME, "", TagName.HTML_COLOR.LIME, TskData.FileKnown.BAD)); - DEFAULT_CATEGORY_DEFINITION.add(new TagNameDefinition(CATEGORY_THREE_NAME, "", TagName.HTML_COLOR.YELLOW, TskData.FileKnown.BAD)); - DEFAULT_CATEGORY_DEFINITION.add(new TagNameDefinition(CATEGORY_FOUR_NAME, "", TagName.HTML_COLOR.PURPLE, TskData.FileKnown.UNKNOWN)); - DEFAULT_CATEGORY_DEFINITION.add(new TagNameDefinition(CATEGORY_FIVE_NAME, "", TagName.HTML_COLOR.FUCHSIA, TskData.FileKnown.UNKNOWN)); + // NOTE: The colors here are what will be shown in the border + PROJECT_VIC_US_CATEGORIES.add(new TagNameDefinition(PV_US_CAT0, "", TagName.HTML_COLOR.GREEN, TskData.FileKnown.UNKNOWN)); + PROJECT_VIC_US_CATEGORIES.add(new TagNameDefinition(PV_US_CAT1, "", TagName.HTML_COLOR.RED, TskData.FileKnown.BAD)); + PROJECT_VIC_US_CATEGORIES.add(new TagNameDefinition(PV_US_CAT2, "", TagName.HTML_COLOR.YELLOW, TskData.FileKnown.BAD)); + PROJECT_VIC_US_CATEGORIES.add(new TagNameDefinition(PV_US_CAT3, "", TagName.HTML_COLOR.FUCHSIA, TskData.FileKnown.BAD)); + PROJECT_VIC_US_CATEGORIES.add(new TagNameDefinition(PV_US_CAT4, "", TagName.HTML_COLOR.BLUE, TskData.FileKnown.UNKNOWN)); } @Override @@ -91,17 +103,17 @@ public class ImageGalleryService implements AutopsyService { // Check to see if the Project VIC tag set exists, if not create a // tag set using the default tags. - boolean addDefaultTagSet = true; + boolean addProjVicTagSet = true; List tagSets = context.getCase().getServices().getTagsManager().getAllTagSets(); for (TagSet set : tagSets) { - if (set.getName().equals(ImageGalleryController.getCategoryTagSetName())) { - addDefaultTagSet = false; + if (set.getName().equals(PROJECT_VIC_TAG_SET_NAME)) { + addProjVicTagSet = false; break; } } - if (addDefaultTagSet) { - addDefaultTagSet(context.getCase()); + if (addProjVicTagSet) { + addProjetVicTagSet(context.getCase()); } ImageGalleryController.createController(context.getCase()); @@ -134,13 +146,11 @@ public class ImageGalleryService implements AutopsyService { * * @throws TskCoreException */ - private void addDefaultTagSet(Case currentCase) throws TskCoreException { + private void addProjetVicTagSet(Case currentCase) throws TskCoreException { List tagNames = new ArrayList<>(); - for (TagNameDefinition def : DEFAULT_CATEGORY_DEFINITION) { + for (TagNameDefinition def : PROJECT_VIC_US_CATEGORIES) { tagNames.add(currentCase.getSleuthkitCase().addOrUpdateTagName(def.getDisplayName(), def.getDescription(), def.getColor(), def.getKnownStatus())); } - - currentCase.getServices().getTagsManager().addTagSet(ImageGalleryController.getCategoryTagSetName(), tagNames); + currentCase.getServices().getTagsManager().addTagSet(PROJECT_VIC_TAG_SET_NAME, tagNames); } - } diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/actions/OpenAction.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/actions/OpenAction.java index 2df0ee2ed1..92ea324105 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/actions/OpenAction.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/actions/OpenAction.java @@ -157,7 +157,10 @@ public final class OpenAction extends CallableSystemAction { } Platform.runLater(() -> { ImageGalleryController controller; + // @@@ This call gets a lock. We shouldn't do this in the UI.... controller = ImageGalleryController.getController(currentCase); + + // Display an error if we could not get the controller and return if (controller == null) { Alert errorDIalog = new Alert(Alert.AlertType.ERROR); errorDIalog.initModality(Modality.APPLICATION_MODAL); @@ -174,6 +177,7 @@ public final class OpenAction extends CallableSystemAction { return; } + // Make sure the user is aware of Single vs Multi-user behaviors if (currentCase.getCaseType() == Case.CaseType.MULTI_USER_CASE && ImageGalleryPreferences.isMultiUserCaseInfoDialogDisabled() == false) { Alert dialog = new Alert(Alert.AlertType.INFORMATION); diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/CategoryManager.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/CategoryManager.java index 8419fb40fe..4464a60fc7 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/CategoryManager.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/CategoryManager.java @@ -99,9 +99,9 @@ public class CategoryManager { } /** - * get the number of file with the given {@link DhsImageCategory} + * get the number of file with the given tag * - * @param cat get the number of files with Category = cat + * @param tagName get the number of files with Category = tagName * * @return the number of files with the given Category */ @@ -110,20 +110,18 @@ public class CategoryManager { } /** - * increment the cached value for the number of files with the given - * {@link DhsImageCategory} + * increment the cached value for the number of files with the given tag * - * @param cat the Category to increment + * @param tagName the Category to increment */ synchronized public void incrementCategoryCount(TagName tagName) { categoryCounts.getUnchecked(tagName).increment(); } /** - * decrement the cached value for the number of files with the given - * DhsImageCategory + * decrement the cached value for the number of files with the given tag * - * @param cat the Category to decrement + * @param tagName the Category to decrement */ synchronized public void decrementCategoryCount(TagName tagName) { categoryCounts.getUnchecked(tagName).decrement(); diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java index 3aa39bd2fd..ab4a2cbe6a 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java @@ -207,19 +207,19 @@ public final class DrawableDB { */ UNKNOWN, /** - * Analyis (an ingest job or image gallery database rebuild) for the + * Analysis (an ingest job or image gallery database rebuild) for the * data source is in progress. */ IN_PROGRESS, /** - * Analyis (an ingest job or image gallery database rebuild) for the + * Analysis (an ingest job or image gallery database rebuild) for the * data source has been completed and at least one file in the data * source has a MIME type (ingest filters may have been applied, so some * files may not have been typed). */ COMPLETE, /** - * Analyis (an ingest job or image gallery database rebuild) for the + * Analysis (an ingest job or image gallery database rebuild) for the * data source has been completed, but the files for the data source * were not assigned a MIME type (file typing was not enabled). */ diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties index b4e2850586..d2ab5f76c7 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties @@ -187,7 +187,7 @@ Server.query.exception.msg=Error running query: {0} Server.query2.exception.msg=Error running query: {0} Server.queryTerms.exception.msg=Error running terms query: {0} Server.connect.exception.msg=Failed to connect to Solr server: {0} -Server.openCore.exception.msg=Keyword search service not yet running +Server.openCore.exception.msg=Local keyword search service not yet running Server.openCore.exception.cantOpen.msg=Could not create or open index Server.openCore.exception.noIndexDir.msg=Index directory could not be created or is missing Server.request.exception.exception.msg=Could not issue Solr request diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Installer.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Installer.java index 04bcbf6940..126d7fa4ba 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Installer.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Installer.java @@ -18,6 +18,9 @@ */ package org.sleuthkit.autopsy.keywordsearch; +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.logging.Level; import org.apache.solr.client.solrj.SolrServerException; import org.openide.modules.ModuleInstall; @@ -39,6 +42,7 @@ class Installer extends ModuleInstall { private static final Logger logger = Logger.getLogger(Installer.class.getName()); private static final long serialVersionUID = 1L; + private static final String KWS_START_THREAD_NAME = "KWS-server-start-%d"; @Override public void restored() { @@ -46,19 +50,29 @@ class Installer extends ModuleInstall { KeywordSearchSettings.setDefaults(); final Server server = KeywordSearch.getServer(); - try { - server.start(); - } catch (SolrServerNoPortException ex) { - logger.log(Level.SEVERE, "Failed to start Keyword Search server: ", ex); //NON-NLS - if (ex.getPortNumber() == server.getLocalSolrServerPort()) { - reportPortError(ex.getPortNumber()); - } else { - reportStopPortError(ex.getPortNumber()); + + ExecutorService jobProcessingExecutor = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setNameFormat(KWS_START_THREAD_NAME).build()); + Runnable kwsStartTask = new Runnable() { + public void run() { + try { + server.start(); + } catch (SolrServerNoPortException ex) { + logger.log(Level.SEVERE, "Failed to start Keyword Search server: ", ex); //NON-NLS + if (ex.getPortNumber() == server.getLocalSolrServerPort()) { + reportPortError(ex.getPortNumber()); + } else { + reportStopPortError(ex.getPortNumber()); + } + } catch (KeywordSearchModuleException | SolrServerException ex) { + logger.log(Level.SEVERE, "Failed to start Keyword Search server: ", ex); //NON-NLS + reportInitError(ex.getMessage()); + } } - } catch (KeywordSearchModuleException | SolrServerException ex) { - logger.log(Level.SEVERE, "Failed to start Keyword Search server: ", ex); //NON-NLS - reportInitError(ex.getMessage()); - } + }; + + // start KWS service on the background thread. Currently all it does is start the embedded Solr server. + jobProcessingExecutor.submit(kwsStartTask); + jobProcessingExecutor.shutdown(); // tell executor no more work is coming } @Override diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java index 019f42e3de..5135b3cf30 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java @@ -31,6 +31,7 @@ import java.io.InputStreamReader; import java.io.OutputStream; import java.io.OutputStreamWriter; import java.net.ConnectException; +import java.net.DatagramSocket; import java.net.ServerSocket; import java.net.SocketException; import java.nio.charset.Charset; @@ -247,6 +248,8 @@ public class Server { private static final String CORE_PROPERTIES = "core.properties"; private static final boolean DEBUG = false;//(Version.getBuildType() == Version.Type.DEVELOPMENT); private static final int NUM_COLLECTION_CREATION_RETRIES = 5; + private static final int NUM_EMBEDDED_SERVER_RETRIES = 12; // attempt to connect to embedded Solr server for 1 minute + private static final int EMBEDDED_SERVER_RETRY_WAIT_SEC = 5; public enum CORE_EVT_STATES { @@ -277,6 +280,8 @@ public class Server { */ Server() { initSettings(); + + localSolrServer = getSolrClient("http://localhost:" + localSolrServerPort + "/solr"); serverAction = new ServerAction(); File solr8Folder = InstalledFileLocator.getDefault().locate("solr", Server.class.getPackage().getName(), false); //NON-NLS @@ -672,11 +677,13 @@ public class Server { */ @NbBundle.Messages({ "Server.status.failed.msg=Local Solr server did not respond to status request. This may be because the server failed to start or is taking too long to initialize.",}) - void startLocalSolr(SOLR_VERSION version) throws KeywordSearchModuleException, SolrServerNoPortException, SolrServerException { + synchronized void startLocalSolr(SOLR_VERSION version) throws KeywordSearchModuleException, SolrServerNoPortException, SolrServerException { + logger.log(Level.INFO, "Starting local Solr " + version + " server"); //NON-NLS if (isLocalSolrRunning()) { if (localServerVersion.equals(version)) { // this version of local server is already running + logger.log(Level.INFO, "Local Solr " + version + " server is already running"); //NON-NLS return; } else { // wrong version of local server is running, stop it @@ -720,7 +727,7 @@ public class Server { logger.log(Level.INFO, "Starting Solr 8 server"); //NON-NLS localSolrFolder = InstalledFileLocator.getDefault().locate("solr", Server.class.getPackage().getName(), false); //NON-NLS curSolrProcess = runLocalSolr8ControlCommand(new ArrayList<>(Arrays.asList("start", "-p", //NON-NLS - Integer.toString(localSolrServerPort)))); //NON-NLS + Integer.toString(localSolrServerPort)))); //NON-NLS } else { // solr4 localSolrFolder = InstalledFileLocator.getDefault().locate("solr4", Server.class.getPackage().getName(), false); //NON-NLS @@ -729,11 +736,10 @@ public class Server { Arrays.asList("-Dbootstrap_confdir=../solr/configsets/AutopsyConfig/conf", //NON-NLS "-Dcollection.configName=AutopsyConfig"))); //NON-NLS } - + // Wait for the Solr server to start and respond to a statusRequest request. - for (int numRetries = 0; numRetries < 6; numRetries++) { + for (int numRetries = 0; numRetries < NUM_EMBEDDED_SERVER_RETRIES; numRetries++) { if (isLocalSolrRunning()) { - localSolrServer = getSolrClient("http://localhost:" + localSolrServerPort + "/solr"); final List pids = this.getSolrPIDs(); logger.log(Level.INFO, "New Solr process PID: {0}", pids); //NON-NLS return; @@ -742,7 +748,7 @@ public class Server { // Local Solr server did not respond so we sleep for // 5 seconds before trying again. try { - TimeUnit.SECONDS.sleep(5); + TimeUnit.SECONDS.sleep(EMBEDDED_SERVER_RETRY_WAIT_SEC); } catch (InterruptedException ex) { logger.log(Level.WARNING, "Timer interrupted"); //NON-NLS } @@ -775,6 +781,23 @@ public class Server { * @param port the port to check for availability */ static boolean isPortAvailable(int port) { + final String osName = PlatformUtil.getOSName().toLowerCase(); + if (osName != null && osName.toLowerCase().startsWith("mac")) { + return isPortAvailableOSX(port); + } else { + return isPortAvailableDefault(port); + } + } + + /** + * Checks to see if a specific port is available. + * + * NOTE: This is used on non-OS X systems as of right now but could be + * replaced with the OS X version. + * + * @param port the port to check for availability + */ + static boolean isPortAvailableDefault(int port) { ServerSocket ss = null; try { @@ -800,6 +823,48 @@ public class Server { return false; } + /** + * Checks to see if a specific port is available. + * + * NOTE: This is only used on OSX for now, but could replace default + * implementation in the future. + * + * @param port The port to check for availability. + * @throws IllegalArgumentException If port is outside range of possible ports. + */ + static boolean isPortAvailableOSX(int port) { + // implementation taken from https://stackoverflow.com/a/435579 + if (port < 1 || port > 65535) { + throw new IllegalArgumentException("Invalid start port: " + port); + } + + ServerSocket ss = null; + DatagramSocket ds = null; + try { + ss = new ServerSocket(port); + ss.setReuseAddress(true); + ds = new DatagramSocket(port); + ds.setReuseAddress(true); + return true; + } catch (IOException e) { + } finally { + if (ds != null) { + ds.close(); + } + + if (ss != null) { + try { + ss.close(); + } catch (IOException e) { + /* should not be thrown */ + } + } + } + + return false; + } + + /** * Changes the current solr server port. Only call this after available. * @@ -1902,13 +1967,22 @@ public class Server { * @throws IOException */ private void connectToEmbeddedSolrServer() throws SolrServerException, IOException { - HttpSolrClient solrServer = getSolrClient("http://localhost:" + localSolrServerPort + "/solr"); TimingMetric metric = HealthMonitor.getTimingMetric("Solr: Connectivity check"); - CoreAdminRequest.getStatus(null, solrServer); + CoreAdminRequest.getStatus(null, localSolrServer); HealthMonitor.submitTimingMetric(metric); } - + /** + * Attempts to connect to the given Solr server, which is running in + * SoulrCloud mode. This API does not work for the local Solr which is NOT + * running in SolrCloud mode. + * + * @param host Host name of the remote Solr server + * @param port Port of the remote Solr server + * + * @throws SolrServerException + * @throws IOException + */ void connectToSolrServer(String host, String port) throws SolrServerException, IOException { try (HttpSolrClient solrServer = getSolrClient("http://" + host + ":" + port + "/solr")) { connectToSolrServer(solrServer); @@ -1972,47 +2046,7 @@ public class Server { throw new KeywordSearchModuleException( NbBundle.getMessage(this.getClass(), "Server.serverList.exception.msg", solrServer.getBaseURL())); } - } - - /* ELTODO leaving this for reference, will delete later - private boolean clusterStatusWithCollection(String collectionName) throws IOException, SolrServerException { - ModifiableSolrParams params = new ModifiableSolrParams(); - params.set("action", CollectionParams.CollectionAction.CLUSTERSTATUS.toString()); - params.set("collection", collectionName); - SolrRequest request = new QueryRequest(params); - request.setPath("/admin/collections"); - - NamedList statusResponse; - try { - statusResponse = currentSolrServer.request(request); - } catch (RemoteSolrException ex) { - // collection doesn't exist - return false; - } - - if (statusResponse == null) { - logger.log(Level.SEVERE, "Collections response should not be null"); //NON-NLS - return false; - } - - NamedList cluster = (NamedList) statusResponse.get("cluster"); - if (cluster == null) { - logger.log(Level.SEVERE, "Cluster should not be null"); //NON-NLS - return false; - } - NamedList collections = (NamedList) cluster.get("collections"); - if (cluster == null) { - logger.log(Level.SEVERE, "Collections should not be null in cluster state"); //NON-NLS - return false; - } - if (collections.size() == 0) { - logger.log(Level.SEVERE, "Collections should not be empty in cluster state"); //NON-NLS - return false; - } - - Object collection = collections.get(collectionName); - return (collection != null); - }*/ + } class Collection { diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java index 7eea0c3c2f..218be24de8 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java @@ -401,7 +401,9 @@ public class SolrSearchService implements KeywordSearchService, AutopsyService { throw new AutopsyServiceException(String.format("Failed to close core for %s", context.getCase().getCaseDirectory()), ex); } - context.getCase().getSleuthkitCase().unregisterForEvents(this); + if (context.getCase().getSleuthkitCase() != null) { + context.getCase().getSleuthkitCase().unregisterForEvents(this); + } } /** diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Chromium.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Chromium.java index 9fd2acf40f..35417a9aa2 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Chromium.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Chromium.java @@ -2,7 +2,7 @@ * * Autopsy Forensic Browser * - * Copyright 2012-2020 Basis Technology Corp. + * Copyright 2012-2021 Basis Technology Corp. * * Copyright 2012 42six Solutions. * @@ -487,7 +487,7 @@ class Chromium extends Extract { bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL, RecentActivityExtracterModuleFactory.getModuleName(), ((result.get("host_key").toString() != null) ? result.get("host_key").toString() : ""))); //NON-NLS - bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME, + bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_ACCESSED, RecentActivityExtracterModuleFactory.getModuleName(), (Long.valueOf(result.get("last_access_utc").toString()) / 1000000) - Long.valueOf("11644473600"))); //NON-NLS diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Extract.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Extract.java index ebf483b5d0..fa23d8564f 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Extract.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Extract.java @@ -2,7 +2,7 @@ * * Autopsy Forensic Browser * - * Copyright 2012-2019 Basis Technology Corp. + * Copyright 2012-2021 Basis Technology Corp. * * Copyright 2012 42six Solutions. * Contact: aebadirad 42six com @@ -325,17 +325,27 @@ abstract class Extract { * @return List of BlackboarAttributes for the passed in attributes */ protected Collection createCookieAttributes(String url, - Long creationTime, String name, String value, String programName, String domain) { + Long creationTime, Long accessTime, Long endTime, String name, String value, String programName, String domain) { Collection bbattributes = new ArrayList<>(); bbattributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_URL, RecentActivityExtracterModuleFactory.getModuleName(), (url != null) ? url : "")); //NON-NLS - if (creationTime != null) { - bbattributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME, + if (creationTime != null && creationTime != 0) { + bbattributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME_CREATED, RecentActivityExtracterModuleFactory.getModuleName(), creationTime)); } + + if (accessTime != null && accessTime != 0) { + bbattributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME_ACCESSED, + RecentActivityExtracterModuleFactory.getModuleName(), accessTime)); + } + + if(endTime != null && endTime != 0) { + bbattributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME_END, + RecentActivityExtracterModuleFactory.getModuleName(), endTime)); + } bbattributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_NAME, RecentActivityExtracterModuleFactory.getModuleName(), diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractEdge.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractEdge.java index 77939c3b6f..6db3d75c02 100755 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractEdge.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractEdge.java @@ -2,7 +2,7 @@ * * Autopsy Forensic Browser * - * Copyright 2019-2020 Basis Technology Corp. + * Copyright 2019-2021 Basis Technology Corp. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -659,7 +659,7 @@ final class ExtractEdge extends Extract { String url = flipDomain(domain); BlackboardArtifact bbart = origFile.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_COOKIE); - bbart.addAttributes(createCookieAttributes(url, ftime, name, value, this.getName(), NetworkUtils.extractDomain(url))); + bbart.addAttributes(createCookieAttributes(url, null, ftime, null, name, value, this.getName(), NetworkUtils.extractDomain(url))); return bbart; } diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractIE.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractIE.java index 50568a2b2e..93f901d1a2 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractIE.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractIE.java @@ -267,7 +267,7 @@ class ExtractIE extends Extract { Collection bbattributes = new ArrayList<>(); bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL, RecentActivityExtracterModuleFactory.getModuleName(), url)); - bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME, + bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_CREATED, RecentActivityExtracterModuleFactory.getModuleName(), datetime)); bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_NAME, RecentActivityExtracterModuleFactory.getModuleName(), (name != null) ? name : "")); diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractSafari.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractSafari.java index 5e1194d78b..676ff7923d 100755 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractSafari.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractSafari.java @@ -2,7 +2,7 @@ * * Autopsy Forensic Browser * - * Copyright 2019 Basis Technology Corp. + * Copyright 2019-2021 Basis Technology Corp. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -566,7 +566,7 @@ final class ExtractSafari extends Extract { Cookie cookie = iter.next(); BlackboardArtifact bbart = origFile.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_COOKIE); - bbart.addAttributes(createCookieAttributes(cookie.getURL(), cookie.getCreationDate(), cookie.getName(), cookie.getValue(), this.getName(), NetworkUtils.extractDomain(cookie.getURL()))); + bbart.addAttributes(createCookieAttributes(cookie.getURL(), cookie.getCreationDate(), null, cookie.getExpirationDate(), cookie.getName(), cookie.getValue(), this.getName(), NetworkUtils.extractDomain(cookie.getURL()))); bbartifacts.add(bbart); } } diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Firefox.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Firefox.java index ba1cba3394..684d373519 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Firefox.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Firefox.java @@ -2,7 +2,7 @@ * * Autopsy Forensic Browser * - * Copyright 2012-2020 Basis Technology Corp. + * Copyright 2012-2021 Basis Technology Corp. * * Copyright 2012 42six Solutions. * Contact: aebadirad 42six com @@ -423,7 +423,7 @@ class Firefox extends Extract { bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL, RecentActivityExtracterModuleFactory.getModuleName(), ((host != null) ? host : ""))); //NON-NLS - bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME, + bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_ACCESSED, RecentActivityExtracterModuleFactory.getModuleName(), (Long.valueOf(result.get("lastAccessed").toString())))); //NON-NLS bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_NAME, diff --git a/test/script/tskdbdiff.py b/test/script/tskdbdiff.py index 1c8f83e481..762735c8e3 100644 --- a/test/script/tskdbdiff.py +++ b/test/script/tskdbdiff.py @@ -469,9 +469,9 @@ def normalize_db_entry(line, files_table, vs_parts_table, vs_info_table, fs_info # Ignore TIFF size and hash if extracted from PDFs. # See JIRA-6951 for more details. - # index -1 = last element in the list, which is extension - # index -3 = 3rd from the end, which is the parent path. - if fields_list[-1] == "'tif'" and fields_list[-3].endswith(".pdf/'"): + # index -3 = 3rd from the end, which is extension + # index -5 = 5th from the end, which is the parent path. + if fields_list[-3] == "'tif'" and fields_list[-5].endswith(".pdf/'"): fields_list[15] = "'SIZE_IGNORED'" fields_list[23] = "'MD5_IGNORED'" fields_list[24] = "'SHA256_IGNORED'" diff --git a/thirdparty/java-libpst/java-libpst-0.9.5-SNAPSHOT.jar b/thirdparty/java-libpst/java-libpst-0.9.5-SNAPSHOT.jar new file mode 100644 index 0000000000..93ff797a6a Binary files /dev/null and b/thirdparty/java-libpst/java-libpst-0.9.5-SNAPSHOT.jar differ diff --git a/thirdparty/java-libpst/java-libpst-1.0-SNAPSHOT.jar b/thirdparty/java-libpst/java-libpst-1.0-SNAPSHOT.jar deleted file mode 100755 index 8489f660b9..0000000000 Binary files a/thirdparty/java-libpst/java-libpst-1.0-SNAPSHOT.jar and /dev/null differ diff --git a/thunderbirdparser/build.xml b/thunderbirdparser/build.xml index caabb5f0de..8926bca7a9 100644 --- a/thunderbirdparser/build.xml +++ b/thunderbirdparser/build.xml @@ -19,7 +19,7 @@ - + diff --git a/thunderbirdparser/nbproject/project.properties b/thunderbirdparser/nbproject/project.properties index d85c634cf8..5ed2afa7b7 100644 --- a/thunderbirdparser/nbproject/project.properties +++ b/thunderbirdparser/nbproject/project.properties @@ -7,14 +7,16 @@ file.reference.apache-mime4j-dom-0.8.0.jar=release/modules/ext/apache-mime4j-dom file.reference.apache-mime4j-mbox-iterator-0.8.0.jar=release/modules/ext/apache-mime4j-mbox-iterator-0.8.0-SNAPSHOT.jar file.reference.commons-validator-1.6.jar=release/modules/ext/commons-validator-1.6.jar file.reference.guava-19.0.jar=release/modules/ext/guava-19.0.jar -file.reference.java-libpst-1.0-SNAPSHOT.jar=release/modules/ext/java-libpst-1.0-SNAPSHOT.jar file.reference.ez-vcard-0.10.5.jar=release/modules/ext/ez-vcard-0.10.5.jar +file.reference.java-libpst-0.9.5-SNAPSHOT.jar=release/modules/ext/java-libpst-0.9.5-SNAPSHOT.jar file.reference.vinnie-2.0.2.jar=release/modules/ext/vinnie-2.0.2.jar javac.source=1.8 javac.compilerargs=-Xlint -Xlint:-serial javadoc.reference.guava-19.0.jar=release/modules/ext/guava-19.0-javadoc.jar +javadoc.reference.java-libpst-0.9.5-SNAPSHOT.jar=release/modules/ext/java-libpst-0.9.5-SNAPSHOT.jar license.file=../LICENSE-2.0.txt nbm.homepage=http://www.sleuthkit.org/autopsy/ nbm.needs.restart=true source.reference.guava-19.0.jar=release/modules/ext/guava-19.0-sources.jar +source.reference.java-libpst-0.9.5-SNAPSHOT.jar=release/modules/ext/java-libpst-0.9.5-SNAPSHOT.jar spec.version.base=4.0 diff --git a/thunderbirdparser/nbproject/project.xml b/thunderbirdparser/nbproject/project.xml index 9a1249b697..318b4e36ee 100644 --- a/thunderbirdparser/nbproject/project.xml +++ b/thunderbirdparser/nbproject/project.xml @@ -84,14 +84,14 @@ ext/apache-mime4j-core-0.8.0-SNAPSHOT.jar release/modules/ext/apache-mime4j-core-0.8.0-SNAPSHOT.jar - - ext/java-libpst-1.0-SNAPSHOT.jar - release/modules/ext/java-libpst-1.0-SNAPSHOT.jar - ext/guava-19.0.jar release/modules/ext/guava-19.0.jar + + ext/java-libpst-0.9.5-SNAPSHOT.jar + release/modules/ext/java-libpst-0.9.5-SNAPSHOT.jar + ext/apache-mime4j-dom-0.8.0-SNAPSHOT.jar release/modules/ext/apache-mime4j-dom-0.8.0-SNAPSHOT.jar diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/EmailMessage.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/EmailMessage.java index 09a6637e6e..5e70a5eae2 100644 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/EmailMessage.java +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/EmailMessage.java @@ -43,7 +43,7 @@ class EmailMessage { private String localPath = ""; private boolean hasAttachment = false; private long sentDate = 0L; - private List attachments = new ArrayList<>(); + private final List attachments = new ArrayList<>(); private long id = -1L; private String messageID = ""; private String inReplyToID = ""; @@ -410,4 +410,16 @@ class EmailMessage { } } + + static class AttachedEmailMessage extends Attachment { + private final EmailMessage emailMessage; + + AttachedEmailMessage(EmailMessage emailMessage) { + this.emailMessage = emailMessage; + } + + EmailMessage getEmailMessage() { + return emailMessage; + } + } } diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/MimeJ4MessageParser.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/MimeJ4MessageParser.java index ecefe871f1..ba5b24842f 100755 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/MimeJ4MessageParser.java +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/MimeJ4MessageParser.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2019 Basis Technology Corp. + * Copyright 2019-2020 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -26,12 +26,11 @@ import java.util.ArrayList; import java.util.List; import java.util.UUID; import java.util.logging.Level; -import org.apache.james.mime4j.dom.BinaryBody; import org.apache.james.mime4j.dom.Body; import org.apache.james.mime4j.dom.Entity; import org.apache.james.mime4j.dom.Message; +import org.apache.james.mime4j.dom.MessageWriter; import org.apache.james.mime4j.dom.Multipart; -import org.apache.james.mime4j.dom.SingleBody; import org.apache.james.mime4j.dom.TextBody; import org.apache.james.mime4j.dom.address.AddressList; import org.apache.james.mime4j.dom.address.Mailbox; @@ -39,6 +38,7 @@ import org.apache.james.mime4j.dom.address.MailboxList; import org.apache.james.mime4j.dom.field.ContentDispositionField; import org.apache.james.mime4j.dom.field.ContentTypeField; import org.apache.james.mime4j.message.DefaultMessageBuilder; +import org.apache.james.mime4j.message.DefaultMessageWriter; import org.apache.james.mime4j.stream.Field; import org.apache.james.mime4j.stream.MimeConfig; import org.openide.util.NbBundle; @@ -293,7 +293,7 @@ class MimeJ4MessageParser implements AutoCloseable{ * @param e */ @NbBundle.Messages({"MimeJ4MessageParser.handleAttch.noOpenCase.errMsg=Exception while getting open case."}) - private static void handleAttachment(EmailMessage email, Entity e, long fileID, int index) { + private void handleAttachment(EmailMessage email, Entity e, long fileID, int index) { String outputDirPath; String relModuleOutputPath; try { @@ -322,25 +322,31 @@ class MimeJ4MessageParser implements AutoCloseable{ String outPath = outputDirPath + uniqueFilename; Body body = e.getBody(); - if (body instanceof SingleBody) { + if (body != null) { long fileLength; try (EncodedFileOutputStream fos = new EncodedFileOutputStream(new FileOutputStream(outPath), TskData.EncodingType.XOR1)) { - ((SingleBody) body).writeTo(fos); + + EmailMessage.Attachment attach; + MessageWriter msgWriter = new DefaultMessageWriter(); + + if(body instanceof Message) { + msgWriter.writeMessage((Message)body, fos); + attach = new EmailMessage.AttachedEmailMessage(extractEmail((Message)body, email.getLocalPath(), fileID)); + } else { + msgWriter.writeBody(body, fos); + attach = new EmailMessage.Attachment(); + } fileLength = fos.getBytesWritten(); + attach.setName(filename); + attach.setLocalPath(relModuleOutputPath + uniqueFilename); + attach.setSize(fileLength); + attach.setEncodingType(TskData.EncodingType.XOR1); + email.addAttachment(attach); + } catch (IOException ex) { logger.log(Level.WARNING, "Failed to create file output stream for: " + outPath, ex); //NON-NLS - return; } - - EmailMessage.Attachment attach = new EmailMessage.Attachment(); - attach.setName(filename); - attach.setLocalPath(relModuleOutputPath + uniqueFilename); - attach.setSize(fileLength); - attach.setEncodingType(TskData.EncodingType.XOR1); - email.addAttachment(attach); } - - } /** diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/PstParser.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/PstParser.java index 89ffe3910e..9cbc8c6a6f 100644 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/PstParser.java +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/PstParser.java @@ -277,16 +277,30 @@ class PstParser implements AutoCloseable{ */ private EmailMessage extractEmailMessage(PSTMessage msg, String localPath, long fileID) { EmailMessage email = new EmailMessage(); - email.setRecipients(msg.getDisplayTo()); - email.setCc(msg.getDisplayCC()); - email.setBcc(msg.getDisplayBCC()); - email.setSender(getSender(msg.getSenderName(), msg.getSenderEmailAddress())); + String toAddress = msg.getDisplayTo(); + String ccAddress = msg.getDisplayCC(); + String bccAddress = msg.getDisplayBCC(); + String receivedByName = msg.getReceivedByName(); + String receivedBySMTPAddress = msg.getReceivedBySMTPAddress(); + + if (toAddress.contains(receivedByName)) { + toAddress = toAddress.replace(receivedByName, receivedBySMTPAddress); + } + if (ccAddress.contains(receivedByName)) { + ccAddress = ccAddress.replace(receivedByName, receivedBySMTPAddress); + } + if (bccAddress.contains(receivedByName)) { + bccAddress = bccAddress.replace(receivedByName, receivedBySMTPAddress); + } + email.setRecipients(toAddress); + email.setCc(ccAddress); + email.setBcc(bccAddress); + email.setSender(getSender(msg.getSenderName(), msg.getSentRepresentingSMTPAddress())); email.setSentDate(msg.getMessageDeliveryTime()); email.setTextBody(msg.getBody()); if (false == msg.getTransportMessageHeaders().isEmpty()) { email.setHeaders("\n-----HEADERS-----\n\n" + msg.getTransportMessageHeaders() + "\n\n---END HEADERS--\n\n"); } - email.setHtmlBody(msg.getBodyHTML()); String rtf = ""; try { diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java index f97bde95fb..1c165ef535 100644 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java @@ -48,6 +48,7 @@ import org.sleuthkit.autopsy.ingest.IngestModule.ProcessResult; import org.sleuthkit.autopsy.ingest.IngestMonitor; import org.sleuthkit.autopsy.ingest.IngestServices; import org.sleuthkit.autopsy.ingest.ModuleContentEvent; +import org.sleuthkit.autopsy.thunderbirdparser.EmailMessage.AttachedEmailMessage; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.Account; import org.sleuthkit.datamodel.AccountFileInstance; @@ -72,13 +73,14 @@ import org.sleuthkit.datamodel.blackboardutils.attributes.MessageAttachments.Fil * structure and metadata. */ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { + private static final Logger logger = Logger.getLogger(ThunderbirdMboxFileIngestModule.class.getName()); private final IngestServices services = IngestServices.getInstance(); private FileManager fileManager; private IngestJobContext context; private Blackboard blackboard; private CommunicationArtifactsHelper communicationArtifactsHelper; - + private static final int MBOX_SIZE_TO_SPLIT = 1048576000; private Case currentCase; @@ -89,7 +91,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { } @Override - @Messages ({"ThunderbirdMboxFileIngestModule.noOpenCase.errMsg=Exception while getting open case."}) + @Messages({"ThunderbirdMboxFileIngestModule.noOpenCase.errMsg=Exception while getting open case."}) public void startUp(IngestJobContext context) throws IngestModuleException { this.context = context; try { @@ -112,8 +114,8 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { } //skip unalloc - if ((abstractFile.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS)) || - (abstractFile.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.SLACK))) { + if ((abstractFile.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS)) + || (abstractFile.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.SLACK))) { return ProcessResult.OK; } @@ -124,7 +126,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { // check its signature boolean isMbox = false; boolean isEMLFile = false; - + try { byte[] t = new byte[64]; if (abstractFile.getSize() > 64) { @@ -137,15 +139,15 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { } catch (TskException ex) { logger.log(Level.WARNING, null, ex); } - + boolean isPstFile = PstParser.isPstFile(abstractFile); boolean isVcardFile = VcardParser.isVcardFile(abstractFile); - + if (context.fileIngestIsCancelled()) { return ProcessResult.OK; } - - if (isMbox || isEMLFile || isPstFile || isVcardFile ) { + + if (isMbox || isEMLFile || isPstFile || isVcardFile) { try { communicationArtifactsHelper = new CommunicationArtifactsHelper(currentCase.getSleuthkitCase(), EmailParserModuleFactory.getModuleName(), abstractFile, Account.Type.EMAIL); @@ -158,7 +160,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { if (isMbox) { return processMBox(abstractFile); } - + if (isEMLFile) { return processEMLFile(abstractFile); } @@ -166,11 +168,11 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { if (isPstFile) { return processPst(abstractFile); } - + if (isVcardFile) { return processVcard(abstractFile); } - + return ProcessResult.OK; } @@ -186,7 +188,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { String fileName; try { fileName = getTempPath() + File.separator + abstractFile.getName() - + "-" + String.valueOf(abstractFile.getId()); + + "-" + String.valueOf(abstractFile.getId()); } catch (NoCurrentCaseException ex) { logger.log(Level.SEVERE, "Exception while getting open case.", ex); //NON-NLS return ProcessResult.ERROR; @@ -203,8 +205,8 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { services.postMessage(msg); return ProcessResult.OK; } - - try (PstParser parser = new PstParser(services)){ + + try (PstParser parser = new PstParser(services)) { try { ContentUtils.writeToFile(abstractFile, file, context::fileIngestIsCancelled); } catch (IOException ex) { @@ -214,7 +216,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { PstParser.ParseResult result = parser.open(file, abstractFile.getId()); - switch( result) { + switch (result) { case OK: Iterator pstMsgIterator = parser.getEmailMessageIterator(); if (pstMsgIterator != null) { @@ -238,20 +240,20 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { // encrypted pst: Add encrypted file artifact try { - BlackboardArtifact artifact = abstractFile.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_ENCRYPTION_DETECTED); - artifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_NAME, EmailParserModuleFactory.getModuleName(), NbBundle.getMessage(this.getClass(), "ThunderbirdMboxFileIngestModule.encryptionFileLevel"))); + BlackboardArtifact artifact = abstractFile.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_ENCRYPTION_DETECTED); + artifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_NAME, EmailParserModuleFactory.getModuleName(), NbBundle.getMessage(this.getClass(), "ThunderbirdMboxFileIngestModule.encryptionFileLevel"))); - try { - // index the artifact for keyword search - blackboard.postArtifact(artifact, EmailParserModuleFactory.getModuleName()); - } catch (Blackboard.BlackboardException ex) { - MessageNotifyUtil.Notify.error(Bundle.ThunderbirdMboxFileIngestModule_processPst_indexError_message(), artifact.getDisplayName()); - logger.log(Level.SEVERE, "Unable to index blackboard artifact " + artifact.getArtifactID(), ex); //NON-NLS - } - } catch (TskCoreException ex) { - logger.log(Level.INFO, "Failed to add encryption attribute to file: {0}", abstractFile.getName()); //NON-NLS + try { + // index the artifact for keyword search + blackboard.postArtifact(artifact, EmailParserModuleFactory.getModuleName()); + } catch (Blackboard.BlackboardException ex) { + MessageNotifyUtil.Notify.error(Bundle.ThunderbirdMboxFileIngestModule_processPst_indexError_message(), artifact.getDisplayName()); + logger.log(Level.SEVERE, "Unable to index blackboard artifact " + artifact.getArtifactID(), ex); //NON-NLS } - break; + } catch (TskCoreException ex) { + logger.log(Level.INFO, "Failed to add encryption attribute to file: {0}", abstractFile.getName()); //NON-NLS + } + break; default: // parsing error: log message postErrorMessage( @@ -262,8 +264,8 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { logger.log(Level.INFO, "PSTParser failed to parse {0}", abstractFile.getName()); //NON-NLS return ProcessResult.ERROR; } - } catch(Exception ex) { - logger.log(Level.WARNING, String.format("Failed to close temp pst file %s", file.getAbsolutePath())); + } catch (Exception ex) { + logger.log(Level.WARNING, String.format("Failed to close temp pst file %s", file.getAbsolutePath())); } finally { file.delete(); } @@ -294,7 +296,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { String fileName; try { fileName = getTempPath() + File.separator + abstractFile.getName() - + "-" + String.valueOf(abstractFile.getId()); + + "-" + String.valueOf(abstractFile.getId()); } catch (NoCurrentCaseException ex) { logger.log(Level.SEVERE, "Exception while getting open case.", ex); //NON-NLS return ProcessResult.ERROR; @@ -313,7 +315,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { } if (abstractFile.getSize() < MBOX_SIZE_TO_SPLIT) { - + try { ContentUtils.writeToFile(abstractFile, file, context::fileIngestIsCancelled); } catch (IOException ex) { @@ -321,25 +323,25 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { return ProcessResult.OK; } - try{ + try { processMboxFile(file, abstractFile, emailFolder); if (context.fileIngestIsCancelled()) { return ProcessResult.OK; } - }finally { + } finally { file.delete(); } } else { - List mboxSplitOffsets = new ArrayList<>(); - try{ + List mboxSplitOffsets = new ArrayList<>(); + try { mboxSplitOffsets = findMboxSplitOffset(abstractFile, file); } catch (IOException ex) { logger.log(Level.WARNING, String.format("Failed finding split offsets for mbox file {0}.", fileName), ex); //NON-NLS return ProcessResult.OK; } - long startingOffset = 0; + long startingOffset = 0; for (Long mboxSplitOffset : mboxSplitOffsets) { File splitFile = new File(fileName + "-" + mboxSplitOffset); try { @@ -348,55 +350,54 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { logger.log(Level.WARNING, "Failed writing split mbox file to disk.", ex); //NON-NLS return ProcessResult.OK; } - try{ + try { processMboxFile(splitFile, abstractFile, emailFolder); - startingOffset = mboxSplitOffset; + startingOffset = mboxSplitOffset; } finally { splitFile.delete(); } - + if (context.fileIngestIsCancelled()) { return ProcessResult.OK; } } - } - + } + return ProcessResult.OK; } - + private List findMboxSplitOffset(AbstractFile abstractFile, File file) throws IOException { - + List mboxSplitOffset = new ArrayList<>(); - + byte[] buffer = new byte[7]; ReadContentInputStream in = new ReadContentInputStream(abstractFile); - in.skip(MBOX_SIZE_TO_SPLIT); + in.skip(MBOX_SIZE_TO_SPLIT); int len = in.read(buffer); while (len != -1) { len = in.read(buffer); - if (buffer[0] == 13 && buffer[1] == 10 && buffer[2] == 70 && buffer[3] == 114 && - buffer[4] == 111 && buffer[5] == 109 && buffer[6] == 32) { - mboxSplitOffset.add(in.getCurPosition() - 5 ); - in.skip(MBOX_SIZE_TO_SPLIT); + if (buffer[0] == 13 && buffer[1] == 10 && buffer[2] == 70 && buffer[3] == 114 + && buffer[4] == 111 && buffer[5] == 109 && buffer[6] == 32) { + mboxSplitOffset.add(in.getCurPosition() - 5); + in.skip(MBOX_SIZE_TO_SPLIT); } } - + return mboxSplitOffset; - + } - - + private void processMboxFile(File file, AbstractFile abstractFile, String emailFolder) { - - try(MboxParser emailIterator = MboxParser.getEmailIterator( emailFolder, file, abstractFile.getId())) { + + try (MboxParser emailIterator = MboxParser.getEmailIterator(emailFolder, file, abstractFile.getId())) { List emails = new ArrayList<>(); - if(emailIterator != null) { - while(emailIterator.hasNext()) { + if (emailIterator != null) { + while (emailIterator.hasNext()) { if (context.fileIngestIsCancelled()) { return; } EmailMessage emailMessage = emailIterator.next(); - if(emailMessage != null) { + if (emailMessage != null) { emails.add(emailMessage); } } @@ -408,13 +409,13 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { abstractFile.getName()), errors); } } - processEmails(emails, MboxParser.getEmailIterator( emailFolder, file, abstractFile.getId()), abstractFile); - } catch(Exception ex) { + processEmails(emails, MboxParser.getEmailIterator(emailFolder, file, abstractFile.getId()), abstractFile); + } catch (Exception ex) { logger.log(Level.WARNING, String.format("Failed to close mbox temp file %s", file.getAbsolutePath())); } } - + /** * Parse and extract data from a vCard file. * @@ -438,8 +439,8 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { } return ProcessResult.OK; } - - private ProcessResult processEMLFile(AbstractFile abstractFile) { + + private ProcessResult processEMLFile(AbstractFile abstractFile) { try { EmailMessage message = EMLParser.parse(abstractFile); @@ -450,13 +451,9 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { List derivedFiles = new ArrayList<>(); AccountFileInstanceCache accountFileInstanceCache = new AccountFileInstanceCache(abstractFile, currentCase); - BlackboardArtifact msgArtifact = addEmailArtifact(message, abstractFile, accountFileInstanceCache); + createEmailArtifact(message, abstractFile, accountFileInstanceCache, derivedFiles); accountFileInstanceCache.clear(); - if ((msgArtifact != null) && (message.hasAttachment())) { - derivedFiles.addAll(handleAttachments(message.getAttachments(), abstractFile, msgArtifact)); - } - if (derivedFiles.isEmpty() == false) { for (AbstractFile derived : derivedFiles) { services.fireModuleContentEvent(new ModuleContentEvent(derived)); @@ -493,7 +490,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { /** * Get a module output folder. - * + * * @throws NoCurrentCaseException if there is no open case. * * @return the module output folder @@ -527,48 +524,43 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { * @param fullMessageIterator * @param abstractFile */ - private void processEmails(List partialEmailsForThreading, Iterator fullMessageIterator, + private void processEmails(List partialEmailsForThreading, Iterator fullMessageIterator, AbstractFile abstractFile) { - + // Create cache for accounts AccountFileInstanceCache accountFileInstanceCache = new AccountFileInstanceCache(abstractFile, currentCase); - + // Putting try/catch around this to catch any exception and still allow // the creation of the artifacts to continue. - try{ + try { EmailMessageThreader.threadMessages(partialEmailsForThreading); - } catch(Exception ex) { + } catch (Exception ex) { logger.log(Level.WARNING, String.format("Exception thrown parsing emails from %s", abstractFile.getName()), ex); } - + List derivedFiles = new ArrayList<>(); int msgCnt = 0; - while(fullMessageIterator.hasNext()) { + while (fullMessageIterator.hasNext()) { if (context.fileIngestIsCancelled()) { return; } - + EmailMessage current = fullMessageIterator.next(); - - if(current == null) { + + if (current == null) { continue; } - if(partialEmailsForThreading.size() > msgCnt) { + if (partialEmailsForThreading.size() > msgCnt) { EmailMessage threaded = partialEmailsForThreading.get(msgCnt++); - - if(threaded.getMessageID().equals(current.getMessageID()) && - threaded.getSubject().equals(current.getSubject())) { + + if (threaded.getMessageID().equals(current.getMessageID()) + && threaded.getSubject().equals(current.getSubject())) { current.setMessageThreadID(threaded.getMessageThreadID()); } } - - BlackboardArtifact msgArtifact = addEmailArtifact(current, abstractFile, accountFileInstanceCache); - - if ((msgArtifact != null) && (current.hasAttachment())) { - derivedFiles.addAll(handleAttachments(current.getAttachments(), abstractFile, msgArtifact )); - } + createEmailArtifact(current, abstractFile, accountFileInstanceCache, derivedFiles); } if (derivedFiles.isEmpty() == false) { @@ -581,6 +573,21 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { } context.addFilesToJob(derivedFiles); } + + void createEmailArtifact(EmailMessage email, AbstractFile abstractFile, AccountFileInstanceCache accountFileInstanceCache, List derivedFiles) { + BlackboardArtifact msgArtifact = addEmailArtifact(email, abstractFile, accountFileInstanceCache); + + if ((msgArtifact != null) && (email.hasAttachment())) { + derivedFiles.addAll(handleAttachments(email.getAttachments(), abstractFile, msgArtifact)); + + for (EmailMessage.Attachment attach : email.getAttachments()) { + if (attach instanceof AttachedEmailMessage) { + createEmailArtifact(((AttachedEmailMessage) attach).getEmailMessage(), abstractFile, accountFileInstanceCache, derivedFiles); + } + } + } + } + /** * Add the given attachments as derived files and reschedule them for * ingest. @@ -592,8 +599,8 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { * @return List of attachments */ @NbBundle.Messages({ - "ThunderbirdMboxFileIngestModule.handleAttch.addAttachmentsErrorMsg=Failed to add attachments to email message." -}) + "ThunderbirdMboxFileIngestModule.handleAttch.addAttachmentsErrorMsg=Failed to add attachments to email message." + }) private List handleAttachments(List attachments, AbstractFile abstractFile, BlackboardArtifact messageArtifact) { List files = new ArrayList<>(); List fileAttachments = new ArrayList<>(); @@ -611,11 +618,11 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { DerivedFile df = fileManager.addDerivedFile(filename, relPath, size, cTime, crTime, aTime, mTime, true, abstractFile, "", EmailParserModuleFactory.getModuleName(), EmailParserModuleFactory.getModuleVersion(), "", encodingType); - + associateAttachmentWithMesssge(messageArtifact, df); - + files.add(df); - + fileAttachments.add(new FileAttachment(df)); } catch (TskCoreException ex) { postErrorMessage( @@ -626,17 +633,16 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { logger.log(Level.INFO, "", ex); } } - - + try { communicationArtifactsHelper.addAttachments(messageArtifact, new MessageAttachments(fileAttachments, Collections.emptyList())); } catch (TskCoreException ex) { - postErrorMessage( - NbBundle.getMessage(this.getClass(), "ThunderbirdMboxFileIngestModule.handleAttch.addAttachmentsErrorMsg"), - ""); - logger.log(Level.INFO, "Failed to add attachments to email message.", ex); + postErrorMessage( + NbBundle.getMessage(this.getClass(), "ThunderbirdMboxFileIngestModule.handleAttch.addAttachmentsErrorMsg"), + ""); + logger.log(Level.INFO, "Failed to add attachments to email message.", ex); } - + return files; } @@ -652,32 +658,33 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { bba.addAttributes(attributes); //write out to bb return bba; } - + /** - * Finds and returns a set of unique email addresses found in the input string + * Finds and returns a set of unique email addresses found in the input + * string * * @param input - input string, like the To/CC line from an email header - * + * * @return Set: set of email addresses found in the input string */ private Set findEmailAddresess(String input) { Pattern p = Pattern.compile("\\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\\.[A-Z]{2,4}\\b", - Pattern.CASE_INSENSITIVE); + Pattern.CASE_INSENSITIVE); Matcher m = p.matcher(input); Set emailAddresses = new HashSet<>(); while (m.find()) { - emailAddresses.add( m.group()); + emailAddresses.add(m.group()); } return emailAddresses; } - + /** * Add a blackboard artifact for the given e-mail message. * - * @param email The e-mail message. - * @param abstractFile The associated file. - * @param accountFileInstanceCache The current cache of account instances. - * + * @param email The e-mail message. + * @param abstractFile The associated file. + * @param accountFileInstanceCache The current cache of account instances. + * * @return The generated e-mail message artifact. */ @Messages({"ThunderbirdMboxFileIngestModule.addArtifact.indexError.message=Failed to index email message detected artifact for keyword search."}) @@ -701,35 +708,33 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { List senderAddressList = new ArrayList<>(); String senderAddress; senderAddressList.addAll(findEmailAddresess(from)); - + if (context.fileIngestIsCancelled()) { return null; } - + AccountFileInstance senderAccountInstance = null; if (senderAddressList.size() == 1) { senderAddress = senderAddressList.get(0); try { senderAccountInstance = accountFileInstanceCache.getAccountInstance(senderAddress); + } catch (TskCoreException ex) { + logger.log(Level.WARNING, "Failed to create account for email address " + senderAddress, ex); //NON-NLS } - catch(TskCoreException ex) { - logger.log(Level.WARNING, "Failed to create account for email address " + senderAddress, ex); //NON-NLS - } + } else { + logger.log(Level.WARNING, "Failed to find sender address, from = {0}", from); //NON-NLS } - else { - logger.log(Level.WARNING, "Failed to find sender address, from = {0}", from); //NON-NLS - } - + if (context.fileIngestIsCancelled()) { return null; } - + List recipientAddresses = new ArrayList<>(); recipientAddresses.addAll(findEmailAddresess(to)); recipientAddresses.addAll(findEmailAddresess(cc)); recipientAddresses.addAll(findEmailAddresess(bcc)); - + List recipientAccountInstances = new ArrayList<>(); for (String addr : recipientAddresses) { if (context.fileIngestIsCancelled()) { @@ -738,56 +743,54 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { try { AccountFileInstance recipientAccountInstance = accountFileInstanceCache.getAccountInstance(addr); recipientAccountInstances.add(recipientAccountInstance); - } - catch(TskCoreException ex) { + } catch (TskCoreException ex) { logger.log(Level.WARNING, "Failed to create account for email address " + addr, ex); //NON-NLS } } - + addArtifactAttribute(headers, ATTRIBUTE_TYPE.TSK_HEADERS, bbattributes); addArtifactAttribute(from, ATTRIBUTE_TYPE.TSK_EMAIL_FROM, bbattributes); addArtifactAttribute(to, ATTRIBUTE_TYPE.TSK_EMAIL_TO, bbattributes); addArtifactAttribute(subject, ATTRIBUTE_TYPE.TSK_SUBJECT, bbattributes); - + addArtifactAttribute(dateL, ATTRIBUTE_TYPE.TSK_DATETIME_RCVD, bbattributes); addArtifactAttribute(dateL, ATTRIBUTE_TYPE.TSK_DATETIME_SENT, bbattributes); - + addArtifactAttribute(body, ATTRIBUTE_TYPE.TSK_EMAIL_CONTENT_PLAIN, bbattributes); - - addArtifactAttribute(((id < 0L) ? NbBundle.getMessage(this.getClass(), "ThunderbirdMboxFileIngestModule.notAvail") : String.valueOf(id)), + + addArtifactAttribute(((id < 0L) ? NbBundle.getMessage(this.getClass(), "ThunderbirdMboxFileIngestModule.notAvail") : String.valueOf(id)), ATTRIBUTE_TYPE.TSK_MSG_ID, bbattributes); - - addArtifactAttribute(((localPath.isEmpty() == false) ? localPath : ""), + + addArtifactAttribute(((localPath.isEmpty() == false) ? localPath : ""), ATTRIBUTE_TYPE.TSK_PATH, bbattributes); - + addArtifactAttribute(cc, ATTRIBUTE_TYPE.TSK_EMAIL_CC, bbattributes); addArtifactAttribute(bodyHTML, ATTRIBUTE_TYPE.TSK_EMAIL_CONTENT_HTML, bbattributes); addArtifactAttribute(rtf, ATTRIBUTE_TYPE.TSK_EMAIL_CONTENT_RTF, bbattributes); addArtifactAttribute(threadID, ATTRIBUTE_TYPE.TSK_THREAD_ID, bbattributes); - - + try { if (context.fileIngestIsCancelled()) { return null; } - + bbart = abstractFile.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_EMAIL_MSG); bbart.addAttributes(bbattributes); if (context.fileIngestIsCancelled()) { return null; } - + // Add account relationships - currentCase.getSleuthkitCase().getCommunicationsManager().addRelationships(senderAccountInstance, recipientAccountInstances, bbart,Relationship.Type.MESSAGE, dateL); + currentCase.getSleuthkitCase().getCommunicationsManager().addRelationships(senderAccountInstance, recipientAccountInstances, bbart, Relationship.Type.MESSAGE, dateL); if (context.fileIngestIsCancelled()) { return null; } - + try { // index the artifact for keyword search - blackboard.postArtifact(bbart, EmailParserModuleFactory.getModuleName()); + blackboard.postArtifact(bbart, EmailParserModuleFactory.getModuleName()); } catch (Blackboard.BlackboardException ex) { logger.log(Level.SEVERE, "Unable to index blackboard artifact " + bbart.getArtifactID(), ex); //NON-NLS MessageNotifyUtil.Notify.error(Bundle.ThunderbirdMboxFileIngestModule_addArtifact_indexError_message(), bbart.getDisplayName()); @@ -798,11 +801,11 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { return bbart; } - + /** * Add an attribute of a specified type to a supplied Collection. - * - * @param stringVal The attribute value. + * + * @param stringVal The attribute value. * @param attrType The type of attribute to be added. * @param bbattributes The Collection to which the attribute will be added. */ @@ -814,7 +817,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { /** * Add an attribute of a specified type to a supplied Collection. - * + * * @param stringVal The attribute value. * @param attrType The type of attribute to be added. * @param bbattributes The Collection to which the attribute will be added. @@ -824,10 +827,10 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { bbattributes.add(new BlackboardAttribute(attrType, EmailParserModuleFactory.getModuleName(), stringVal)); } } - + /** * Add an attribute of a specified type to a supplied Collection. - * + * * @param longVal The attribute value. * @param attrType The type of attribute to be added. * @param bbattributes The Collection to which the attribute will be added. @@ -837,49 +840,51 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { bbattributes.add(new BlackboardAttribute(attrType, EmailParserModuleFactory.getModuleName(), longVal)); } } - + /** - * Cache for storing AccountFileInstance. - * The idea is that emails will be used multiple times in a file and - * we shouldn't do a database lookup each time. + * Cache for storing AccountFileInstance. The idea is that emails will be + * used multiple times in a file and we shouldn't do a database lookup each + * time. */ static private class AccountFileInstanceCache { + private final Map cacheMap; private final AbstractFile file; private final Case currentCase; - + /** * Create a new cache. Caches are linked to a specific file. + * * @param file - * @param currentCase + * @param currentCase */ AccountFileInstanceCache(AbstractFile file, Case currentCase) { - cacheMap= new HashMap<>(); + cacheMap = new HashMap<>(); this.file = file; this.currentCase = currentCase; } - + /** * Get the account file instance from the cache or the database. - * + * * @param email The email for this account. - * + * * @return The corresponding AccountFileInstance - * - * @throws TskCoreException + * + * @throws TskCoreException */ AccountFileInstance getAccountInstance(String email) throws TskCoreException { if (cacheMap.containsKey(email)) { return cacheMap.get(email); } - - AccountFileInstance accountInstance = - currentCase.getSleuthkitCase().getCommunicationsManager().createAccountFileInstance(Account.Type.EMAIL, email, - EmailParserModuleFactory.getModuleName(), file); + + AccountFileInstance accountInstance + = currentCase.getSleuthkitCase().getCommunicationsManager().createAccountFileInstance(Account.Type.EMAIL, email, + EmailParserModuleFactory.getModuleName(), file); cacheMap.put(email, accountInstance); return accountInstance; } - + /** * Clears the cache. */ @@ -887,10 +892,10 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { cacheMap.clear(); } } - + /** * Post an error message for the user. - * + * * @param subj The error subject. * @param details The error details. */ @@ -901,7 +906,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { /** * Get the IngestServices object. - * + * * @return The IngestServices object. */ IngestServices getServices() { @@ -912,5 +917,5 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { public void shutDown() { // nothing to shut down } - + } diff --git a/unix_setup.sh b/unix_setup.sh index 93a5eb4bea..40b1d858b4 100644 --- a/unix_setup.sh +++ b/unix_setup.sh @@ -76,6 +76,9 @@ fi chmod u+x autopsy/markmckinnon/Export* chmod u+x autopsy/markmckinnon/parse* +# allow solr dependencies to execute +chmod -R u+x autopsy/solr/bin + # make sure it is executable chmod u+x bin/autopsy