From 12cf7c0f4d51d048ad2f0d34186fdd628e92fcd3 Mon Sep 17 00:00:00 2001 From: esaunders Date: Tue, 10 Dec 2019 14:27:01 -0500 Subject: [PATCH 01/22] Extract strings from carved text files. --- .../keywordsearch/KeywordSearchIngestModule.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java index d4c9228c69..af7ce2e771 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java @@ -578,8 +578,16 @@ public final class KeywordSearchIngestModule implements FileIngestModule { TskData.TSK_DB_FILES_TYPE_ENUM aType = aFile.getType(); - // unallocated and unused blocks can only have strings extracted from them. - if ((aType.equals(TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS) || aType.equals(TskData.TSK_DB_FILES_TYPE_ENUM.UNUSED_BLOCKS))) { + /** + * Extract unicode strings from unallocated and unused blocks and + * carved text files. The reason for performing string extraction + * on these is because they all may contain multiple encodings which + * can cause text to be missed by the more specialized text extractors + * used below. + */ + if ((aType.equals(TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS) + || aType.equals(TskData.TSK_DB_FILES_TYPE_ENUM.UNUSED_BLOCKS)) + || (aType.equals(TskData.TSK_DB_FILES_TYPE_ENUM.CARVED) && aFile.getNameExtension().equalsIgnoreCase("txt"))) { if (context.fileIngestIsCancelled()) { return; } From c0cdc38e71d372d2893d15bbca9da083997a3000 Mon Sep 17 00:00:00 2001 From: Raman Arora Date: Wed, 11 Dec 2019 12:54:59 -0500 Subject: [PATCH 02/22] 5709: Email attachments - Use CommunicationsArtifactsHelper to add the attachments to email messages. - Updated the DirectoryTree & DataResultsTable to not show message artifacts for 'new' cases, but continue to show them for older cases. --- Core/nbproject/project.xml | 1 + .../directorytree/DataResultFilterNode.java | 23 +++---- .../DirectoryTreeFilterNode.java | 23 +++---- .../directorytree/DirectoryTreeUtils.java | 60 +++++++++++++++++++ .../Bundle.properties-MERGED | 1 + .../ThunderbirdMboxFileIngestModule.java | 35 ++++++++++- 6 files changed, 120 insertions(+), 23 deletions(-) create mode 100644 Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeUtils.java diff --git a/Core/nbproject/project.xml b/Core/nbproject/project.xml index 83aefea7c5..17e5d82284 100644 --- a/Core/nbproject/project.xml +++ b/Core/nbproject/project.xml @@ -345,6 +345,7 @@ org.sleuthkit.autopsy.textextractors.configs org.sleuthkit.autopsy.texttranslation org.sleuthkit.datamodel + org.sleuthkit.datamodel.blackboardutils ext/commons-lang3-3.8.1.jar diff --git a/Core/src/org/sleuthkit/autopsy/directorytree/DataResultFilterNode.java b/Core/src/org/sleuthkit/autopsy/directorytree/DataResultFilterNode.java index 548fb881be..262ea986a7 100644 --- a/Core/src/org/sleuthkit/autopsy/directorytree/DataResultFilterNode.java +++ b/Core/src/org/sleuthkit/autopsy/directorytree/DataResultFilterNode.java @@ -41,7 +41,6 @@ import org.sleuthkit.autopsy.actions.DeleteFileBlackboardArtifactTagAction; import org.sleuthkit.autopsy.actions.DeleteFileContentTagAction; import org.sleuthkit.autopsy.coreutils.ContextMenuExtensionPoint; import org.sleuthkit.autopsy.coreutils.Logger; -import org.sleuthkit.autopsy.datamodel.AbstractAbstractFileNode.AbstractFilePropertyType; import org.sleuthkit.autopsy.datamodel.AbstractFsContentNode; import org.sleuthkit.autopsy.datamodel.BlackboardArtifactNode; import org.sleuthkit.autopsy.datamodel.DataModelActionsFactory; @@ -258,16 +257,20 @@ public class DataResultFilterNode extends FilterNode { @Override protected Node[] createNodes(Node key) { - // filter out all non-message artifacts, if displaying the results from the Data Source tree + // if displaying the results from the Data Source tree + // filter out artifacts + // unless there are message artifacts with attachments as children BlackboardArtifact art = key.getLookup().lookup(BlackboardArtifact.class); - if (art != null - && filterArtifacts - && art.getArtifactTypeID() != BlackboardArtifact.ARTIFACT_TYPE.TSK_EMAIL_MSG.getTypeID() - && art.getArtifactTypeID() != BlackboardArtifact.ARTIFACT_TYPE.TSK_MESSAGE.getTypeID()) { - return new Node[]{}; - } - - return new Node[]{new DataResultFilterNode(key, sourceEm)}; + if (art != null && filterArtifacts) { + + if ((DirectoryTreeUtils.showMessagesInDirTree() == false) || + (DirectoryTreeUtils.showMessagesInDirTree() && + art.getArtifactTypeID() != BlackboardArtifact.ARTIFACT_TYPE.TSK_EMAIL_MSG.getTypeID() && + art.getArtifactTypeID() != BlackboardArtifact.ARTIFACT_TYPE.TSK_MESSAGE.getTypeID()) ) { + return new Node[]{}; + } + } + return new Node[]{new DataResultFilterNode(key, sourceEm)}; } } diff --git a/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeFilterNode.java b/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeFilterNode.java index c9070015f2..c5d5712ec8 100644 --- a/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeFilterNode.java +++ b/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeFilterNode.java @@ -20,7 +20,6 @@ package org.sleuthkit.autopsy.directorytree; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.List; import java.util.logging.Level; import javax.swing.Action; @@ -33,17 +32,12 @@ import org.sleuthkit.autopsy.core.UserPreferences; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.datamodel.AbstractContentNode; import org.sleuthkit.autopsy.datamodel.BlackboardArtifactNode; -import org.sleuthkit.autopsy.ingest.runIngestModuleWizard.RunIngestModulesAction; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE; import org.sleuthkit.datamodel.Content; -import org.sleuthkit.datamodel.Directory; -import org.sleuthkit.datamodel.Image; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskData; -import org.sleuthkit.datamodel.VirtualDirectory; -import org.sleuthkit.datamodel.Volume; /** * A node filter (decorator) that sets the actions for a node in the tree view @@ -137,11 +131,18 @@ class DirectoryTreeFilterNode extends FilterNode { numVisibleChildren--; } } else if (child instanceof BlackboardArtifact) { - BlackboardArtifact bba = (BlackboardArtifact) child; - - // Only message type artifacts are displayed in the tree - if ((bba.getArtifactTypeID() != ARTIFACT_TYPE.TSK_EMAIL_MSG.getTypeID()) - && (bba.getArtifactTypeID() != ARTIFACT_TYPE.TSK_MESSAGE.getTypeID())) { + + if (DirectoryTreeUtils.showMessagesInDirTree()) { + // In older versions of Autopsy, attachments were children of email/message artifacts + // and hence email/messages with attachments are shown in the directory tree. + BlackboardArtifact bba = (BlackboardArtifact) child; + // Only message type artifacts are displayed in the tree + if ((bba.getArtifactTypeID() != ARTIFACT_TYPE.TSK_EMAIL_MSG.getTypeID()) + && (bba.getArtifactTypeID() != ARTIFACT_TYPE.TSK_MESSAGE.getTypeID())) { + numVisibleChildren--; + } + } + else { numVisibleChildren--; } } diff --git a/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeUtils.java b/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeUtils.java new file mode 100644 index 0000000000..bf84561f21 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeUtils.java @@ -0,0 +1,60 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2019 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.sleuthkit.autopsy.directorytree; + +import org.sleuthkit.autopsy.casemodule.Case; +import org.sleuthkit.datamodel.CaseDbSchemaVersionNumber; + +/** + * Utility class for Directory tree. + * + */ +final class DirectoryTreeUtils { + + private static final int ATTACHMENT_CHILDOF_MSG_MAX_DB_MAJOR_VER = 8; + private static final int ATTACHMENT_CHILDOF_MSG_MAX_DB_MINOR_VER = 4; + + + /** + * Prior to schema version 8.4, attachments were children of messages and + * hence messages with any attachment children are shown in the directory + * tree. + * + * At 8.4, attachments are tracked as an attribute, and the message artifact + * don't need to be shown in the directory tree. + * + * This method may be used to check the schema version and behave + * accordingly, in order to maintain backward compatibility. + * + * @return True if messages with attachment children should be shown in + * directory tree. + */ + static boolean showMessagesInDirTree() { + boolean showMessagesInDirTree = true; + if (Case.isCaseOpen()) { + CaseDbSchemaVersionNumber version = Case.getCurrentCase().getSleuthkitCase().getDBSchemaCreationVersion(); + showMessagesInDirTree + = ((version.getMajor() < ATTACHMENT_CHILDOF_MSG_MAX_DB_MAJOR_VER) + || (version.getMajor() == ATTACHMENT_CHILDOF_MSG_MAX_DB_MAJOR_VER && version.getMinor() < ATTACHMENT_CHILDOF_MSG_MAX_DB_MINOR_VER)); + } + return showMessagesInDirTree; + } + +} diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/Bundle.properties-MERGED b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/Bundle.properties-MERGED index cdfd241886..90ce00170c 100755 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/Bundle.properties-MERGED +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/Bundle.properties-MERGED @@ -15,6 +15,7 @@ ThunderbirdMboxFileIngestModule.addArtifact.indexError.message=Failed to index e # {0} - file name # {1} - file ID ThunderbirdMboxFileIngestModule.errorMessage.outOfDiskSpace=Out of disk space. Cannot copy '{0}' (id={1}) to parse. +ThunderbirdMboxFileIngestModule.handleAttch.addAttachmentsErrorMsg=Failed to add attachments to email message. ThunderbirdMboxFileIngestModule.moduleName=Email Parser ThunderbirdMboxFileIngestModule.noOpenCase.errMsg=Exception while getting open case. ThunderbirdMboxFileIngestModule.processPst.errMsg.outOfDiskSpace=Out of disk space. Cannot copy {0} to parse. diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java index f1260f269f..419173fcff 100644 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java @@ -22,6 +22,7 @@ import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashSet; import java.util.Iterator; import java.util.List; @@ -58,6 +59,9 @@ import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskData; import org.sleuthkit.datamodel.TskDataException; import org.sleuthkit.datamodel.TskException; +import org.sleuthkit.datamodel.blackboardutils.CommunicationArtifactsHelper; +import org.sleuthkit.datamodel.blackboardutils.FileAttachment; +import org.sleuthkit.datamodel.blackboardutils.MessageAttachments; /** * File-level ingest module that detects MBOX, PST, and vCard files based on @@ -70,6 +74,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { private FileManager fileManager; private IngestJobContext context; private Blackboard blackboard; + private CommunicationArtifactsHelper communicationArtifactsHelper; private Case currentCase; @@ -129,6 +134,15 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { logger.log(Level.WARNING, null, ex); } + try { + communicationArtifactsHelper = new CommunicationArtifactsHelper(currentCase.getSleuthkitCase(), + EmailParserModuleFactory.getModuleName(), abstractFile, Account.Type.EMAIL); + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, String.format("Failed to create CommunicationArtifactsHelper for file %s", abstractFile.getName()), ex); + return ProcessResult.ERROR; + } + + if (isMbox) { return processMBox(abstractFile); } @@ -267,7 +281,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { } else if (mboxParentDir.contains("/ImapMail/")) { //NON-NLS emailFolder = mboxParentDir.substring(mboxParentDir.indexOf("/ImapMail/") + 9); //NON-NLS } - emailFolder = emailFolder + mboxFileName; + emailFolder += mboxFileName; emailFolder = emailFolder.replaceAll(".sbd", ""); //NON-NLS String fileName; @@ -487,8 +501,12 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { * * @return List of attachments */ + @NbBundle.Messages({ + "ThunderbirdMboxFileIngestModule.handleAttch.addAttachmentsErrorMsg=Failed to add attachments to email message." +}) private List handleAttachments(List attachments, AbstractFile abstractFile, BlackboardArtifact messageArtifact) { List files = new ArrayList<>(); + List fileAttachments = new ArrayList<>(); for (EmailMessage.Attachment attach : attachments) { String filename = attach.getName(); long crTime = attach.getCrTime(); @@ -501,12 +519,14 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { try { DerivedFile df = fileManager.addDerivedFile(filename, relPath, - size, cTime, crTime, aTime, mTime, true, messageArtifact, "", + size, cTime, crTime, aTime, mTime, true, abstractFile, "", EmailParserModuleFactory.getModuleName(), EmailParserModuleFactory.getModuleVersion(), "", encodingType); associateAttachmentWithMesssge(messageArtifact, df); files.add(df); + + fileAttachments.add(new FileAttachment(df)); } catch (TskCoreException ex) { postErrorMessage( NbBundle.getMessage(this.getClass(), "ThunderbirdMboxFileIngestModule.handleAttch.errMsg", @@ -516,6 +536,17 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { logger.log(Level.INFO, "", ex); } } + + + try { + communicationArtifactsHelper.addAttachments(messageArtifact, new MessageAttachments(fileAttachments, Collections.emptyList())); + } catch (TskCoreException ex) { + postErrorMessage( + NbBundle.getMessage(this.getClass(), "ThunderbirdMboxFileIngestModule.handleAttch.addAttachmentsErrorMsg"), + ""); + logger.log(Level.INFO, "Failed to add attachments to email message.", ex); + } + return files; } From 9dd0d99bbea68c80ee31bb35955057447eda23a2 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Wed, 11 Dec 2019 12:58:29 -0500 Subject: [PATCH 03/22] 5781 only respond to instances table even with correct viewer --- .../autopsy/filequery/ResultsPanel.java | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/filequery/ResultsPanel.java b/Core/src/org/sleuthkit/autopsy/filequery/ResultsPanel.java index 27086c731c..3e4f8b7a68 100644 --- a/Core/src/org/sleuthkit/autopsy/filequery/ResultsPanel.java +++ b/Core/src/org/sleuthkit/autopsy/filequery/ResultsPanel.java @@ -88,17 +88,21 @@ public class ResultsPanel extends javax.swing.JPanel { imageThumbnailViewer = new ImageThumbnailViewer(); videoThumbnailViewer = new VideoThumbnailViewer(); videoThumbnailViewer.addListSelectionListener((e) -> { - if (!e.getValueIsAdjusting()) { - populateInstancesList(); - } else { - instancesList.clearSelection(); + if (resultType == FileSearchData.FileType.VIDEO) { + if (!e.getValueIsAdjusting()) { + populateInstancesList(); + } else { + instancesList.clearSelection(); + } } }); imageThumbnailViewer.addListSelectionListener((e) -> { - if (!e.getValueIsAdjusting()) { - populateInstancesList(); - } else { - instancesList.clearSelection(); + if (resultType == FileSearchData.FileType.IMAGE) { + if (!e.getValueIsAdjusting()) { + populateInstancesList(); + } else { + instancesList.clearSelection(); + } } }); //Add the context menu when right clicking From 858d50765c14b9c1a513bd188b89215fc3cb7efd Mon Sep 17 00:00:00 2001 From: Raman Arora Date: Wed, 11 Dec 2019 13:26:46 -0500 Subject: [PATCH 04/22] Addressed Codacy comments. --- .../directorytree/DataResultFilterNode.java | 15 +++++++-------- .../autopsy/directorytree/DirectoryTreeUtils.java | 8 +++++++- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/directorytree/DataResultFilterNode.java b/Core/src/org/sleuthkit/autopsy/directorytree/DataResultFilterNode.java index 262ea986a7..1eb22316c4 100644 --- a/Core/src/org/sleuthkit/autopsy/directorytree/DataResultFilterNode.java +++ b/Core/src/org/sleuthkit/autopsy/directorytree/DataResultFilterNode.java @@ -261,15 +261,14 @@ public class DataResultFilterNode extends FilterNode { // filter out artifacts // unless there are message artifacts with attachments as children BlackboardArtifact art = key.getLookup().lookup(BlackboardArtifact.class); - if (art != null && filterArtifacts) { + if (art != null && filterArtifacts + && ((DirectoryTreeUtils.showMessagesInDirTree() == false) + || (DirectoryTreeUtils.showMessagesInDirTree() + && art.getArtifactTypeID() != BlackboardArtifact.ARTIFACT_TYPE.TSK_EMAIL_MSG.getTypeID() + && art.getArtifactTypeID() != BlackboardArtifact.ARTIFACT_TYPE.TSK_MESSAGE.getTypeID()))) { + return new Node[]{}; + } - if ((DirectoryTreeUtils.showMessagesInDirTree() == false) || - (DirectoryTreeUtils.showMessagesInDirTree() && - art.getArtifactTypeID() != BlackboardArtifact.ARTIFACT_TYPE.TSK_EMAIL_MSG.getTypeID() && - art.getArtifactTypeID() != BlackboardArtifact.ARTIFACT_TYPE.TSK_MESSAGE.getTypeID()) ) { - return new Node[]{}; - } - } return new Node[]{new DataResultFilterNode(key, sourceEm)}; } } diff --git a/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeUtils.java b/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeUtils.java index bf84561f21..40b3954897 100644 --- a/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeUtils.java +++ b/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeUtils.java @@ -30,7 +30,13 @@ final class DirectoryTreeUtils { private static final int ATTACHMENT_CHILDOF_MSG_MAX_DB_MAJOR_VER = 8; private static final int ATTACHMENT_CHILDOF_MSG_MAX_DB_MINOR_VER = 4; - + + /** + * Empty private constructor + */ + private DirectoryTreeUtils() { + + } /** * Prior to schema version 8.4, attachments were children of messages and From 0b5ab334aa522887b27e0f9864a82d11b8560e85 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Wed, 11 Dec 2019 14:17:41 -0500 Subject: [PATCH 05/22] 5781 fix bug with assigning type to ResultFile objects --- .../filequery/DiscoveryThumbnailChildren.java | 2 +- .../sleuthkit/autopsy/filequery/FileSearch.java | 10 ---------- .../sleuthkit/autopsy/filequery/ResultFile.java | 14 ++++---------- 3 files changed, 5 insertions(+), 21 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/filequery/DiscoveryThumbnailChildren.java b/Core/src/org/sleuthkit/autopsy/filequery/DiscoveryThumbnailChildren.java index 1fad5d82ae..35ee60ac5c 100644 --- a/Core/src/org/sleuthkit/autopsy/filequery/DiscoveryThumbnailChildren.java +++ b/Core/src/org/sleuthkit/autopsy/filequery/DiscoveryThumbnailChildren.java @@ -32,7 +32,7 @@ import org.sleuthkit.autopsy.datamodel.FileNode; import org.sleuthkit.datamodel.AbstractFile; /** - * Create a node containing the children for the to display in the + * Create a node containing the children to display in the * DataResultViewerThumbnail */ class DiscoveryThumbnailChildren extends Children.Keys { diff --git a/Core/src/org/sleuthkit/autopsy/filequery/FileSearch.java b/Core/src/org/sleuthkit/autopsy/filequery/FileSearch.java index 648eb6457a..495b70f4e4 100644 --- a/Core/src/org/sleuthkit/autopsy/filequery/FileSearch.java +++ b/Core/src/org/sleuthkit/autopsy/filequery/FileSearch.java @@ -977,16 +977,6 @@ class FileSearch { GroupKey getGroupKey(ResultFile file) { return new FileTypeGroupKey(file); } - - @Override - void addAttributeToResultFiles(List files, SleuthkitCase caseDb, - EamDb centralRepoDb) throws FileSearchException { - for (ResultFile file : files) { - if (file.getFileType().equals(FileType.OTHER)) { - file.setFileType(FileType.fromMIMEtype(file.getFirstInstance().getMIMEType())); - } - } - } } /** diff --git a/Core/src/org/sleuthkit/autopsy/filequery/ResultFile.java b/Core/src/org/sleuthkit/autopsy/filequery/ResultFile.java index 3b28a5e97d..8b903bf8b9 100644 --- a/Core/src/org/sleuthkit/autopsy/filequery/ResultFile.java +++ b/Core/src/org/sleuthkit/autopsy/filequery/ResultFile.java @@ -72,7 +72,7 @@ class ResultFile { tagNames = new ArrayList<>(); interestingSetNames = new ArrayList<>(); objectDetectedNames = new ArrayList<>(); - fileType = FileType.OTHER; + fileType = FileType.fromMIMEtype(abstractFile.getMIMEType()); } /** @@ -103,6 +103,9 @@ class ResultFile { if (deleted && !duplicate.isDirNameFlagSet(TskData.TSK_FS_NAME_FLAG_ENUM.UNALLOC)) { deleted = false; } + if (fileType == FileType.OTHER) { + fileType = FileType.fromMIMEtype(duplicate.getMIMEType()); + } updateScoreAndDescription(duplicate); instances.add(duplicate); } @@ -156,15 +159,6 @@ class ResultFile { return fileType; } - /** - * Set the file type - * - * @param fileType the type - */ - void setFileType(FileType fileType) { - this.fileType = fileType; - } - /** * Add a keyword list name that matched this file. * From a96e7d6859ec2ba6411ae09f2a7c957d8b25c7d4 Mon Sep 17 00:00:00 2001 From: Raman Arora Date: Wed, 11 Dec 2019 19:21:50 -0500 Subject: [PATCH 06/22] 5709: Email attachments. - address review comments. --- .../autopsy/datamodel/ExtractedContent.java | 1 + .../directorytree/DataResultFilterNode.java | 10 ++++++---- .../directorytree/DirectoryTreeFilterNode.java | 2 +- ...ectoryTreeUtils.java => FilterNodeUtils.java} | 16 ++++++++-------- .../ThunderbirdMboxFileIngestModule.java | 2 +- 5 files changed, 17 insertions(+), 14 deletions(-) rename Core/src/org/sleuthkit/autopsy/directorytree/{DirectoryTreeUtils.java => FilterNodeUtils.java} (83%) diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/ExtractedContent.java b/Core/src/org/sleuthkit/autopsy/datamodel/ExtractedContent.java index d97e980a8b..9de0e95462 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/ExtractedContent.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/ExtractedContent.java @@ -227,6 +227,7 @@ public class ExtractedContent implements AutopsyVisitableItem { // maps the artifact type to its child node private final HashMap typeNodeList = new HashMap<>(); + @SuppressWarnings("deprecation") TypeFactory() { super(); diff --git a/Core/src/org/sleuthkit/autopsy/directorytree/DataResultFilterNode.java b/Core/src/org/sleuthkit/autopsy/directorytree/DataResultFilterNode.java index 1eb22316c4..ea7a3220b6 100644 --- a/Core/src/org/sleuthkit/autopsy/directorytree/DataResultFilterNode.java +++ b/Core/src/org/sleuthkit/autopsy/directorytree/DataResultFilterNode.java @@ -259,17 +259,19 @@ public class DataResultFilterNode extends FilterNode { protected Node[] createNodes(Node key) { // if displaying the results from the Data Source tree // filter out artifacts - // unless there are message artifacts with attachments as children + + // In older versions of Autopsy, attachments were children of email/message artifacts + // and hence email/messages with attachments are shown in the tree data source tree, BlackboardArtifact art = key.getLookup().lookup(BlackboardArtifact.class); if (art != null && filterArtifacts - && ((DirectoryTreeUtils.showMessagesInDirTree() == false) - || (DirectoryTreeUtils.showMessagesInDirTree() + && ((FilterNodeUtils.showMessagesInDatasourceTree() == false) + || (FilterNodeUtils.showMessagesInDatasourceTree() && art.getArtifactTypeID() != BlackboardArtifact.ARTIFACT_TYPE.TSK_EMAIL_MSG.getTypeID() && art.getArtifactTypeID() != BlackboardArtifact.ARTIFACT_TYPE.TSK_MESSAGE.getTypeID()))) { return new Node[]{}; } - return new Node[]{new DataResultFilterNode(key, sourceEm)}; + return new Node[]{new DataResultFilterNode(key, sourceEm)}; } } diff --git a/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeFilterNode.java b/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeFilterNode.java index c5d5712ec8..06a0b18c10 100644 --- a/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeFilterNode.java +++ b/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeFilterNode.java @@ -132,7 +132,7 @@ class DirectoryTreeFilterNode extends FilterNode { } } else if (child instanceof BlackboardArtifact) { - if (DirectoryTreeUtils.showMessagesInDirTree()) { + if (FilterNodeUtils.showMessagesInDatasourceTree()) { // In older versions of Autopsy, attachments were children of email/message artifacts // and hence email/messages with attachments are shown in the directory tree. BlackboardArtifact bba = (BlackboardArtifact) child; diff --git a/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeUtils.java b/Core/src/org/sleuthkit/autopsy/directorytree/FilterNodeUtils.java similarity index 83% rename from Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeUtils.java rename to Core/src/org/sleuthkit/autopsy/directorytree/FilterNodeUtils.java index 40b3954897..ce8344862c 100644 --- a/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeUtils.java +++ b/Core/src/org/sleuthkit/autopsy/directorytree/FilterNodeUtils.java @@ -26,7 +26,7 @@ import org.sleuthkit.datamodel.CaseDbSchemaVersionNumber; * Utility class for Directory tree. * */ -final class DirectoryTreeUtils { +final class FilterNodeUtils { private static final int ATTACHMENT_CHILDOF_MSG_MAX_DB_MAJOR_VER = 8; private static final int ATTACHMENT_CHILDOF_MSG_MAX_DB_MINOR_VER = 4; @@ -34,7 +34,7 @@ final class DirectoryTreeUtils { /** * Empty private constructor */ - private DirectoryTreeUtils() { + private FilterNodeUtils() { } @@ -43,8 +43,8 @@ final class DirectoryTreeUtils { * hence messages with any attachment children are shown in the directory * tree. * - * At 8.4, attachments are tracked as an attribute, and the message artifact - * don't need to be shown in the directory tree. + * At 8.4 and later, attachments are tracked as an attribute, and the message + * artifacts don't need to be shown in the directory tree. * * This method may be used to check the schema version and behave * accordingly, in order to maintain backward compatibility. @@ -52,15 +52,15 @@ final class DirectoryTreeUtils { * @return True if messages with attachment children should be shown in * directory tree. */ - static boolean showMessagesInDirTree() { - boolean showMessagesInDirTree = true; + static boolean showMessagesInDatasourceTree() { + boolean showMessagesInDatasourceTree = true; if (Case.isCaseOpen()) { CaseDbSchemaVersionNumber version = Case.getCurrentCase().getSleuthkitCase().getDBSchemaCreationVersion(); - showMessagesInDirTree + showMessagesInDatasourceTree = ((version.getMajor() < ATTACHMENT_CHILDOF_MSG_MAX_DB_MAJOR_VER) || (version.getMajor() == ATTACHMENT_CHILDOF_MSG_MAX_DB_MAJOR_VER && version.getMinor() < ATTACHMENT_CHILDOF_MSG_MAX_DB_MINOR_VER)); } - return showMessagesInDirTree; + return showMessagesInDatasourceTree; } } diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java index 419173fcff..983e59e6b4 100644 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java @@ -138,7 +138,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { communicationArtifactsHelper = new CommunicationArtifactsHelper(currentCase.getSleuthkitCase(), EmailParserModuleFactory.getModuleName(), abstractFile, Account.Type.EMAIL); } catch (TskCoreException ex) { - logger.log(Level.SEVERE, String.format("Failed to create CommunicationArtifactsHelper for file %s", abstractFile.getName()), ex); + logger.log(Level.SEVERE, String.format("Failed to create CommunicationArtifactsHelper for file with object id = %d", abstractFile.getId()), ex); return ProcessResult.ERROR; } From 40f4f11badb684d0c27d70fdf48a6fab0f1c9b3f Mon Sep 17 00:00:00 2001 From: Kelly Kelly Date: Thu, 12 Dec 2019 13:15:31 -0500 Subject: [PATCH 07/22] Fixed typo and test button size --- .../autopsy/geolocation/Bundle.properties | 2 +- .../geolocation/Bundle.properties-MERGED | 2 +- .../geolocation/GeolocationSettingsPanel.form | 65 +++++++++++-------- .../geolocation/GeolocationSettingsPanel.java | 28 ++++---- 4 files changed, 53 insertions(+), 44 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/geolocation/Bundle.properties b/Core/src/org/sleuthkit/autopsy/geolocation/Bundle.properties index 5525fed84c..e21db15ee9 100755 --- a/Core/src/org/sleuthkit/autopsy/geolocation/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/geolocation/Bundle.properties @@ -24,7 +24,7 @@ GeolocationSettingsPanel.mbtileFileField.toolTipText= GeolocationSettingsPanel.mbtileFileField.text= GeolocationSettingsPanel.defaultDataSource.text=Default online tile server (bing.com/maps) GeolocationSettingsPanel.osmServerRBnt.text=OpenStreetMap server -GeolocationSettingsPanel.zipFileRBnt.text=OpenStreeMap zip file +GeolocationSettingsPanel.zipFileRBnt.text=OpenStreetMap zip file GeolocationSettingsPanel.zipFileRBnt.actionCommand=OpenStreeMap tile ZIP file GeolocationSettingsPanel.mbtilesRBtn.text=MBTiles file GeolocationSettingsPanel.osmServerAddressField.text= diff --git a/Core/src/org/sleuthkit/autopsy/geolocation/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/geolocation/Bundle.properties-MERGED index 19ce2c9a59..1fbba07220 100755 --- a/Core/src/org/sleuthkit/autopsy/geolocation/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/geolocation/Bundle.properties-MERGED @@ -59,7 +59,7 @@ GeolocationSettingsPanel.mbtileFileField.toolTipText= GeolocationSettingsPanel.mbtileFileField.text= GeolocationSettingsPanel.defaultDataSource.text=Default online tile server (bing.com/maps) GeolocationSettingsPanel.osmServerRBnt.text=OpenStreetMap server -GeolocationSettingsPanel.zipFileRBnt.text=OpenStreeMap zip file +GeolocationSettingsPanel.zipFileRBnt.text=OpenStreetMap zip file GeolocationSettingsPanel.zipFileRBnt.actionCommand=OpenStreeMap tile ZIP file GeolocationSettingsPanel.mbtilesRBtn.text=MBTiles file GeolocationSettingsPanel.osmServerAddressField.text= diff --git a/Core/src/org/sleuthkit/autopsy/geolocation/GeolocationSettingsPanel.form b/Core/src/org/sleuthkit/autopsy/geolocation/GeolocationSettingsPanel.form index e2121d0772..ed4de8758a 100755 --- a/Core/src/org/sleuthkit/autopsy/geolocation/GeolocationSettingsPanel.form +++ b/Core/src/org/sleuthkit/autopsy/geolocation/GeolocationSettingsPanel.form @@ -145,7 +145,7 @@ - + @@ -160,7 +160,7 @@ - + @@ -200,36 +200,45 @@ - - - - - - - - - + + + + + - + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Core/src/org/sleuthkit/autopsy/geolocation/GeolocationSettingsPanel.java b/Core/src/org/sleuthkit/autopsy/geolocation/GeolocationSettingsPanel.java index f66d05c08d..c2819d4a4f 100755 --- a/Core/src/org/sleuthkit/autopsy/geolocation/GeolocationSettingsPanel.java +++ b/Core/src/org/sleuthkit/autopsy/geolocation/GeolocationSettingsPanel.java @@ -159,6 +159,7 @@ final class GeolocationSettingsPanel extends javax.swing.JPanel implements Optio serverTestBtn = new javax.swing.JButton(); mbtilesRBtn = new javax.swing.JRadioButton(); mbtileFileField = new javax.swing.JTextField(); + javax.swing.JPanel MBTilesBtnPanel = new javax.swing.JPanel(); mbtilesBrowseBtn = new javax.swing.JButton(); mbtileTestBtn = new javax.swing.JButton(); @@ -236,7 +237,7 @@ final class GeolocationSettingsPanel extends javax.swing.JPanel implements Optio gridBagConstraints = new java.awt.GridBagConstraints(); gridBagConstraints.gridx = 2; gridBagConstraints.gridy = 2; - gridBagConstraints.anchor = java.awt.GridBagConstraints.NORTHWEST; + gridBagConstraints.anchor = java.awt.GridBagConstraints.WEST; gridBagConstraints.insets = new java.awt.Insets(0, 9, 9, 9); tilePane.add(zipFileBrowseBnt, gridBagConstraints); @@ -249,8 +250,8 @@ final class GeolocationSettingsPanel extends javax.swing.JPanel implements Optio gridBagConstraints = new java.awt.GridBagConstraints(); gridBagConstraints.gridx = 2; gridBagConstraints.gridy = 1; - gridBagConstraints.ipadx = 20; - gridBagConstraints.anchor = java.awt.GridBagConstraints.NORTHWEST; + gridBagConstraints.fill = java.awt.GridBagConstraints.HORIZONTAL; + gridBagConstraints.anchor = java.awt.GridBagConstraints.WEST; gridBagConstraints.insets = new java.awt.Insets(0, 9, 9, 9); tilePane.add(serverTestBtn, gridBagConstraints); @@ -277,18 +278,15 @@ final class GeolocationSettingsPanel extends javax.swing.JPanel implements Optio gridBagConstraints.insets = new java.awt.Insets(0, 0, 9, 0); tilePane.add(mbtileFileField, gridBagConstraints); + MBTilesBtnPanel.setLayout(new java.awt.GridLayout(1, 0, 5, 0)); + org.openide.awt.Mnemonics.setLocalizedText(mbtilesBrowseBtn, org.openide.util.NbBundle.getMessage(GeolocationSettingsPanel.class, "GeolocationSettingsPanel.mbtilesBrowseBtn.text")); // NOI18N mbtilesBrowseBtn.addActionListener(new java.awt.event.ActionListener() { public void actionPerformed(java.awt.event.ActionEvent evt) { mbtilesBrowseBtnActionPerformed(evt); } }); - gridBagConstraints = new java.awt.GridBagConstraints(); - gridBagConstraints.gridx = 2; - gridBagConstraints.gridy = 3; - gridBagConstraints.anchor = java.awt.GridBagConstraints.NORTHWEST; - gridBagConstraints.insets = new java.awt.Insets(0, 9, 9, 9); - tilePane.add(mbtilesBrowseBtn, gridBagConstraints); + MBTilesBtnPanel.add(mbtilesBrowseBtn); org.openide.awt.Mnemonics.setLocalizedText(mbtileTestBtn, org.openide.util.NbBundle.getMessage(GeolocationSettingsPanel.class, "GeolocationSettingsPanel.mbtileTestBtn.text")); // NOI18N mbtileTestBtn.addActionListener(new java.awt.event.ActionListener() { @@ -296,13 +294,15 @@ final class GeolocationSettingsPanel extends javax.swing.JPanel implements Optio mbtileTestBtnActionPerformed(evt); } }); + MBTilesBtnPanel.add(mbtileTestBtn); + gridBagConstraints = new java.awt.GridBagConstraints(); - gridBagConstraints.gridx = 3; + gridBagConstraints.gridx = 2; gridBagConstraints.gridy = 3; - gridBagConstraints.ipadx = 20; - gridBagConstraints.anchor = java.awt.GridBagConstraints.NORTHWEST; - gridBagConstraints.weightx = 1.0; - tilePane.add(mbtileTestBtn, gridBagConstraints); + gridBagConstraints.gridwidth = 2; + gridBagConstraints.anchor = java.awt.GridBagConstraints.WEST; + gridBagConstraints.insets = new java.awt.Insets(0, 9, 9, 9); + tilePane.add(MBTilesBtnPanel, gridBagConstraints); gridBagConstraints = new java.awt.GridBagConstraints(); gridBagConstraints.fill = java.awt.GridBagConstraints.HORIZONTAL; From 366c03fccdea76f968838ebea3b2a25c5aed8910 Mon Sep 17 00:00:00 2001 From: Kelly Kelly Date: Fri, 13 Dec 2019 14:33:02 -0500 Subject: [PATCH 08/22] Added support for artifacts that have long and lat but are not one of the expected 6 --- .../geolocation/GeolocationTopComponent.java | 2 + .../datamodel/CustomArtifactWaypoint.java | 79 +++++++++++++++++++ .../datamodel/WaypointBuilder.java | 5 +- 3 files changed, 85 insertions(+), 1 deletion(-) create mode 100755 Core/src/org/sleuthkit/autopsy/geolocation/datamodel/CustomArtifactWaypoint.java diff --git a/Core/src/org/sleuthkit/autopsy/geolocation/GeolocationTopComponent.java b/Core/src/org/sleuthkit/autopsy/geolocation/GeolocationTopComponent.java index b1dee1cbb4..aafd468957 100755 --- a/Core/src/org/sleuthkit/autopsy/geolocation/GeolocationTopComponent.java +++ b/Core/src/org/sleuthkit/autopsy/geolocation/GeolocationTopComponent.java @@ -431,6 +431,8 @@ public final class GeolocationTopComponent extends TopComponent { Bundle.GeoTopComponent_filter_exception_Title(), Bundle.GeoTopComponent_filter_exception_msg(), JOptionPane.ERROR_MESSAGE); + + setWaypointLoading(false); } }); } diff --git a/Core/src/org/sleuthkit/autopsy/geolocation/datamodel/CustomArtifactWaypoint.java b/Core/src/org/sleuthkit/autopsy/geolocation/datamodel/CustomArtifactWaypoint.java new file mode 100755 index 0000000000..bc3480ea40 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/geolocation/datamodel/CustomArtifactWaypoint.java @@ -0,0 +1,79 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2019 Basis Technology Corp. + * contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.geolocation.datamodel; + +import java.util.Map; +import org.sleuthkit.datamodel.BlackboardArtifact; +import org.sleuthkit.datamodel.BlackboardAttribute; + +/** + * Class wraps any artifact that is not one of the known types, but have the + * TSK_GEO_LONGITUDE and TSK_GEO_LATITUDE attributes. + * + */ +final class CustomArtifactWaypoint extends Waypoint { + + /** + * Constructs a new waypoint from the given artifact. + * + * @param artifact BlackboardArtifact for this waypoint + * + * @throws GeoLocationDataException + */ + CustomArtifactWaypoint(BlackboardArtifact artifact) throws GeoLocationDataException { + this(artifact, getAttributesFromArtifactAsMap(artifact)); + } + + /** + * Constructs a new CustomArtifactWaypoint. + * + * @param artifact BlackboardArtifact for this waypoint + * @param attributeMap A Map of the BlackboardAttributes for the given + * artifact. + * + * @throws GeoLocationDataException + */ + private CustomArtifactWaypoint(BlackboardArtifact artifact, Map attributeMap) throws GeoLocationDataException { + super(artifact, + getLabelFromArtifact(attributeMap), + attributeMap.get(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME) != null ? attributeMap.get(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME).getValueLong() : null, + attributeMap.get(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_GEO_LATITUDE) != null ? attributeMap.get(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_GEO_LATITUDE).getValueDouble() : null, + attributeMap.get(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_GEO_LONGITUDE) != null ? attributeMap.get(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_GEO_LONGITUDE).getValueDouble() : null, + attributeMap.get(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_GEO_ALTITUDE) != null ? attributeMap.get(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_GEO_ALTITUDE).getValueDouble() : null, + null, attributeMap, null); + } + + /** + * Gets the label for this waypoint. + * + * @param artifact BlackboardArtifact for waypoint + * + * @return Returns a label for the waypoint, or empty string if no label was + * found. + */ + private static String getLabelFromArtifact(Map attributeMap) { + BlackboardAttribute attribute = attributeMap.get(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_NAME); + if (attribute != null) { + return attribute.getDisplayString(); + } + + return ""; + } + +} diff --git a/Core/src/org/sleuthkit/autopsy/geolocation/datamodel/WaypointBuilder.java b/Core/src/org/sleuthkit/autopsy/geolocation/datamodel/WaypointBuilder.java index 86539412be..bccf5118a5 100755 --- a/Core/src/org/sleuthkit/autopsy/geolocation/datamodel/WaypointBuilder.java +++ b/Core/src/org/sleuthkit/autopsy/geolocation/datamodel/WaypointBuilder.java @@ -592,8 +592,11 @@ public final class WaypointBuilder { Route route = new Route(artifact); waypoints.addAll(route.getRoute()); break; + case TSK_GPS_LAST_KNOWN_LOCATION: + waypoints.add(new LastKnownWaypoint(artifact)); + break; default: - throw new GeoLocationDataException(String.format("Unable to create waypoint for artifact of type %s", type.toString())); + waypoints.add(new CustomArtifactWaypoint(artifact)); } return waypoints; From f4de27095c18bff5a6fdc1fb60655f38fa9f9ad0 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Fri, 13 Dec 2019 15:26:45 -0500 Subject: [PATCH 09/22] Updated parsers, refactored to eliminate complexty and duplication, made the config panel save last used path --- .../xry/AbstractSingleKeyValueParser.java | 52 +- .../xry/XRYCallsFileParser.java | 152 +++-- .../xry/XRYContactsFileParser.java | 22 +- .../XRYDataSourceProcessorConfigPanel.java | 43 +- .../xry/XRYDeviceGenInfoFileParser.java | 219 +++--- .../xry/XRYKeyValuePair.java | 126 ++++ .../xry/XRYMessagesFileParser.java | 624 +++++++++--------- .../xry/XRYWebBookmarksFileParser.java | 10 +- 8 files changed, 711 insertions(+), 537 deletions(-) create mode 100755 Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYKeyValuePair.java diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/AbstractSingleKeyValueParser.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/AbstractSingleKeyValueParser.java index 93b5051657..88fd334522 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/AbstractSingleKeyValueParser.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/AbstractSingleKeyValueParser.java @@ -38,8 +38,6 @@ import org.sleuthkit.datamodel.TskCoreException; abstract class AbstractSingleKeyValueParser implements XRYFileParser { private static final Logger logger = Logger.getLogger(AbstractSingleKeyValueParser.class.getName()); - - private static final char KEY_VALUE_DELIMITER = ':'; protected static final String PARSER_NAME = "XRY DSP"; @@ -73,32 +71,31 @@ abstract class AbstractSingleKeyValueParser implements XRYFileParser { //Find the XRY key on this line. Assume key is the value between //the start of the line and the first delimiter. - int keyDelimiter = xryLine.indexOf(KEY_VALUE_DELIMITER); - if (keyDelimiter == -1) { + if(!XRYKeyValuePair.isPair(xryLine)) { logger.log(Level.WARNING, String.format("[XRY DSP] Expected a key value " - + "pair on this line (in brackets) [ %s ], but one was not detected." - + " Here is the previous line [ %s ]. What does this mean?", xryLine, xryLines[i - 1])); + + "pair on this line (in brackets) [ %s ], but one was not detected.", + xryLine)); continue; } - String key = xryLine.substring(0, keyDelimiter).trim(); - String value = xryLine.substring(keyDelimiter + 1).trim(); + + XRYKeyValuePair pair = XRYKeyValuePair.from(xryLine, namespace); - if (!isKey(key)) { + if (!canProcess(pair)) { logger.log(Level.WARNING, String.format("[XRY DSP] The following key, " - + "value pair (in brackets, respectively) [ %s ], [ %s ] was not recognized. Discarding..." - + " Here is the previous line [ %s ] for context. What does this key mean?", key, value, xryLines[i - 1])); + + "value pair (in brackets) [ %s ] was not recognized. Discarding...", + pair)); continue; } - if (value.isEmpty()) { - logger.log(Level.WARNING, String.format("[XRY DSP] The following key " - + "(in brackets) [ %s ] was recognized, but the value was empty. Discarding..." - + " Here is the previous line for context [ %s ]. What does this mean?", key, xryLines[i - 1])); + if (pair.getValue().isEmpty()) { + logger.log(Level.WARNING, String.format("[XRY DSP] The following key value pair" + + "(in brackets) [ %s ] was recognized, but the value was empty. Discarding...", + pair)); continue; } //Create the attribute, if any. - Optional attribute = makeAttribute(namespace, key, value); + Optional attribute = getBlackboardAttribute(namespace, pair); if(attribute.isPresent()) { attributes.add(attribute.get()); } @@ -112,19 +109,13 @@ abstract class AbstractSingleKeyValueParser implements XRYFileParser { } /** - * Determines if the key candidate is a known key. A key candidate is a - * string literal that begins a line and is terminated by a semi-colon. - * - * Ex: - * - * Call Type : Missed - * - * "Call Type" would be the key candidate that was extracted. - * - * @param key Key to test. These keys are trimmed of whitespace only. - * @return Indication if this key can be processed. + * Determines if the XRY key value pair can be processed by the + * XRY report parser. + * + * @param pair + * @return */ - abstract boolean isKey(String key); + abstract boolean canProcess(XRYKeyValuePair pair); /** * Determines if the namespace candidate is a known namespace. A namespace @@ -148,11 +139,10 @@ abstract class AbstractSingleKeyValueParser implements XRYFileParser { * * @param nameSpace The namespace of this key value pair. * It will have been verified with isNamespace, otherwise it will be empty. - * @param key The key that was verified with isKey. - * @param value The value associated with that key. + * @param pair The XRYKeyValuePair to process * @return The corresponding blackboard attribute, if any. */ - abstract Optional makeAttribute(String nameSpace, String key, String value); + abstract Optional getBlackboardAttribute(String nameSpace, XRYKeyValuePair pair); /** * Makes an artifact from the parsed attributes. diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java index e0ae43721a..fde9b397aa 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java @@ -18,9 +18,15 @@ */ package org.sleuthkit.autopsy.datasourceprocessors.xry; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.OffsetDateTime; +import java.time.ZoneId; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; +import java.time.temporal.TemporalAccessor; +import java.time.temporal.TemporalQueries; import java.util.List; import java.util.Optional; import java.util.logging.Level; @@ -40,25 +46,37 @@ final class XRYCallsFileParser extends AbstractSingleKeyValueParser { //Pattern is in reverse due to a Java 8 bug, see calculateSecondsSinceEpoch() //function for more details. private static final DateTimeFormatter DATE_TIME_PARSER - = DateTimeFormatter.ofPattern("O a h:m:s M/d/y"); + = DateTimeFormatter.ofPattern("[(XXX) ][O ][(O) ]a h:m:s M/d/y"); + + private static final String DEVICE_LOCALE = "(device)"; + private static final String NETWORK_LOCALE = "(network)"; + /** - * All of the known XRY keys for call reports. + * All of the known XRY keys for call reports and the blackboard + * attribute types they map to. */ private enum XryKey { - TEL("tel"), - NAME_MATCHED("name (matched)"), - TIME("time"), - DIRECTION("direction"), - CALL_TYPE("call type"), - DURATION("duration"), - STORAGE("storage"), - INDEX("index"), - NAME("name"), - NUMBER("number"); + NUMBER("number", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER), + NAME_MATCHED("name (matched)", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_NAME), + TIME("time", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME), + DIRECTION("direction", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DIRECTION), + CALL_TYPE("call type", null), + TEL("tel", null), + DURATION("duration", null), + STORAGE("storage", null), + INDEX("index", null), + NAME("name", null); private final String name; - XryKey(String name) { + private final BlackboardAttribute.ATTRIBUTE_TYPE type; + + XryKey(String name, BlackboardAttribute.ATTRIBUTE_TYPE type) { this.name = name; + this.type = type; + } + + public BlackboardAttribute.ATTRIBUTE_TYPE getType() { + return type; } /** @@ -67,15 +85,13 @@ final class XRYCallsFileParser extends AbstractSingleKeyValueParser { * @param xryKey * @return */ - public static boolean contains(String xryKey) { - String normalizedKey = xryKey.trim().toLowerCase(); - for(XryKey keyChoice : XryKey.values()) { - if(keyChoice.name.equals(normalizedKey)) { - return true; - } + public static boolean contains(XRYKeyValuePair pair) { + try { + XryKey.fromPair(pair); + return true; + } catch (IllegalArgumentException ex) { + return false; } - - return false; } /** @@ -88,16 +104,15 @@ final class XRYCallsFileParser extends AbstractSingleKeyValueParser { * @param xryKey * @return */ - public static XryKey fromDisplayName(String xryKey) { - String normalizedKey = xryKey.trim().toLowerCase(); + public static XryKey fromPair(XRYKeyValuePair pair) { for(XryKey keyChoice : XryKey.values()) { - if(keyChoice.name.equals(normalizedKey)) { + if(pair.hasKey(keyChoice.name)) { return keyChoice; } } throw new IllegalArgumentException(String.format("Key [%s] was not found." - + " All keys should be tested with contains.", xryKey)); + + " All keys should be tested with contains.", pair.getKey())); } } @@ -155,8 +170,8 @@ final class XRYCallsFileParser extends AbstractSingleKeyValueParser { } @Override - boolean isKey(String key) { - return XryKey.contains(key); + boolean canProcess(XRYKeyValuePair pair) { + return XryKey.contains(pair); } @Override @@ -165,70 +180,55 @@ final class XRYCallsFileParser extends AbstractSingleKeyValueParser { } @Override - Optional makeAttribute(String nameSpace, String key, String value) { - XryKey xryKey = XryKey.fromDisplayName(key); + Optional getBlackboardAttribute(String nameSpace, XRYKeyValuePair pair) { + XryKey xryKey = XryKey.fromPair(pair); XryNamespace xryNamespace = XryNamespace.NONE; if(XryNamespace.contains(nameSpace)) { xryNamespace = XryNamespace.fromDisplayName(nameSpace); } switch (xryKey) { - case DIRECTION: - return Optional.of(new BlackboardAttribute( - BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DIRECTION, - PARSER_NAME, value)); - case NAME_MATCHED: - return Optional.of(new BlackboardAttribute( - BlackboardAttribute.ATTRIBUTE_TYPE.TSK_NAME, - PARSER_NAME, value)); - case NUMBER: - return Optional.of(new BlackboardAttribute( - BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER, - PARSER_NAME, value)); case TEL: //Apply the namespace switch (xryNamespace) { case FROM: return Optional.of(new BlackboardAttribute( BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_FROM, - PARSER_NAME, value)); + PARSER_NAME, pair.getValue())); case TO: return Optional.of(new BlackboardAttribute( BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_TO, - PARSER_NAME, value)); + PARSER_NAME, pair.getValue())); default: return Optional.of(new BlackboardAttribute( BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER, - PARSER_NAME, value)); + PARSER_NAME, pair.getValue())); } case TIME: try { //Tranform value to seconds since epoch - long dateTimeSinceEpoch = calculateSecondsSinceEpoch(value); - return Optional.of(new BlackboardAttribute( - BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME_START, + long dateTimeSinceEpoch = calculateSecondsSinceEpoch(pair.getValue()); + return Optional.of(new BlackboardAttribute(xryKey.getType(), PARSER_NAME, dateTimeSinceEpoch)); } catch (DateTimeParseException ex) { logger.log(Level.WARNING, String.format("[XRY DSP] Assumption" + " about the date time formatting of call logs is " - + "not right. Here is the value [ %s ]", value), ex); + + "not right. Here is the value [ %s ]", pair.getValue()), ex); return Optional.empty(); } - case DURATION: - case STORAGE: - case INDEX: - case CALL_TYPE: - //Ignore for now, don't need more data. - return Optional.empty(); - case NAME: - logger.log(Level.WARNING, String.format("[XRY DSP] Key [%s] was " - + "recognized but more examples of its values are needed " - + "to make a decision on an appropriate TSK attribute. " - + "Here is the value [%s].", key, value)); - return Optional.empty(); default: - throw new IllegalArgumentException(String.format("Key [ %s ] " - + "passed the isKey() test but was not matched.", key)); + //Otherwise, the XryKey enum contains the correct BlackboardAttribute + //type. + if(xryKey.getType() != null) { + return Optional.of(new BlackboardAttribute(xryKey.getType(), + PARSER_NAME, pair.getValue())); + } + + logger.log(Level.WARNING, String.format("[XRY DSP] Key value pair " + + "(in brackets) [ %s ] was recognized but " + + "more data or time is needed to finish implementation. Discarding... ", + pair)); + return Optional.empty(); } } @@ -247,12 +247,16 @@ final class XRYCallsFileParser extends AbstractSingleKeyValueParser { * @return A purer date time value. */ private String removeDateTimeLocale(String dateTime) { - int index = dateTime.indexOf('('); - if (index == -1) { - return dateTime; + String result = dateTime; + int deviceIndex = result.toLowerCase().indexOf(DEVICE_LOCALE); + if (deviceIndex != -1) { + result = result.substring(0, deviceIndex); } - - return dateTime.substring(0, index); + int networkIndex = result.toLowerCase().indexOf(NETWORK_LOCALE); + if(networkIndex != -1) { + result = result.substring(0, networkIndex); + } + return result; } /** @@ -289,8 +293,18 @@ final class XRYCallsFileParser extends AbstractSingleKeyValueParser { * will only introduce a trivial amount of error. */ String reversedDateTimeWithGMT = reversedDateTime.replace("UTC", "GMT"); - ZonedDateTime zonedDateTime = ZonedDateTime.parse(reversedDateTimeWithGMT, DATE_TIME_PARSER); - return zonedDateTime.toEpochSecond(); + TemporalAccessor result = DATE_TIME_PARSER.parseBest(reversedDateTimeWithGMT, + ZonedDateTime::from, + LocalDateTime::from, + OffsetDateTime::from); + //Query for the ZoneID + if(result.query(TemporalQueries.zoneId()) == null) { + //If none, assumed GMT+0. + return ZonedDateTime.of(LocalDateTime.from(result), + ZoneId.of("GMT")).toEpochSecond(); + } else { + return Instant.from(result).getEpochSecond(); + } } /** diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYContactsFileParser.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYContactsFileParser.java index 496d13d2ad..976caabbfe 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYContactsFileParser.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYContactsFileParser.java @@ -54,10 +54,10 @@ final class XRYContactsFileParser extends AbstractSingleKeyValueParser { put("account name", null); }}; - + @Override - boolean isKey(String key) { - String normalizedKey = key.toLowerCase(); + boolean canProcess(XRYKeyValuePair pair) { + String normalizedKey = pair.getKey().toLowerCase(); return XRY_KEYS.containsKey(normalizedKey); } @@ -68,23 +68,23 @@ final class XRYContactsFileParser extends AbstractSingleKeyValueParser { } @Override - Optional makeAttribute(String nameSpace, String key, String value) { - String normalizedKey = key.toLowerCase(); + Optional getBlackboardAttribute(String nameSpace, XRYKeyValuePair pair) { + String normalizedKey = pair.getKey().toLowerCase(); if(XRY_KEYS.containsKey(normalizedKey)) { BlackboardAttribute.ATTRIBUTE_TYPE attrType = XRY_KEYS.get(normalizedKey); if(attrType != null) { - return Optional.of(new BlackboardAttribute(attrType, PARSER_NAME, value)); + return Optional.of(new BlackboardAttribute(attrType, PARSER_NAME, pair.getValue())); } - logger.log(Level.WARNING, String.format("[XRY DSP] Key [%s] was " - + "recognized but more examples of its values are needed " - + "to make a decision on an appropriate TSK attribute. " - + "Here is the value [%s].", key, value)); + logger.log(Level.WARNING, String.format("[XRY DSP] Key value pair " + + "(in brackets) [ %s ] was recognized but we need " + + "more data or time to finish implementation. Discarding... ", + pair)); return Optional.empty(); } throw new IllegalArgumentException(String.format("Key [ %s ] passed the " - + "isKey() test but was not matched.", key)); + + "isKey() test but was not matched.", pair.getKey())); } @Override diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYDataSourceProcessorConfigPanel.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYDataSourceProcessorConfigPanel.java index c35666d9c1..a641ea1f73 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYDataSourceProcessorConfigPanel.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYDataSourceProcessorConfigPanel.java @@ -21,9 +21,15 @@ package org.sleuthkit.autopsy.datasourceprocessors.xry; import java.beans.PropertyChangeListener; import java.beans.PropertyChangeSupport; import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Optional; import javax.swing.JFileChooser; import javax.swing.JPanel; +import org.apache.commons.lang3.StringUtils; import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessor; +import org.sleuthkit.autopsy.coreutils.ModuleSettings; /** * Allows an examiner to configure the XRY Data source processor. @@ -32,10 +38,15 @@ import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessor; final class XRYDataSourceProcessorConfigPanel extends JPanel { private static final long serialVersionUID = 1L; + + private static final String PROP_LAST_USED_PATH = "LAST_USED_PATH"; + private static final String SETTINGS_CONTEXT = "XRYDataSourceProcessorConfigPanel_Settings"; + private static final XRYDataSourceProcessorConfigPanel INSTANCE = new XRYDataSourceProcessorConfigPanel(); - //Communicates + //Used to communicate with the DSP infrastructure. This config + //panel will indicate when it is ready for an update. private final PropertyChangeSupport pcs; /** @@ -47,6 +58,31 @@ final class XRYDataSourceProcessorConfigPanel extends JPanel { pcs = new PropertyChangeSupport(this); } + /** + * Persists the last used path between application runs. + */ + private void setLastUsedPath(Path selection) { + Path parent = selection.getParent(); + ModuleSettings.setConfigSetting(SETTINGS_CONTEXT, + PROP_LAST_USED_PATH, parent.toString()); + } + + /** + * Retrieves the last used path, if any. This path will be saved across + * application runs. + */ + private Optional getLastUsedPath() { + String lastFolderPath = ModuleSettings.getConfigSetting( + SETTINGS_CONTEXT, PROP_LAST_USED_PATH); + if (StringUtils.isNotBlank(lastFolderPath)) { + Path lastPath = Paths.get(lastFolderPath); + if (Files.exists(lastPath)) { + return Optional.of(lastPath); + } + } + return Optional.empty(); + } + /** * Gets the singleton XRYDataSourceProcessorConfigPanel. */ @@ -158,9 +194,14 @@ final class XRYDataSourceProcessorConfigPanel extends JPanel { JFileChooser fileChooser = new JFileChooser(); fileChooser.setMultiSelectionEnabled(false); fileChooser.setFileSelectionMode(JFileChooser.DIRECTORIES_ONLY); + Optional lastUsedPath = getLastUsedPath(); + if(lastUsedPath.isPresent()) { + fileChooser.setCurrentDirectory(lastUsedPath.get().toFile()); + } int returnVal = fileChooser.showOpenDialog(this); if (returnVal == JFileChooser.APPROVE_OPTION) { File selection = fileChooser.getSelectedFile(); + setLastUsedPath(selection.toPath()); filePathTextField.setText(selection.getAbsolutePath()); //This will notify the wizard to revalidate the data source processor. diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYDeviceGenInfoFileParser.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYDeviceGenInfoFileParser.java index 0665eabc64..4f21b57595 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYDeviceGenInfoFileParser.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYDeviceGenInfoFileParser.java @@ -41,7 +41,6 @@ final class XRYDeviceGenInfoFileParser implements XRYFileParser { //Human readable name of this parser. private static final String PARSER_NAME = "XRY DSP"; - private static final char KEY_VALUE_DELIMITER = ':'; //All known XRY keys for Device Gen Info reports. private static final String ATTRIBUTE_KEY = "attribute"; @@ -57,16 +56,18 @@ final class XRYDeviceGenInfoFileParser implements XRYFileParser { put("device type", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DEVICE_MAKE); put("mobile id (imei)", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_IMEI); put("security code", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PASSWORD); + put("unlock code", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PASSWORD); put("imei/meid", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_IMEI); put("model", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DEVICE_MODEL); put("wifi address", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_MAC_ADDRESS); + put("subscriber id (imsi)", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_IMSI); //There could be two of these on an artifact, not aware of a way //to distinguish between two DATE_TIMEs such as the ones below. put("device clock", null); put("pc clock", null); - //Ignore these for now, need more data. + //Ignore these for now, need more data or time to finish implementation. put("device family", null); put("advertising id", null); put("device status", null); @@ -78,18 +79,20 @@ final class XRYDeviceGenInfoFileParser implements XRYFileParser { }; /** - * Device-General Information reports have 2 key value pairs for every - * attribute. The two only known keys are "Data" and "Attribute", where data - * is some generic information that the Attribute key describes. + * Device-General Information reports generally have 2 key value pairs for + * every blackboard attribute. The two only known keys are "Data" and + * "Attribute", where data is some generic information that the Attribute + * key describes. * * Example: * - * Data: Nokia XYZ + * Data: Nokia XYZ * Attribute: Device Name * * This parse implementation assumes that the data field does not span - * multiple lines. If the data does span multiple lines, it will log an - * error describing an expectation for an "Attribute" key that is not found. + * multiple lines. If the data does span multiple lines, then on the next + * iteration it will log an exception proclaiming a failed attempt to make a + * 'Data' and 'Attribute' pair. * * @param reader The XRYFileReader that reads XRY entities from the * Device-General Information report. @@ -106,10 +109,11 @@ final class XRYDeviceGenInfoFileParser implements XRYFileParser { while (reader.hasNextEntity()) { String xryEntity = reader.nextEntity(); //Extract attributes from this entity. - List attributes = createTSKAttributes(xryEntity); + List attributes = getBlackboardAttributes(xryEntity); if (!attributes.isEmpty()) { //Save the artifact. - BlackboardArtifact artifact = parent.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_DEVICE_INFO); + BlackboardArtifact artifact = parent.newArtifact( + BlackboardArtifact.ARTIFACT_TYPE.TSK_DEVICE_INFO); artifact.addAttributes(attributes); } } @@ -118,72 +122,36 @@ final class XRYDeviceGenInfoFileParser implements XRYFileParser { /** * Parses the XRY entity, extracts key value pairs and creates blackboard * attributes from these key value pairs. - * - * @param xryEntity + * + * @param xryEntity XRY entity to parse * @return A collection of attributes from the XRY entity. */ - private List createTSKAttributes(String xryEntity) { - //Examine this XRY entity line by line. + private List getBlackboardAttributes(String xryEntity) { String[] xryLines = xryEntity.split("\n"); - List attributes = new ArrayList<>(); //First line of the entity is the title, the entity will always be non-empty. logger.log(Level.INFO, String.format("[XRY DSP] Processing [ %s ]", xryLines[0])); + List attributes = new ArrayList<>(); - for (int i = 1; i < xryLines.length; i++) { - String xryLine = xryLines[i]; - - //Expecting to see a "Data" key. - if (!hasDataKey(xryLine)) { - logger.log(Level.WARNING, String.format("[XRY DSP] Expected a " - + "'Data' key on this line (in brackets) [ %s ], but none " - + "was found. Discarding... Here is the previous line for" - + " context [ %s ]. What does this mean?", - xryLine, xryLines[i - 1])); + //Iterate two lines at a time. For Device-General Information, we generally + //need two XRY Key Value pairs per blackboard attribute. + for (int i = 1; i < xryLines.length; i += 2) { + if (!XRYKeyValuePair.isPair(xryLines[i])) { + //log continue; } - int dataKeyIndex = xryLine.indexOf(KEY_VALUE_DELIMITER); - String dataValue = xryLine.substring(dataKeyIndex + 1).trim(); - - /** - * If there is only a Data key in the XRY Entity, then assume it is - * the path to the device. - */ + XRYKeyValuePair firstPair = XRYKeyValuePair.from(xryLines[i]); + Optional attribute = Optional.empty(); if (i + 1 == xryLines.length) { - attributes.add(new BlackboardAttribute( - BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PATH, - PARSER_NAME, dataValue)); - continue; + attribute = getBlackboardAttribute(firstPair); + } else if (XRYKeyValuePair.isPair(xryLines[i + 1])) { + XRYKeyValuePair secondPair = XRYKeyValuePair.from(xryLines[i + 1]); + attribute = getBlackboardAttribute(firstPair, secondPair); + } else { + //log } - String nextXryLine = xryLines[++i]; - - //Expecting to see an "Attribute" key - if (!hasXRYAttributeKey(nextXryLine)) { - logger.log(Level.WARNING, String.format("[XRY DSP] Expected an " - + "'Attribute' key on this line (in brackets) [ %s ], " - + "but none was found. Discarding... Here is the previous " - + "line for context [ %s ]. What does this mean?", - nextXryLine, xryLine)); - continue; - } - - int attributeKeyIndex = nextXryLine.indexOf(KEY_VALUE_DELIMITER); - String attributeValue = nextXryLine.substring(attributeKeyIndex + 1).trim(); - String normalizedAttributeValue = attributeValue.toLowerCase(); - - //Check if this value is known. - if (!isXRYAttributeValueRecognized(normalizedAttributeValue)) { - logger.log(Level.WARNING, String.format("[XRY DSP] Attribute value " - + "(in brackets) [ %s ] was not recognized. Discarding... " - + "Here is the data field for context [ %s ]. " - + "What does this mean?", attributeValue, dataValue)); - continue; - } - - Optional attribute = createTSKAttribute( - normalizedAttributeValue, dataValue); if (attribute.isPresent()) { attributes.add(attribute.get()); } @@ -192,76 +160,75 @@ final class XRYDeviceGenInfoFileParser implements XRYFileParser { } /** - * Creates the appropriate blackboard attribute given the XRY Key Value pair. - * If the value is recognized but has no corresponding Blackboard - * attribute type, the Optional will be empty. + * Creates the appropriate blackboard attribute given a single XRY Key Value + * pair. It is assumed that the only 'Data' keys can appear by themselves in + * Device-Gen Info reports. If a Data key is by itself, then it's assumed to + * be a TSK_PATH attribute. + * + * A WARNING will be logged if this input is not a Data key. * - * A WARNING message will be logged for all recognized values that don't have - * a type. More data is needed to make a decision about the appropriate type. - * - * @param normalizedAttributeValue Normalized (trimmed and lowercased) - * attribute value to map. - * @param dataValue The value of the blackboard attribute. - * @return Corresponding BlackboardAttribute, if any. + * @param pair KeyValuePair to + * @return */ - private Optional createTSKAttribute( - String normalizedAttributeValue, String dataValue) { + private Optional getBlackboardAttribute(XRYKeyValuePair pair) { + if (pair.hasKey(DATA_KEY)) { + return Optional.of(new BlackboardAttribute( + BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PATH, + PARSER_NAME, pair.getValue())); + } + + logger.log(Level.WARNING, "Expected a 'Data' key value pair, but [ %s ] " + + "was found.", pair); + + return Optional.empty(); + } + + /** + * Creates the appropriate blackboard attribute given the XRY Key Value + * pairs. If the attribute value is recognized but has no corresponding + * Blackboard attribute type, the Optional will be empty. + * + * A WARNING message will be logged for all recognized attribute values that + * don't have a type. More data is needed to make a decision about the + * appropriate type. + * + * @param firstPair + * @param secondPair + * @return + */ + private Optional getBlackboardAttribute(XRYKeyValuePair firstPair, XRYKeyValuePair secondPair) { + String attributeValue; + String dataValue; + if (firstPair.hasKey(DATA_KEY) && secondPair.hasKey(ATTRIBUTE_KEY)) { + dataValue = firstPair.getValue(); + attributeValue = secondPair.getValue(); + } else if (firstPair.hasKey(ATTRIBUTE_KEY) && secondPair.hasKey(DATA_KEY)) { + dataValue = secondPair.getValue(); + attributeValue = firstPair.getValue(); + } else { + logger.log(Level.WARNING, String.format("[XRY DSP] Expected these key value" + + " pairs (in brackets) [ %s ], [ %s ] to be an 'Attribute' and 'Data' " + + "pair.", firstPair, secondPair)); + return Optional.empty(); + } + + String normalizedAttributeValue = attributeValue.toLowerCase(); + if (!KEY_TO_TYPE.containsKey(normalizedAttributeValue)) { + logger.log(Level.WARNING, String.format("[XRY DSP] Key value pair " + + "(in brackets) [ %s : %s ] was not recognized. Discarding... ", + attributeValue, dataValue)); + return Optional.empty(); + } + BlackboardAttribute.ATTRIBUTE_TYPE attrType = KEY_TO_TYPE.get(normalizedAttributeValue); if (attrType == null) { - logger.log(Level.WARNING, String.format("[XRY DSP] Key [%s] was " - + "recognized but more examples of its values are needed " - + "to make a decision on an appropriate TSK attribute. " - + "Here is the value [%s].", normalizedAttributeValue, dataValue)); + logger.log(Level.WARNING, String.format("[XRY DSP] Key value pair " + + "(in brackets) [ %s : %s ] was recognized but we need " + + "more data or time to finish implementation. Discarding... ", + attributeValue, dataValue)); return Optional.empty(); } return Optional.of(new BlackboardAttribute(attrType, PARSER_NAME, dataValue)); } - - /** - * Tests if the attribute value is a recognized type. - * - * @param normalizedAttributeValue Normalized (trimmed and lowercased) value - * to test. - * @return True if the attribute value is known, False otherwise. - */ - private boolean isXRYAttributeValueRecognized(String normalizedAttributeValue) { - return KEY_TO_TYPE.containsKey(normalizedAttributeValue); - } - - /** - * Tests if the XRY line has a data key on it. - * - * @param xryLine - * @return - */ - private boolean hasDataKey(String xryLine) { - int dataKeyIndex = xryLine.indexOf(KEY_VALUE_DELIMITER); - //No key structure found. - if (dataKeyIndex == -1) { - return false; - } - - String normalizedDataKey = xryLine.substring(0, - dataKeyIndex).trim().toLowerCase(); - return normalizedDataKey.equals(DATA_KEY); - } - - /** - * Tests if the XRY line has an attribute key on it. - * - * @param xryLine - * @return - */ - private boolean hasXRYAttributeKey(String xryLine) { - int attributeKeyIndex = xryLine.indexOf(KEY_VALUE_DELIMITER); - //No key structure found. - if (attributeKeyIndex == -1) { - return false; - } - - String normalizedDataKey = xryLine.substring(0, - attributeKeyIndex).trim().toLowerCase(); - return normalizedDataKey.equals(ATTRIBUTE_KEY); - } } diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYKeyValuePair.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYKeyValuePair.java new file mode 100755 index 0000000000..0befc4bbf7 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYKeyValuePair.java @@ -0,0 +1,126 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2019 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.datasourceprocessors.xry; + +/** + * A XRY Key Value pair. + */ +class XRYKeyValuePair { + + private static final char KEY_VALUE_DELIMITER = ':'; + + private final String key; + private final String value; + private final String namespace; + + public XRYKeyValuePair(String key, String value, String namespace) { + this.key = key.trim(); + this.value = value.trim(); + this.namespace = namespace.trim(); + } + + /** + * Tests if the key of this pair matches some expected value. + * + * The expected value and the key of this pair will both be + * normalized (trimmed and lowercased). + * + * @param targetKey Key name to test. + */ + public boolean hasKey(String targetKey) { + String normalizedKey = key.toLowerCase(); + String normalizedTargetKey = targetKey.trim().toLowerCase(); + return normalizedKey.equals(normalizedTargetKey); + } + + /** + * Retrieves the value contained within this pair. + */ + public String getValue() { + return value; + } + + /** + * Retrieves the key contained within this pair. + */ + public String getKey() { + return key; + } + + /** + * Retrieves the namespace contained within this pair. + */ + public String getNamespace() { + return namespace; + } + + /** + * Tests if the input has the structure of a key value pair. + * + * @param xryLine XRY entity line to test. + */ + static boolean isPair(String input) { + int dataKeyIndex = input.indexOf(KEY_VALUE_DELIMITER); + //No key structure found. + return dataKeyIndex != -1; + } + + /** + * Extracts a key value pair from the input. + * + * This function assumes that there is a key value structure on the line. It + * can be verified using hasKeyValueForm(). + * + * @param input Input key value string to parse. + * @param namespace Namespace to assign to this pair. + * @return + */ + static XRYKeyValuePair from(String input, String namespace) { + if(!isPair(input)) { + throw new IllegalArgumentException("Input does not have the structure" + + " of a key value pair"); + } + + int keyIndex = input.indexOf(KEY_VALUE_DELIMITER); + String parsedKey = input.substring(0, keyIndex); + String parsedValue = input.substring(keyIndex + 1); + return new XRYKeyValuePair(parsedKey, parsedValue, namespace); + } + + /** + * Extracts a key value pair from the input. + * + * This function assumes that there is a key value structure on the line. It + * can be verified using hasKeyValueForm(). + * + * @param input Input key value string to parse. + * @return + */ + static XRYKeyValuePair from(String input) { + return from(input, ""); + } + + @Override + public String toString() { + if(namespace.isEmpty()) { + return key + " : " + value; + } + return "(Namespace: " + namespace +") " + key + " : " + value; + } +} diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYMessagesFileParser.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYMessagesFileParser.java index 941b26ecc2..37c2ecacc9 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYMessagesFileParser.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYMessagesFileParser.java @@ -20,12 +20,19 @@ package org.sleuthkit.autopsy.datasourceprocessors.xry; import java.io.IOException; import java.nio.file.Path; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.OffsetDateTime; +import java.time.ZoneId; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; +import java.time.temporal.TemporalAccessor; +import java.time.temporal.TemporalQueries; import java.util.ArrayList; import java.util.HashSet; import java.util.List; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.logging.Level; @@ -44,39 +51,56 @@ final class XRYMessagesFileParser implements XRYFileParser { XRYMessagesFileParser.class.getName()); private static final String PARSER_NAME = "XRY DSP"; - private static final char KEY_VALUE_DELIMITER = ':'; //Pattern is in reverse due to a Java 8 bug, see calculateSecondsSinceEpoch() //function for more details. private static final DateTimeFormatter DATE_TIME_PARSER - = DateTimeFormatter.ofPattern("O a h:m:s M/d/y"); - - //A more readable version of these values. Referring to if the user - //has read the message. + = DateTimeFormatter.ofPattern("[(XXX) ][O ][(O) ]a h:m:s M/d/y"); + + private static final String DEVICE_LOCALE = "(device)"; + private static final String NETWORK_LOCALE = "(network)"; + private static final int READ = 1; private static final int UNREAD = 0; /** - * All of the known XRY keys for message reports. + * All of the known XRY keys for message reports and the blackboard + * attribute types they map to. */ private enum XryKey { - TEXT("text"), - DIRECTION("direction"), - TIME("time"), - STATUS("status"), - TEL("tel"), - STORAGE("storage"), - INDEX("index"), - FOLDER("folder"), - SERVICE_CENTER("service center"), - TYPE("type"), - NAME("name"), - NAME_MATCHED("name (matched)"); + DELETED("deleted", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_ISDELETED), + DIRECTION("direction", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DIRECTION), + MESSAGE("message", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_TEXT), + NAME_MATCHED("name (matched)", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_NAME_PERSON), + TEXT("text", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_TEXT), + TIME("time", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME), + SERVICE_CENTER("service center", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER), + FROM("from", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_FROM), + TO("to", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_TO), + //The following keys either need special processing or more time and data to find a type. + STORAGE("storage", null), + NUMBER("number", null), + TYPE("type", null), + TEL("tel", null), + FOLDER("folder", null), + NAME("name", null), + INDEX("index", null), + STATUS("status", null); private final String name; + private final BlackboardAttribute.ATTRIBUTE_TYPE type; - XryKey(String name) { + XryKey(String name, BlackboardAttribute.ATTRIBUTE_TYPE type) { this.name = name; + this.type = type; + } + + public BlackboardAttribute.ATTRIBUTE_TYPE getType() { + return type; + } + + public String getDisplayName() { + return name; } /** @@ -85,15 +109,13 @@ final class XRYMessagesFileParser implements XRYFileParser { * @param xryKey * @return */ - public static boolean contains(String xryKey) { - String normalizedKey = xryKey.trim().toLowerCase(); - for(XryKey keyChoice : XryKey.values()) { - if(keyChoice.name.equals(normalizedKey)) { - return true; - } + public static boolean contains(String name) { + try { + XryKey.fromDisplayName(name); + return true; + } catch (IllegalArgumentException ex) { + return false; } - - return false; } /** @@ -106,16 +128,16 @@ final class XRYMessagesFileParser implements XRYFileParser { * @param xryKey * @return */ - public static XryKey fromDisplayName(String xryKey) { - String normalizedKey = xryKey.trim().toLowerCase(); + public static XryKey fromDisplayName(String name) { + String normalizedName = name.trim().toLowerCase(); for(XryKey keyChoice : XryKey.values()) { - if(keyChoice.name.equals(normalizedKey)) { + if(normalizedName.equals(keyChoice.name)) { return keyChoice; } } - throw new IllegalArgumentException(String.format("Key [%s] was not found." - + " All keys should be tested with contains.", xryKey)); + throw new IllegalArgumentException(String.format("Key [ %s ] was not found." + + " All keys should be tested with contains.", name)); } } @@ -123,9 +145,9 @@ final class XRYMessagesFileParser implements XRYFileParser { * All of the known XRY namespaces for message reports. */ private enum XryNamespace { - TO("to"), FROM("from"), PARTICIPANT("participant"), + TO("to"), NONE(null); private final String name; @@ -141,14 +163,12 @@ final class XRYMessagesFileParser implements XRYFileParser { * @return */ public static boolean contains(String xryNamespace) { - String normalizedNamespace = xryNamespace.trim().toLowerCase(); - for(XryNamespace keyChoice : XryNamespace.values()) { - if(normalizedNamespace.equals(keyChoice.name)) { - return true; - } + try { + XryNamespace.fromDisplayName(xryNamespace); + return true; + } catch (IllegalArgumentException ex) { + return false; } - - return false; } /** @@ -179,8 +199,8 @@ final class XRYMessagesFileParser implements XRYFileParser { */ private enum XryMetaKey { REFERENCE_NUMBER("reference number"), - SEGMENT_NUMBER("segment number"), - SEGMENT_COUNT("segments"); + SEGMENT_COUNT("segments"), + SEGMENT_NUMBER("segment number"); private final String name; @@ -188,51 +208,54 @@ final class XRYMessagesFileParser implements XRYFileParser { this.name = name; } + public String getDisplayName() { + return name; + } + + /** - * Indicates if the display name of the XRY meta key is a recognized type. + * Indicates if the display name of the XRY key is a recognized type. * - * @param xryMetaKey + * @param xryKey * @return */ - public static boolean contains(String xryMetaKey) { - String normalizedMetaKey = xryMetaKey.trim().toLowerCase(); - for(XryMetaKey keyChoice : XryMetaKey.values()) { - if(keyChoice.name.equals(normalizedMetaKey)) { - return true; - } + public static boolean contains(String name) { + try { + XryMetaKey.fromDisplayName(name); + return true; + } catch (IllegalArgumentException ex) { + return false; } - - return false; } /** - * Matches the display name of the xry meta key to the appropriate enum type. + * Matches the display name of the xry key to the appropriate enum type. * - * It is assumed that XRY meta key string is recognized. Otherwise, + * It is assumed that XRY key string is recognized. Otherwise, * an IllegalArgumentException is thrown. Test all membership * with contains() before hand. * - * @param xryMetaKey + * @param xryKey * @return */ - public static XryMetaKey fromDisplayName(String xryMetaKey) { - String normalizedMetaKey = xryMetaKey.trim().toLowerCase(); + public static XryMetaKey fromDisplayName(String name) { + String normalizedName = name.trim().toLowerCase(); for(XryMetaKey keyChoice : XryMetaKey.values()) { - if(keyChoice.name.equals(normalizedMetaKey)) { + if(normalizedName.equals(keyChoice.name)) { return keyChoice; } } - throw new IllegalArgumentException(String.format("Meta key [%s] was not found." - + " All meta keys should be tested with contains.", xryMetaKey)); + throw new IllegalArgumentException(String.format("Key [ %s ] was not found." + + " All keys should be tested with contains.", name)); } } /** * Message-SMS report artifacts can span multiple XRY entities and their - * attributes can span multiple lines. The "Text" key is the only known key + * attributes can span multiple lines. The "Text" and "Message" keys are the only known key * value pair that can span multiple lines. Messages can be segmented, - * meaning that their "Text" content can appear in multiple XRY entities. + * meaning that their "Text" and "Message" content can appear in multiple XRY entities. * Our goal for a segmented message is to aggregate all of the text pieces and * create 1 artifact. * @@ -258,110 +281,7 @@ final class XRYMessagesFileParser implements XRYFileParser { while (reader.hasNextEntity()) { String xryEntity = reader.nextEntity(); - String[] xryLines = xryEntity.split("\n"); - - //First line of the entity is the title, each XRY entity is non-empty. - logger.log(Level.INFO, String.format("[XRY DSP] Processing [ %s ]", xryLines[0])); - - List attributes = new ArrayList<>(); - - XryNamespace namespace = XryNamespace.NONE; - for (int i = 1; i < xryLines.length; i++) { - String xryLine = xryLines[i]; - - if (XryNamespace.contains(xryLine)) { - namespace = XryNamespace.fromDisplayName(xryLine); - continue; - } - - //Find the XRY key on this line. - int keyDelimiter = xryLine.indexOf(KEY_VALUE_DELIMITER); - if (keyDelimiter == -1) { - logger.log(Level.SEVERE, String.format("[XRY DSP] Expected a key value " - + "pair on this line (in brackets) [ %s ], but one was not detected." - + " Is this the continuation of a previous line?" - + " Here is the previous line (in brackets) [ %s ]. " - + "What does this key mean?", xryLine, xryLines[i - 1])); - continue; - } - - //Extract the key value pair - String key = xryLine.substring(0, keyDelimiter); - String value = xryLine.substring(keyDelimiter + 1).trim(); - - if (XryMetaKey.contains(key)) { - //Skip meta keys, they are being handled seperately. - continue; - } - - if (!XryKey.contains(key)) { - logger.log(Level.SEVERE, String.format("[XRY DSP] The following key, " - + "value pair (in brackets, respectively) [ %s ], [ %s ] " - + "was not recognized. Discarding... Here is the previous line " - + "[ %s ] for context. What does this key mean?", key, value, xryLines[i - 1])); - continue; - } - - if (value.isEmpty()) { - logger.log(Level.SEVERE, String.format("[XRY DSP] The following key " - + "(in brackets) [ %s ] was recognized, but the value " - + "was empty. Discarding... Here is the previous line " - + "for context [ %s ]. Is this a continuation of this line? " - + "What does an empty key mean?", key, xryLines[i - 1])); - continue; - } - - XryKey xryKey = XryKey.fromDisplayName(key); - - //Assume text is the only field that can span multiple lines. - if (xryKey.equals(XryKey.TEXT)) { - //Build up multiple lines. - for (; (i + 1) < xryLines.length - && !hasKey(xryLines[i + 1]) - && !hasNamespace(xryLines[i + 1]); i++) { - String continuedValue = xryLines[i + 1].trim(); - //Assume multi lined values are split by word. - value = value + " " + continuedValue; - } - - Optional referenceNumber = getMetaInfo(xryLines, XryMetaKey.REFERENCE_NUMBER); - //Check if there is any segmented text. - if (referenceNumber.isPresent()) { - logger.log(Level.INFO, String.format("[XRY DSP] Message entity " - + "appears to be segmented with reference number [ %d ]", referenceNumber.get())); - - if (referenceNumbersSeen.contains(referenceNumber.get())) { - logger.log(Level.SEVERE, String.format("[XRY DSP] This reference [ %d ] has already " - + "been seen. This means that the segments are not " - + "contiguous. Any segments contiguous with this " - + "one will be aggregated and another " - + "(otherwise duplicate) artifact will be created.", referenceNumber.get())); - } - - referenceNumbersSeen.add(referenceNumber.get()); - - Optional segmentNumber = getMetaInfo(xryLines, XryMetaKey.SEGMENT_NUMBER); - if(segmentNumber.isPresent()) { - //Unify segmented text - String segmentedText = getSegmentedText(referenceNumber.get(), - segmentNumber.get(), reader); - //Assume it was segmented by word. - value = value + " " + segmentedText; - } else { - logger.log(Level.SEVERE, String.format("No segment " - + "number was found on the message entity" - + "with reference number [%d]", referenceNumber.get())); - } - } - } - - //Get the corresponding blackboard attribute, if any. - Optional attribute = makeAttribute(namespace, xryKey, value); - if (attribute.isPresent()) { - attributes.add(attribute.get()); - } - } - + List attributes = getBlackboardAttributes(xryEntity, reader, referenceNumbersSeen); //Only create artifacts with non-empty attributes. if(!attributes.isEmpty()) { BlackboardArtifact artifact = parent.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_MESSAGE); @@ -369,6 +289,82 @@ final class XRYMessagesFileParser implements XRYFileParser { } } } + + /** + * + * @param xryEntity + * @param reader + * @param referenceValues + * @return + * @throws IOException + */ + private List getBlackboardAttributes(String xryEntity, + XRYFileReader reader, Set referenceValues) throws IOException { + String[] xryLines = xryEntity.split("\n"); + //First line of the entity is the title, each XRY entity is non-empty. + logger.log(Level.INFO, String.format("[XRY DSP] Processing [ %s ]", xryLines[0])); + + List attributes = new ArrayList<>(); + + //Count the key value pairs in the XRY entity. + int keyCount = countKeys(xryLines); + for (int i = 1; i <= keyCount; i++) { + //Get the ith key value pair in the entity. Always expect to have + //a valid value. + XRYKeyValuePair pair = getKeyValuePairByIndex(xryLines, i).get(); + if (XryMetaKey.contains(pair.getKey())) { + //Skip meta keys, they are being handled seperately. + continue; + } + + if (!XryKey.contains(pair.getKey())) { + logger.log(Level.WARNING, String.format("[XRY DSP] The following key, " + + "value pair (in brackets) [ %s ], " + + "was not recognized. Discarding...", pair)); + continue; + } + + if (pair.getValue().isEmpty()) { + logger.log(Level.WARNING, String.format("[XRY DSP] The following key " + + "(in brackets) [ %s ] was recognized, but the value " + + "was empty. Discarding...", pair.getKey())); + continue; + } + + //Assume text and message are the only fields that can be segmented + //among multiple XRY entities. + if (pair.hasKey(XryKey.TEXT.getDisplayName()) || + pair.hasKey(XryKey.MESSAGE.getDisplayName())) { + String segmentedText = getSegmentedText(xryLines, reader, referenceValues); + pair = new XRYKeyValuePair(pair.getKey(), + //Assume text is segmented by word. + pair.getValue() + " " + segmentedText, + pair.getNamespace()); + } + + //Get the corresponding blackboard attribute, if any. + Optional attribute = getBlackboardAttribute(pair); + if (attribute.isPresent()) { + attributes.add(attribute.get()); + } + } + + return attributes; + } + + /** + * Counts the key value pairs in an XRY entity. + * Skips counting the first line as it is assumed to be the title. + */ + private Integer countKeys(String[] xryEntity) { + int count = 0; + for (int i = 1; i < xryEntity.length; i++) { + if(XRYKeyValuePair.isPair(xryEntity[i])) { + count++; + } + } + return count; + } /** * Builds up segmented message entities so that the text is unified in the @@ -380,17 +376,46 @@ final class XRYMessagesFileParser implements XRYFileParser { * @return * @throws IOException */ - private String getSegmentedText(int referenceNumber, int segmentNumber, XRYFileReader reader) throws IOException { + private String getSegmentedText(String[] xryEntity, XRYFileReader reader, + Set referenceNumbersSeen) throws IOException { + Optional referenceNumber = getMetaKeyValue(xryEntity, XryMetaKey.REFERENCE_NUMBER); + //Check if there is any segmented text. + if(!referenceNumber.isPresent()) { + return ""; + } + + logger.log(Level.INFO, String.format("[XRY DSP] Message entity " + + "appears to be segmented with reference number [ %d ]", referenceNumber.get())); + + if (referenceNumbersSeen.contains(referenceNumber.get())) { + logger.log(Level.SEVERE, String.format("[XRY DSP] This reference [ %d ] has already " + + "been seen. This means that the segments are not " + + "contiguous. Any segments contiguous with this " + + "one will be aggregated and another " + + "(otherwise duplicate) artifact will be created.", referenceNumber.get())); + } + + referenceNumbersSeen.add(referenceNumber.get()); + + Optional segmentNumber = getMetaKeyValue(xryEntity, XryMetaKey.SEGMENT_NUMBER); + if(!segmentNumber.isPresent()) { + logger.log(Level.SEVERE, String.format("No segment " + + "number was found on the message entity" + + "with reference number [%d]", referenceNumber.get())); + return ""; + } + StringBuilder segmentedText = new StringBuilder(); - int currentSegmentNumber = segmentNumber; + int currentSegmentNumber = segmentNumber.get(); while (reader.hasNextEntity()) { //Peek at the next to see if it has the same reference number. String nextEntity = reader.peek(); String[] nextEntityLines = nextEntity.split("\n"); - Optional nextReferenceNumber = getMetaInfo(nextEntityLines, XryMetaKey.REFERENCE_NUMBER); + Optional nextReferenceNumber = getMetaKeyValue(nextEntityLines, XryMetaKey.REFERENCE_NUMBER); - if (!nextReferenceNumber.isPresent() || nextReferenceNumber.get() != referenceNumber) { + if (!nextReferenceNumber.isPresent() || + !Objects.equals(nextReferenceNumber, referenceNumber)) { //Don't consume the next entity. It is not related //to the current message thread. break; @@ -399,44 +424,28 @@ final class XRYMessagesFileParser implements XRYFileParser { //Consume the entity, it is a part of the message thread. reader.nextEntity(); - Optional nextSegmentNumber = getMetaInfo(nextEntityLines, XryMetaKey.SEGMENT_NUMBER); + Optional nextSegmentNumber = getMetaKeyValue(nextEntityLines, XryMetaKey.SEGMENT_NUMBER); logger.log(Level.INFO, String.format("[XRY DSP] Processing [ %s ] " - + "segment with reference number [ %d ]", nextEntityLines[0], referenceNumber)); + + "segment with reference number [ %d ]", nextEntityLines[0], referenceNumber.get())); if(!nextSegmentNumber.isPresent()) { logger.log(Level.SEVERE, String.format("[XRY DSP] Segment with reference" + " number [ %d ] did not have a segment number associated with it." - + " It cannot be determined if the reconstructed text will be in order.", referenceNumber)); + + " It cannot be determined if the reconstructed text will be in order.", referenceNumber.get())); } else if (nextSegmentNumber.get() != currentSegmentNumber + 1) { logger.log(Level.SEVERE, String.format("[XRY DSP] Contiguous " + "segments are not ascending incrementally. Encountered " + "segment [ %d ] after segment [ %d ]. This means the reconstructed " - + "text will be out of order.", nextSegmentNumber, currentSegmentNumber)); + + "text will be out of order.", nextSegmentNumber.get(), currentSegmentNumber)); } - for (int i = 1; i < nextEntityLines.length; i++) { - String xryLine = nextEntityLines[i]; - //Find the XRY key on this line. - int keyDelimiter = xryLine.indexOf(KEY_VALUE_DELIMITER); - if (keyDelimiter == -1) { - //Skip this line, we are searching only for a text key-value pair. - continue; - } - - //Extract the text key from the entity - String key = xryLine.substring(0, keyDelimiter); - if(XryKey.contains(key) && XryKey.fromDisplayName(key).equals(XryKey.TEXT)) { - String value = xryLine.substring(keyDelimiter + 1).trim(); - segmentedText.append(value).append(' '); - - //Build up multiple lines. - for (; (i + 1) < nextEntityLines.length - && !hasKey(nextEntityLines[i + 1]) - && !hasNamespace(nextEntityLines[i + 1]); i++) { - String continuedValue = nextEntityLines[i + 1].trim(); - segmentedText.append(continuedValue).append(' '); - } + int keyCount = countKeys(nextEntityLines); + for (int i = 1; i <= keyCount; i++) { + XRYKeyValuePair pair = getKeyValuePairByIndex(nextEntityLines, i).get(); + if(pair.hasKey(XryKey.TEXT.getDisplayName()) || + pair.hasKey(XryKey.MESSAGE.getDisplayName())) { + segmentedText.append(pair.getValue()).append(' '); } } @@ -449,65 +458,73 @@ final class XRYMessagesFileParser implements XRYFileParser { if (segmentedText.length() > 0) { segmentedText.setLength(segmentedText.length() - 1); } + return segmentedText.toString(); } - + /** - * Determines if the line has recognized key value on it. - * - * @param xryLine - * @return - */ - private boolean hasKey(String xryLine) { - int delimiter = xryLine.indexOf(':'); - if(delimiter == -1) { - return false; - } - - String key = xryLine.substring(0, delimiter); - return XryKey.contains(key); - } - - /** - * Determines if the line is a recognized namespace. - * - * @param xryLine - * @return - */ - private boolean hasNamespace(String xryLine) { - return XryNamespace.contains(xryLine); - } - - /** - * Extracts meta keys from the XRY entity. All of the known meta - * keys are assumed integers and part of the XRY_META_KEY enum. * - * @param xryLines Current XRY entity - * @param expectedKey The meta key to search for - * @return The interpreted integer value or Integer.MIN_VALUE if - * no meta key was found. + * @param xryLines + * @param metaKey + * @return */ - private Optional getMetaInfo(String[] xryLines, XryMetaKey metaKey) { - for (int i = 0; i < xryLines.length; i++) { - String xryLine = xryLines[i]; + private Optional getMetaKeyValue(String[] xryLines, XryMetaKey metaKey) { + for (String xryLine : xryLines) { + if (!XRYKeyValuePair.isPair(xryLine)) { + continue; + } + + XRYKeyValuePair pair = XRYKeyValuePair.from(xryLine); + if(pair.hasKey(metaKey.getDisplayName())) { + try { + return Optional.of(Integer.parseInt(pair.getValue())); + } catch (NumberFormatException ex) { + logger.log(Level.SEVERE, String.format("[XRY DSP] Value [ %s ] for " + + "meta key [ %s ] was not an integer.", pair.getValue(), metaKey), ex); + } + } + } + return Optional.empty(); + } - int firstDelimiter = xryLine.indexOf(KEY_VALUE_DELIMITER); - if (firstDelimiter != -1) { - String key = xryLine.substring(0, firstDelimiter); - if(!XryMetaKey.contains(key)) { - continue; - } - - XryMetaKey currentMetaKey = XryMetaKey.fromDisplayName(key); - if (currentMetaKey.equals(metaKey)) { - String value = xryLine.substring(firstDelimiter + 1).trim(); - try { - return Optional.of(Integer.parseInt(value)); - } catch (NumberFormatException ex) { - logger.log(Level.SEVERE, String.format("[XRY DSP] Value [ %s ] for " - + "meta key [ %s ] was not an integer.", value, metaKey), ex); - } - } + /** + * + * @param xryLines + * @param index + * @return + */ + private Optional getKeyValuePairByIndex(String[] xryLines, int index) { + int pairsParsed = 0; + String namespace = ""; + for (int i = 1; i < xryLines.length; i++) { + String xryLine = xryLines[i]; + if(XryNamespace.contains(xryLine)) { + namespace = xryLine.trim(); + continue; + } + + if(!XRYKeyValuePair.isPair(xryLine)) { + logger.log(Level.SEVERE, String.format("[XRY DSP] Expected a key value " + + "pair on this line (in brackets) [ %s ], but one was not detected." + + " Discarding...", xryLine)); + continue; + } + + XRYKeyValuePair pair = XRYKeyValuePair.from(xryLine); + String value = pair.getValue(); + //Build up multiple lines. + for (; (i+1) < xryLines.length + && !XRYKeyValuePair.isPair(xryLines[i+1]) + && !XryNamespace.contains(xryLines[i+1]); i++) { + String continuedValue = xryLines[i+1].trim(); + //Assume multi lined values are split by word. + value = value + " " + continuedValue; + } + + pair = new XRYKeyValuePair(pair.getKey(), value, namespace); + pairsParsed++; + if(pairsParsed == index) { + return Optional.of(pair); } } @@ -523,47 +540,52 @@ final class XRYMessagesFileParser implements XRYFileParser { * @param value The value associated with that key. * @return Corresponding blackboard attribute, if any. */ - private Optional makeAttribute(XryNamespace namespace, XryKey key, String value) { - String normalizedValue = value.toLowerCase().trim(); + private Optional getBlackboardAttribute(XRYKeyValuePair pair) { + XryNamespace namespace = XryNamespace.NONE; + if(XryNamespace.contains(pair.getNamespace())) { + namespace = XryNamespace.fromDisplayName(pair.getNamespace()); + } + XryKey key = XryKey.fromDisplayName(pair.getKey()); + String normalizedValue = pair.getValue().toLowerCase().trim(); + switch (key) { - case DIRECTION: - return Optional.of(new BlackboardAttribute( - BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DIRECTION, - PARSER_NAME, value)); - case NAME_MATCHED: - return Optional.of(new BlackboardAttribute( - BlackboardAttribute.ATTRIBUTE_TYPE.TSK_NAME_PERSON, - PARSER_NAME, value)); case TEL: - if(namespace.equals(XryNamespace.FROM)) { - return Optional.of(new BlackboardAttribute( + case NUMBER: + switch (namespace) { + case FROM: + return Optional.of(new BlackboardAttribute( BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_FROM, - PARSER_NAME, value)); - } else { - //Assume TO and PARTICIPANT are TSK_PHONE_NUMBER_TOs - return Optional.of(new BlackboardAttribute( + PARSER_NAME, pair.getValue())); + case TO: + case PARTICIPANT: + return Optional.of(new BlackboardAttribute( BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_TO, - PARSER_NAME, value)); - } - case TEXT: - return Optional.of(new BlackboardAttribute( - BlackboardAttribute.ATTRIBUTE_TYPE.TSK_TEXT, - PARSER_NAME, value)); + PARSER_NAME, pair.getValue())); + default: + return Optional.of(new BlackboardAttribute( + BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER, + PARSER_NAME, pair.getValue())); + } case TIME: try { //Tranform value to seconds since epoch - long dateTimeSinceInEpoch = calculateSecondsSinceEpoch(value); + long dateTimeSinceInEpoch = calculateSecondsSinceEpoch(pair.getValue()); return Optional.of(new BlackboardAttribute( BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME_START, PARSER_NAME, dateTimeSinceInEpoch)); } catch (DateTimeParseException ex) { logger.log(Level.WARNING, String.format("[XRY DSP] Assumption" + " about the date time formatting of messages is " - + "not right. Here is the value [ %s ]", value), ex); + + "not right. Here is the pair [ %s ]", pair), ex); return Optional.empty(); } case TYPE: switch (normalizedValue) { + case "incoming": + case "outgoing": + return Optional.of(new BlackboardAttribute( + BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DIRECTION, + PARSER_NAME, pair.getValue())); case "deliver": case "submit": case "status report": @@ -571,13 +593,9 @@ final class XRYMessagesFileParser implements XRYFileParser { return Optional.empty(); default: logger.log(Level.WARNING, String.format("[XRY DSP] Unrecognized " - + "type value [ %s ]", value)); + + " value for key pair [ %s ].", pair)); return Optional.empty(); } - case SERVICE_CENTER: - return Optional.of(new BlackboardAttribute( - BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER, - PARSER_NAME, value)); case STATUS: switch (normalizedValue) { case "read": @@ -596,18 +614,22 @@ final class XRYMessagesFileParser implements XRYFileParser { return Optional.empty(); default: logger.log(Level.WARNING, String.format("[XRY DSP] Unrecognized " - + "status value [ %s ].", value)); + + " value for key pair [ %s ].", pair)); return Optional.empty(); } - case STORAGE: - case INDEX: - case FOLDER: - case NAME: - //Ignore for now. - return Optional.empty(); default: - throw new IllegalArgumentException(String.format("Key [ %s ] " - + "passed the isKey() test but was not matched.", key)); + //Otherwise, the XryKey enum contains the correct BlackboardAttribute + //type. + if(key.getType() != null) { + return Optional.of(new BlackboardAttribute(key.getType(), + PARSER_NAME, pair.getValue())); + } + + logger.log(Level.WARNING, String.format("[XRY DSP] Key value pair " + + "(in brackets) [ %s ] was recognized but " + + "more data or time is needed to finish implementation. Discarding... ", pair)); + + return Optional.empty(); } } @@ -620,12 +642,16 @@ final class XRYMessagesFileParser implements XRYFileParser { * @return A purer date time value. */ private String removeDateTimeLocale(String dateTime) { - int index = dateTime.indexOf('('); - if (index == -1) { - return dateTime; + String result = dateTime; + int deviceIndex = result.toLowerCase().indexOf(DEVICE_LOCALE); + if (deviceIndex != -1) { + result = result.substring(0, deviceIndex); } - - return dateTime.substring(0, index); + int networkIndex = result.toLowerCase().indexOf(NETWORK_LOCALE); + if(networkIndex != -1) { + result = result.substring(0, networkIndex); + } + return result; } /** @@ -662,8 +688,18 @@ final class XRYMessagesFileParser implements XRYFileParser { * will only introduce a trivial amount of error. */ String reversedDateTimeWithGMT = reversedDateTime.replace("UTC", "GMT"); - ZonedDateTime zonedDateTime = ZonedDateTime.parse(reversedDateTimeWithGMT, DATE_TIME_PARSER); - return zonedDateTime.toEpochSecond(); + TemporalAccessor result = DATE_TIME_PARSER.parseBest(reversedDateTimeWithGMT, + ZonedDateTime::from, + LocalDateTime::from, + OffsetDateTime::from); + //Query for the ZoneID + if(result.query(TemporalQueries.zoneId()) == null) { + //If none, assumed GMT+0. + return ZonedDateTime.of(LocalDateTime.from(result), + ZoneId.of("GMT")).toEpochSecond(); + } else { + return Instant.from(result).getEpochSecond(); + } } /** diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYWebBookmarksFileParser.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYWebBookmarksFileParser.java index 6f21048230..d4edc3b2da 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYWebBookmarksFileParser.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYWebBookmarksFileParser.java @@ -42,8 +42,8 @@ final class XRYWebBookmarksFileParser extends AbstractSingleKeyValueParser { }; @Override - boolean isKey(String key) { - String normalizedKey = key.toLowerCase(); + boolean canProcess(XRYKeyValuePair pair) { + String normalizedKey = pair.getKey().toLowerCase(); return KEY_TO_TYPE.containsKey(normalizedKey); } @@ -54,11 +54,11 @@ final class XRYWebBookmarksFileParser extends AbstractSingleKeyValueParser { } @Override - Optional makeAttribute(String nameSpace, String key, String value) { - String normalizedKey = key.toLowerCase(); + Optional getBlackboardAttribute(String nameSpace, XRYKeyValuePair pair) { + String normalizedKey = pair.getKey().toLowerCase(); return Optional.of(new BlackboardAttribute( KEY_TO_TYPE.get(normalizedKey), - PARSER_NAME, value)); + PARSER_NAME, pair.getValue())); } @Override From d81c2bc22d64dd84333fa71e5191eb1b50bb7a0d Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Fri, 13 Dec 2019 15:46:24 -0500 Subject: [PATCH 10/22] Added comments and general code clean up --- .../xry/XRYCallsFileParser.java | 151 ++++---- .../xry/XRYMessagesFileParser.java | 341 +++++++++--------- 2 files changed, 244 insertions(+), 248 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java index fde9b397aa..fd70a1da17 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java @@ -47,13 +47,13 @@ final class XRYCallsFileParser extends AbstractSingleKeyValueParser { //function for more details. private static final DateTimeFormatter DATE_TIME_PARSER = DateTimeFormatter.ofPattern("[(XXX) ][O ][(O) ]a h:m:s M/d/y"); - + private static final String DEVICE_LOCALE = "(device)"; private static final String NETWORK_LOCALE = "(network)"; - + /** - * All of the known XRY keys for call reports and the blackboard - * attribute types they map to. + * All of the known XRY keys for call reports and their corresponding + * blackboard attribute types, if any. */ private enum XryKey { NUMBER("number", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER), @@ -66,24 +66,24 @@ final class XRYCallsFileParser extends AbstractSingleKeyValueParser { STORAGE("storage", null), INDEX("index", null), NAME("name", null); - + private final String name; private final BlackboardAttribute.ATTRIBUTE_TYPE type; - + XryKey(String name, BlackboardAttribute.ATTRIBUTE_TYPE type) { this.name = name; this.type = type; } - + public BlackboardAttribute.ATTRIBUTE_TYPE getType() { return type; } - + /** * Indicates if the display name of the XRY key is a recognized type. - * + * * @param xryKey - * @return + * @return */ public static boolean contains(XRYKeyValuePair pair) { try { @@ -93,24 +93,24 @@ final class XRYCallsFileParser extends AbstractSingleKeyValueParser { return false; } } - + /** * Matches the display name of the xry key to the appropriate enum type. - * - * It is assumed that XRY key string is recognized. Otherwise, - * an IllegalArgumentException is thrown. Test all membership - * with contains() before hand. - * + * + * It is assumed that XRY key string is recognized. Otherwise, an + * IllegalArgumentException is thrown. Test all membership with + * contains() before hand. + * * @param xryKey - * @return + * @return */ public static XryKey fromPair(XRYKeyValuePair pair) { - for(XryKey keyChoice : XryKey.values()) { - if(pair.hasKey(keyChoice.name)) { + for (XryKey keyChoice : XryKey.values()) { + if (pair.hasKey(keyChoice.name)) { return keyChoice; } } - + throw new IllegalArgumentException(String.format("Key [%s] was not found." + " All keys should be tested with contains.", pair.getKey())); } @@ -123,47 +123,50 @@ final class XRYCallsFileParser extends AbstractSingleKeyValueParser { TO("to"), FROM("from"), NONE(null); - + private final String name; + XryNamespace(String name) { this.name = name; } - + /** - * Indicates if the display name of the XRY namespace is a recognized type. - * + * Indicates if the display name of the XRY namespace is a recognized + * type. + * * @param xryNamespace - * @return + * @return */ public static boolean contains(String xryNamespace) { String normalizedNamespace = xryNamespace.trim().toLowerCase(); - for(XryNamespace keyChoice : XryNamespace.values()) { - if(normalizedNamespace.equals(keyChoice.name)) { + for (XryNamespace keyChoice : XryNamespace.values()) { + if (normalizedNamespace.equals(keyChoice.name)) { return true; } } - + return false; } - + /** - * Matches the display name of the xry namespace to the appropriate enum type. - * - * It is assumed that XRY namespace string is recognized. Otherwise, - * an IllegalArgumentException is thrown. Test all membership - * with contains() before hand. - * + * Matches the display name of the xry namespace to the appropriate enum + * type. + * + * It is assumed that XRY namespace string is recognized. Otherwise, an + * IllegalArgumentException is thrown. Test all membership with + * contains() before hand. + * * @param xryNamespace - * @return + * @return */ public static XryNamespace fromDisplayName(String xryNamespace) { String normalizedNamespace = xryNamespace.trim().toLowerCase(); - for(XryNamespace keyChoice : XryNamespace.values()) { - if(normalizedNamespace.equals(keyChoice.name)) { + for (XryNamespace keyChoice : XryNamespace.values()) { + if (normalizedNamespace.equals(keyChoice.name)) { return keyChoice; } } - + throw new IllegalArgumentException(String.format("Key [%s] was not found." + " All keys should be tested with contains.", xryNamespace)); } @@ -183,7 +186,7 @@ final class XRYCallsFileParser extends AbstractSingleKeyValueParser { Optional getBlackboardAttribute(String nameSpace, XRYKeyValuePair pair) { XryKey xryKey = XryKey.fromPair(pair); XryNamespace xryNamespace = XryNamespace.NONE; - if(XryNamespace.contains(nameSpace)) { + if (XryNamespace.contains(nameSpace)) { xryNamespace = XryNamespace.fromDisplayName(nameSpace); } @@ -208,7 +211,7 @@ final class XRYCallsFileParser extends AbstractSingleKeyValueParser { try { //Tranform value to seconds since epoch long dateTimeSinceEpoch = calculateSecondsSinceEpoch(pair.getValue()); - return Optional.of(new BlackboardAttribute(xryKey.getType(), + return Optional.of(new BlackboardAttribute(xryKey.getType(), PARSER_NAME, dateTimeSinceEpoch)); } catch (DateTimeParseException ex) { logger.log(Level.WARNING, String.format("[XRY DSP] Assumption" @@ -219,15 +222,15 @@ final class XRYCallsFileParser extends AbstractSingleKeyValueParser { default: //Otherwise, the XryKey enum contains the correct BlackboardAttribute //type. - if(xryKey.getType() != null) { - return Optional.of(new BlackboardAttribute(xryKey.getType(), - PARSER_NAME, pair.getValue())); + if (xryKey.getType() != null) { + return Optional.of(new BlackboardAttribute(xryKey.getType(), + PARSER_NAME, pair.getValue())); } - + logger.log(Level.WARNING, String.format("[XRY DSP] Key value pair " - + "(in brackets) [ %s ] was recognized but " - + "more data or time is needed to finish implementation. Discarding... ", - pair)); + + "(in brackets) [ %s ] was recognized but " + + "more data or time is needed to finish implementation. Discarding... ", + pair)); return Optional.empty(); } } @@ -253,7 +256,7 @@ final class XRYCallsFileParser extends AbstractSingleKeyValueParser { result = result.substring(0, deviceIndex); } int networkIndex = result.toLowerCase().indexOf(NETWORK_LOCALE); - if(networkIndex != -1) { + if (networkIndex != -1) { result = result.substring(0, networkIndex); } return result; @@ -269,28 +272,27 @@ final class XRYCallsFileParser extends AbstractSingleKeyValueParser { String dateTimeWithoutLocale = removeDateTimeLocale(dateTime).trim(); /** * The format of time in XRY Messages reports is of the form: - * - * 1/3/1990 1:23:54 AM UTC+4 - * - * In our current version of Java (openjdk-1.8.0.222), there is - * a bug with having the timezone offset (UTC+4 or GMT-7) at the - * end of the date time input. This is fixed in later versions - * of the JDK (9 and beyond). - * https://bugs.openjdk.java.net/browse/JDK-8154050 - * Rather than update the JDK to accommodate this, the components of - * the date time string are reversed: - * - * UTC+4 AM 1:23:54 1/3/1990 - * + * + * 1/3/1990 1:23:54 AM UTC+4 + * + * In our current version of Java (openjdk-1.8.0.222), there is a bug + * with having the timezone offset (UTC+4 or GMT-7) at the end of the + * date time input. This is fixed in later versions of the JDK (9 and + * beyond). https://bugs.openjdk.java.net/browse/JDK-8154050 Rather than + * update the JDK to accommodate this, the components of the date time + * string are reversed: + * + * UTC+4 AM 1:23:54 1/3/1990 + * * The java time package will correctly parse this date time format. */ String reversedDateTime = reverseOrderOfDateTimeComponents(dateTimeWithoutLocale); /** - * Furthermore, the DateTimeFormatter's timezone offset letter ('O') does - * not recognize UTC but recognizes GMT. According to - * https://en.wikipedia.org/wiki/Coordinated_Universal_Time, - * GMT only differs from UTC by at most 1 second and so substitution - * will only introduce a trivial amount of error. + * Furthermore, the DateTimeFormatter's timezone offset letter ('O') + * does not recognize UTC but recognizes GMT. According to + * https://en.wikipedia.org/wiki/Coordinated_Universal_Time, GMT only + * differs from UTC by at most 1 second and so substitution will only + * introduce a trivial amount of error. */ String reversedDateTimeWithGMT = reversedDateTime.replace("UTC", "GMT"); TemporalAccessor result = DATE_TIME_PARSER.parseBest(reversedDateTimeWithGMT, @@ -298,23 +300,20 @@ final class XRYCallsFileParser extends AbstractSingleKeyValueParser { LocalDateTime::from, OffsetDateTime::from); //Query for the ZoneID - if(result.query(TemporalQueries.zoneId()) == null) { + if (result.query(TemporalQueries.zoneId()) == null) { //If none, assumed GMT+0. - return ZonedDateTime.of(LocalDateTime.from(result), + return ZonedDateTime.of(LocalDateTime.from(result), ZoneId.of("GMT")).toEpochSecond(); } else { return Instant.from(result).getEpochSecond(); } } - + /** * Reverses the order of the date time components. - * - * Example: - * 1/3/1990 1:23:54 AM UTC+4 - * becomes - * UTC+4 AM 1:23:54 1/3/1990 - * + * + * Example: 1/3/1990 1:23:54 AM UTC+4 becomes UTC+4 AM 1:23:54 1/3/1990 + * * @param dateTime * @return */ diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYMessagesFileParser.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYMessagesFileParser.java index 37c2ecacc9..ed29c178a6 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYMessagesFileParser.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYMessagesFileParser.java @@ -49,23 +49,23 @@ final class XRYMessagesFileParser implements XRYFileParser { private static final Logger logger = Logger.getLogger( XRYMessagesFileParser.class.getName()); - + private static final String PARSER_NAME = "XRY DSP"; - + //Pattern is in reverse due to a Java 8 bug, see calculateSecondsSinceEpoch() //function for more details. private static final DateTimeFormatter DATE_TIME_PARSER = DateTimeFormatter.ofPattern("[(XXX) ][O ][(O) ]a h:m:s M/d/y"); - + private static final String DEVICE_LOCALE = "(device)"; private static final String NETWORK_LOCALE = "(network)"; - + private static final int READ = 1; private static final int UNREAD = 0; - + /** - * All of the known XRY keys for message reports and the blackboard - * attribute types they map to. + * All of the known XRY keys for message reports and their corresponding + * blackboard attribute types, if any. */ private enum XryKey { DELETED("deleted", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_ISDELETED), @@ -86,28 +86,28 @@ final class XRYMessagesFileParser implements XRYFileParser { NAME("name", null), INDEX("index", null), STATUS("status", null); - + private final String name; private final BlackboardAttribute.ATTRIBUTE_TYPE type; - + XryKey(String name, BlackboardAttribute.ATTRIBUTE_TYPE type) { this.name = name; this.type = type; } - + public BlackboardAttribute.ATTRIBUTE_TYPE getType() { return type; } - + public String getDisplayName() { return name; } - + /** * Indicates if the display name of the XRY key is a recognized type. - * + * * @param xryKey - * @return + * @return */ public static boolean contains(String name) { try { @@ -117,30 +117,30 @@ final class XRYMessagesFileParser implements XRYFileParser { return false; } } - + /** * Matches the display name of the xry key to the appropriate enum type. - * - * It is assumed that XRY key string is recognized. Otherwise, - * an IllegalArgumentException is thrown. Test all membership - * with contains() before hand. - * + * + * It is assumed that XRY key string is recognized. Otherwise, an + * IllegalArgumentException is thrown. Test all membership with + * contains() before hand. + * * @param xryKey - * @return + * @return */ public static XryKey fromDisplayName(String name) { String normalizedName = name.trim().toLowerCase(); - for(XryKey keyChoice : XryKey.values()) { - if(normalizedName.equals(keyChoice.name)) { + for (XryKey keyChoice : XryKey.values()) { + if (normalizedName.equals(keyChoice.name)) { return keyChoice; } } - + throw new IllegalArgumentException(String.format("Key [ %s ] was not found." + " All keys should be tested with contains.", name)); } } - + /** * All of the known XRY namespaces for message reports. */ @@ -149,18 +149,19 @@ final class XRYMessagesFileParser implements XRYFileParser { PARTICIPANT("participant"), TO("to"), NONE(null); - + private final String name; - + XryNamespace(String name) { this.name = name; } - + /** - * Indicates if the display name of the XRY namespace is a recognized type. - * + * Indicates if the display name of the XRY namespace is a recognized + * type. + * * @param xryNamespace - * @return + * @return */ public static boolean contains(String xryNamespace) { try { @@ -170,30 +171,31 @@ final class XRYMessagesFileParser implements XRYFileParser { return false; } } - + /** - * Matches the display name of the xry namespace to the appropriate enum type. - * - * It is assumed that XRY namespace string is recognized. Otherwise, - * an IllegalArgumentException is thrown. Test all membership - * with contains() before hand. - * + * Matches the display name of the xry namespace to the appropriate enum + * type. + * + * It is assumed that XRY namespace string is recognized. Otherwise, an + * IllegalArgumentException is thrown. Test all membership with + * contains() before hand. + * * @param xryNamespace - * @return + * @return */ public static XryNamespace fromDisplayName(String xryNamespace) { String normalizedNamespace = xryNamespace.trim().toLowerCase(); - for(XryNamespace keyChoice : XryNamespace.values()) { - if(normalizedNamespace.equals(keyChoice.name)) { + for (XryNamespace keyChoice : XryNamespace.values()) { + if (normalizedNamespace.equals(keyChoice.name)) { return keyChoice; } } - + throw new IllegalArgumentException(String.format("Namespace [%s] was not found." + " All namespaces should be tested with contains.", xryNamespace)); } } - + /** * All known XRY meta keys for message reports. */ @@ -201,23 +203,22 @@ final class XRYMessagesFileParser implements XRYFileParser { REFERENCE_NUMBER("reference number"), SEGMENT_COUNT("segments"), SEGMENT_NUMBER("segment number"); - + private final String name; - + XryMetaKey(String name) { this.name = name; } - + public String getDisplayName() { return name; } - - + /** * Indicates if the display name of the XRY key is a recognized type. - * + * * @param xryKey - * @return + * @return */ public static boolean contains(String name) { try { @@ -227,25 +228,25 @@ final class XRYMessagesFileParser implements XRYFileParser { return false; } } - + /** * Matches the display name of the xry key to the appropriate enum type. - * - * It is assumed that XRY key string is recognized. Otherwise, - * an IllegalArgumentException is thrown. Test all membership - * with contains() before hand. - * + * + * It is assumed that XRY key string is recognized. Otherwise, an + * IllegalArgumentException is thrown. Test all membership with + * contains() before hand. + * * @param xryKey - * @return + * @return */ public static XryMetaKey fromDisplayName(String name) { String normalizedName = name.trim().toLowerCase(); - for(XryMetaKey keyChoice : XryMetaKey.values()) { - if(normalizedName.equals(keyChoice.name)) { + for (XryMetaKey keyChoice : XryMetaKey.values()) { + if (normalizedName.equals(keyChoice.name)) { return keyChoice; } } - + throw new IllegalArgumentException(String.format("Key [ %s ] was not found." + " All keys should be tested with contains.", name)); } @@ -253,11 +254,11 @@ final class XRYMessagesFileParser implements XRYFileParser { /** * Message-SMS report artifacts can span multiple XRY entities and their - * attributes can span multiple lines. The "Text" and "Message" keys are the only known key - * value pair that can span multiple lines. Messages can be segmented, - * meaning that their "Text" and "Message" content can appear in multiple XRY entities. - * Our goal for a segmented message is to aggregate all of the text pieces and - * create 1 artifact. + * attributes can span multiple lines. The "Text" and "Message" keys are the + * only known key value pair that can span multiple lines. Messages can be + * segmented, meaning that their "Text" and "Message" content can appear in + * multiple XRY entities. Our goal for a segmented message is to aggregate + * all of the text pieces and create 1 artifact. * * This parse implementation assumes that segments are contiguous and that * they ascend incrementally. There are checks in place to verify this @@ -283,31 +284,27 @@ final class XRYMessagesFileParser implements XRYFileParser { String xryEntity = reader.nextEntity(); List attributes = getBlackboardAttributes(xryEntity, reader, referenceNumbersSeen); //Only create artifacts with non-empty attributes. - if(!attributes.isEmpty()) { + if (!attributes.isEmpty()) { BlackboardArtifact artifact = parent.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_MESSAGE); artifact.addAttributes(attributes); } } } - + /** - * - * @param xryEntity - * @param reader - * @param referenceValues - * @return - * @throws IOException + * Extracts all blackboard attributes from the XRY Entity. This function will + * unify any segmented text, if need be. */ - private List getBlackboardAttributes(String xryEntity, + private List getBlackboardAttributes(String xryEntity, XRYFileReader reader, Set referenceValues) throws IOException { String[] xryLines = xryEntity.split("\n"); //First line of the entity is the title, each XRY entity is non-empty. logger.log(Level.INFO, String.format("[XRY DSP] Processing [ %s ]", xryLines[0])); - + List attributes = new ArrayList<>(); - + //Count the key value pairs in the XRY entity. - int keyCount = countKeys(xryLines); + int keyCount = getCountOfKeyValuePairs(xryLines); for (int i = 1; i <= keyCount; i++) { //Get the ith key value pair in the entity. Always expect to have //a valid value. @@ -333,10 +330,10 @@ final class XRYMessagesFileParser implements XRYFileParser { //Assume text and message are the only fields that can be segmented //among multiple XRY entities. - if (pair.hasKey(XryKey.TEXT.getDisplayName()) || - pair.hasKey(XryKey.MESSAGE.getDisplayName())) { - String segmentedText = getSegmentedText(xryLines, reader, referenceValues); - pair = new XRYKeyValuePair(pair.getKey(), + if (pair.hasKey(XryKey.TEXT.getDisplayName()) + || pair.hasKey(XryKey.MESSAGE.getDisplayName())) { + String segmentedText = getSegmentedText(xryLines, reader, referenceValues); + pair = new XRYKeyValuePair(pair.getKey(), //Assume text is segmented by word. pair.getValue() + " " + segmentedText, pair.getNamespace()); @@ -348,18 +345,18 @@ final class XRYMessagesFileParser implements XRYFileParser { attributes.add(attribute.get()); } } - + return attributes; } - + /** - * Counts the key value pairs in an XRY entity. - * Skips counting the first line as it is assumed to be the title. + * Counts the key value pairs in an XRY entity. Skips counting the first + * line as it is assumed to be the title. */ - private Integer countKeys(String[] xryEntity) { + private Integer getCountOfKeyValuePairs(String[] xryEntity) { int count = 0; for (int i = 1; i < xryEntity.length; i++) { - if(XRYKeyValuePair.isPair(xryEntity[i])) { + if (XRYKeyValuePair.isPair(xryEntity[i])) { count++; } } @@ -367,23 +364,23 @@ final class XRYMessagesFileParser implements XRYFileParser { } /** - * Builds up segmented message entities so that the text is unified in the - * artifact. - * - * @param referenceNumber Reference number that messages are group by - * @param segmentNumber Segment number of the starting segment. - * @param reader + * Builds up segmented message entities so that the text is unified for a + * single artifact. + * + * @param reader File reader that is producing XRY entities. + * @param referenceNumbersSeen All known references numbers up until this point. + * @param xryEntity The source XRY entity. * @return * @throws IOException */ - private String getSegmentedText(String[] xryEntity, XRYFileReader reader, + private String getSegmentedText(String[] xryEntity, XRYFileReader reader, Set referenceNumbersSeen) throws IOException { Optional referenceNumber = getMetaKeyValue(xryEntity, XryMetaKey.REFERENCE_NUMBER); //Check if there is any segmented text. - if(!referenceNumber.isPresent()) { + if (!referenceNumber.isPresent()) { return ""; } - + logger.log(Level.INFO, String.format("[XRY DSP] Message entity " + "appears to be segmented with reference number [ %d ]", referenceNumber.get())); @@ -398,13 +395,13 @@ final class XRYMessagesFileParser implements XRYFileParser { referenceNumbersSeen.add(referenceNumber.get()); Optional segmentNumber = getMetaKeyValue(xryEntity, XryMetaKey.SEGMENT_NUMBER); - if(!segmentNumber.isPresent()) { + if (!segmentNumber.isPresent()) { logger.log(Level.SEVERE, String.format("No segment " + "number was found on the message entity" + "with reference number [%d]", referenceNumber.get())); return ""; } - + StringBuilder segmentedText = new StringBuilder(); int currentSegmentNumber = segmentNumber.get(); @@ -414,8 +411,8 @@ final class XRYMessagesFileParser implements XRYFileParser { String[] nextEntityLines = nextEntity.split("\n"); Optional nextReferenceNumber = getMetaKeyValue(nextEntityLines, XryMetaKey.REFERENCE_NUMBER); - if (!nextReferenceNumber.isPresent() || - !Objects.equals(nextReferenceNumber, referenceNumber)) { + if (!nextReferenceNumber.isPresent() + || !Objects.equals(nextReferenceNumber, referenceNumber)) { //Don't consume the next entity. It is not related //to the current message thread. break; @@ -429,7 +426,7 @@ final class XRYMessagesFileParser implements XRYFileParser { logger.log(Level.INFO, String.format("[XRY DSP] Processing [ %s ] " + "segment with reference number [ %d ]", nextEntityLines[0], referenceNumber.get())); - if(!nextSegmentNumber.isPresent()) { + if (!nextSegmentNumber.isPresent()) { logger.log(Level.SEVERE, String.format("[XRY DSP] Segment with reference" + " number [ %d ] did not have a segment number associated with it." + " It cannot be determined if the reconstructed text will be in order.", referenceNumber.get())); @@ -440,16 +437,16 @@ final class XRYMessagesFileParser implements XRYFileParser { + "text will be out of order.", nextSegmentNumber.get(), currentSegmentNumber)); } - int keyCount = countKeys(nextEntityLines); + int keyCount = getCountOfKeyValuePairs(nextEntityLines); for (int i = 1; i <= keyCount; i++) { XRYKeyValuePair pair = getKeyValuePairByIndex(nextEntityLines, i).get(); - if(pair.hasKey(XryKey.TEXT.getDisplayName()) || - pair.hasKey(XryKey.MESSAGE.getDisplayName())) { + if (pair.hasKey(XryKey.TEXT.getDisplayName()) + || pair.hasKey(XryKey.MESSAGE.getDisplayName())) { segmentedText.append(pair.getValue()).append(' '); } } - if(nextSegmentNumber.isPresent()) { + if (nextSegmentNumber.isPresent()) { currentSegmentNumber = nextSegmentNumber.get(); } } @@ -458,24 +455,25 @@ final class XRYMessagesFileParser implements XRYFileParser { if (segmentedText.length() > 0) { segmentedText.setLength(segmentedText.length() - 1); } - + return segmentedText.toString(); } - + /** + * Extracts the value of the XRY meta key, if any. * - * @param xryLines - * @param metaKey - * @return + * @param xryLines XRY entity to extract from. + * @param metaKey The key type to extract. + * @return */ private Optional getMetaKeyValue(String[] xryLines, XryMetaKey metaKey) { for (String xryLine : xryLines) { if (!XRYKeyValuePair.isPair(xryLine)) { continue; } - + XRYKeyValuePair pair = XRYKeyValuePair.from(xryLine); - if(pair.hasKey(metaKey.getDisplayName())) { + if (pair.hasKey(metaKey.getDisplayName())) { try { return Optional.of(Integer.parseInt(pair.getValue())); } catch (NumberFormatException ex) { @@ -488,42 +486,45 @@ final class XRYMessagesFileParser implements XRYFileParser { } /** + * Extracts the ith XRY Key Value pair in the XRY Entity. * - * @param xryLines - * @param index - * @return + * The total number of pairs can be determined via getCountOfKeyValuePairs(). + * + * @param xryLines XRY entity. + * @param index The requested Key Value pair. + * @return */ private Optional getKeyValuePairByIndex(String[] xryLines, int index) { int pairsParsed = 0; String namespace = ""; for (int i = 1; i < xryLines.length; i++) { String xryLine = xryLines[i]; - if(XryNamespace.contains(xryLine)) { + if (XryNamespace.contains(xryLine)) { namespace = xryLine.trim(); continue; } - - if(!XRYKeyValuePair.isPair(xryLine)) { + + if (!XRYKeyValuePair.isPair(xryLine)) { logger.log(Level.SEVERE, String.format("[XRY DSP] Expected a key value " - + "pair on this line (in brackets) [ %s ], but one was not detected." - + " Discarding...", xryLine)); + + "pair on this line (in brackets) [ %s ], but one was not detected." + + " Discarding...", xryLine)); continue; } - + XRYKeyValuePair pair = XRYKeyValuePair.from(xryLine); String value = pair.getValue(); //Build up multiple lines. - for (; (i+1) < xryLines.length - && !XRYKeyValuePair.isPair(xryLines[i+1]) - && !XryNamespace.contains(xryLines[i+1]); i++) { - String continuedValue = xryLines[i+1].trim(); + for (; (i + 1) < xryLines.length + && !XRYKeyValuePair.isPair(xryLines[i + 1]) + && !XryNamespace.contains(xryLines[i + 1]); i++) { + String continuedValue = xryLines[i + 1].trim(); //Assume multi lined values are split by word. value = value + " " + continuedValue; } pair = new XRYKeyValuePair(pair.getKey(), value, namespace); pairsParsed++; - if(pairsParsed == index) { + if (pairsParsed == index) { return Optional.of(pair); } } @@ -533,21 +534,21 @@ final class XRYMessagesFileParser implements XRYFileParser { /** * Creates an attribute from the extracted key value pair. - * - * @param nameSpace The namespace of this key value pair. - * It will have been verified beforehand, otherwise it will be NONE. + * + * @param nameSpace The namespace of this key value pair. It will have been + * verified beforehand, otherwise it will be NONE. * @param key The recognized XRY key. * @param value The value associated with that key. * @return Corresponding blackboard attribute, if any. */ private Optional getBlackboardAttribute(XRYKeyValuePair pair) { XryNamespace namespace = XryNamespace.NONE; - if(XryNamespace.contains(pair.getNamespace())) { + if (XryNamespace.contains(pair.getNamespace())) { namespace = XryNamespace.fromDisplayName(pair.getNamespace()); } XryKey key = XryKey.fromDisplayName(pair.getKey()); String normalizedValue = pair.getValue().toLowerCase().trim(); - + switch (key) { case TEL: case NUMBER: @@ -565,13 +566,13 @@ final class XRYMessagesFileParser implements XRYFileParser { return Optional.of(new BlackboardAttribute( BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER, PARSER_NAME, pair.getValue())); - } + } case TIME: try { //Tranform value to seconds since epoch long dateTimeSinceInEpoch = calculateSecondsSinceEpoch(pair.getValue()); return Optional.of(new BlackboardAttribute( - BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME_START, + BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME_START, PARSER_NAME, dateTimeSinceInEpoch)); } catch (DateTimeParseException ex) { logger.log(Level.WARNING, String.format("[XRY DSP] Assumption" @@ -584,8 +585,8 @@ final class XRYMessagesFileParser implements XRYFileParser { case "incoming": case "outgoing": return Optional.of(new BlackboardAttribute( - BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DIRECTION, - PARSER_NAME, pair.getValue())); + BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DIRECTION, + PARSER_NAME, pair.getValue())); case "deliver": case "submit": case "status report": @@ -600,11 +601,11 @@ final class XRYMessagesFileParser implements XRYFileParser { switch (normalizedValue) { case "read": return Optional.of(new BlackboardAttribute( - BlackboardAttribute.ATTRIBUTE_TYPE.TSK_READ_STATUS, + BlackboardAttribute.ATTRIBUTE_TYPE.TSK_READ_STATUS, PARSER_NAME, READ)); case "unread": return Optional.of(new BlackboardAttribute( - BlackboardAttribute.ATTRIBUTE_TYPE.TSK_READ_STATUS, + BlackboardAttribute.ATTRIBUTE_TYPE.TSK_READ_STATUS, PARSER_NAME, UNREAD)); case "sending failed": case "deleted": @@ -620,15 +621,15 @@ final class XRYMessagesFileParser implements XRYFileParser { default: //Otherwise, the XryKey enum contains the correct BlackboardAttribute //type. - if(key.getType() != null) { - return Optional.of(new BlackboardAttribute(key.getType(), - PARSER_NAME, pair.getValue())); + if (key.getType() != null) { + return Optional.of(new BlackboardAttribute(key.getType(), + PARSER_NAME, pair.getValue())); } - + logger.log(Level.WARNING, String.format("[XRY DSP] Key value pair " - + "(in brackets) [ %s ] was recognized but " - + "more data or time is needed to finish implementation. Discarding... ", pair)); - + + "(in brackets) [ %s ] was recognized but " + + "more data or time is needed to finish implementation. Discarding... ", pair)); + return Optional.empty(); } } @@ -648,7 +649,7 @@ final class XRYMessagesFileParser implements XRYFileParser { result = result.substring(0, deviceIndex); } int networkIndex = result.toLowerCase().indexOf(NETWORK_LOCALE); - if(networkIndex != -1) { + if (networkIndex != -1) { result = result.substring(0, networkIndex); } return result; @@ -664,28 +665,27 @@ final class XRYMessagesFileParser implements XRYFileParser { String dateTimeWithoutLocale = removeDateTimeLocale(dateTime).trim(); /** * The format of time in XRY Messages reports is of the form: - * - * 1/3/1990 1:23:54 AM UTC+4 - * - * In our current version of Java (openjdk-1.8.0.222), there is - * a bug with having the timezone offset (UTC+4 or GMT-7) at the - * end of the date time input. This is fixed in later versions - * of the JDK (9 and beyond). - * https://bugs.openjdk.java.net/browse/JDK-8154050 - * Rather than update the JDK to accommodate this, the components of - * the date time string are reversed: - * - * UTC+4 AM 1:23:54 1/3/1990 - * + * + * 1/3/1990 1:23:54 AM UTC+4 + * + * In our current version of Java (openjdk-1.8.0.222), there is a bug + * with having the timezone offset (UTC+4 or GMT-7) at the end of the + * date time input. This is fixed in later versions of the JDK (9 and + * beyond). https://bugs.openjdk.java.net/browse/JDK-8154050 Rather than + * update the JDK to accommodate this, the components of the date time + * string are reversed: + * + * UTC+4 AM 1:23:54 1/3/1990 + * * The java time package will correctly parse this date time format. */ String reversedDateTime = reverseOrderOfDateTimeComponents(dateTimeWithoutLocale); /** - * Furthermore, the DateTimeFormatter's timezone offset letter ('O') does - * not recognize UTC but recognizes GMT. According to - * https://en.wikipedia.org/wiki/Coordinated_Universal_Time, - * GMT only differs from UTC by at most 1 second and so substitution - * will only introduce a trivial amount of error. + * Furthermore, the DateTimeFormatter's timezone offset letter ('O') + * does not recognize UTC but recognizes GMT. According to + * https://en.wikipedia.org/wiki/Coordinated_Universal_Time, GMT only + * differs from UTC by at most 1 second and so substitution will only + * introduce a trivial amount of error. */ String reversedDateTimeWithGMT = reversedDateTime.replace("UTC", "GMT"); TemporalAccessor result = DATE_TIME_PARSER.parseBest(reversedDateTimeWithGMT, @@ -693,23 +693,20 @@ final class XRYMessagesFileParser implements XRYFileParser { LocalDateTime::from, OffsetDateTime::from); //Query for the ZoneID - if(result.query(TemporalQueries.zoneId()) == null) { + if (result.query(TemporalQueries.zoneId()) == null) { //If none, assumed GMT+0. - return ZonedDateTime.of(LocalDateTime.from(result), + return ZonedDateTime.of(LocalDateTime.from(result), ZoneId.of("GMT")).toEpochSecond(); } else { return Instant.from(result).getEpochSecond(); } } - + /** * Reverses the order of the date time components. - * - * Example: - * 1/3/1990 1:23:54 AM UTC+4 - * becomes - * UTC+4 AM 1:23:54 1/3/1990 - * + * + * Example: 1/3/1990 1:23:54 AM UTC+4 becomes UTC+4 AM 1:23:54 1/3/1990 + * * @param dateTime * @return */ From 224cf3c5bd8ad745550e6d564fb1d1ec37506dc3 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Fri, 13 Dec 2019 15:56:13 -0500 Subject: [PATCH 11/22] Filled in log messages and updated comments --- .../xry/XRYCallsFileParser.java | 14 +++++++------- .../xry/XRYDeviceGenInfoFileParser.java | 8 ++++++-- .../datasourceprocessors/xry/XRYKeyValuePair.java | 8 +++++++- 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java index fd70a1da17..6b01a665a5 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java @@ -85,9 +85,9 @@ final class XRYCallsFileParser extends AbstractSingleKeyValueParser { * @param xryKey * @return */ - public static boolean contains(XRYKeyValuePair pair) { + public static boolean contains(String key) { try { - XryKey.fromPair(pair); + XryKey.fromDisplayName(key); return true; } catch (IllegalArgumentException ex) { return false; @@ -104,15 +104,15 @@ final class XRYCallsFileParser extends AbstractSingleKeyValueParser { * @param xryKey * @return */ - public static XryKey fromPair(XRYKeyValuePair pair) { + public static XryKey fromDisplayName(String key) { for (XryKey keyChoice : XryKey.values()) { - if (pair.hasKey(keyChoice.name)) { + if (key.equals(keyChoice.name)) { return keyChoice; } } throw new IllegalArgumentException(String.format("Key [%s] was not found." - + " All keys should be tested with contains.", pair.getKey())); + + " All keys should be tested with contains.", key)); } } @@ -174,7 +174,7 @@ final class XRYCallsFileParser extends AbstractSingleKeyValueParser { @Override boolean canProcess(XRYKeyValuePair pair) { - return XryKey.contains(pair); + return XryKey.contains(pair.getKey()); } @Override @@ -184,7 +184,7 @@ final class XRYCallsFileParser extends AbstractSingleKeyValueParser { @Override Optional getBlackboardAttribute(String nameSpace, XRYKeyValuePair pair) { - XryKey xryKey = XryKey.fromPair(pair); + XryKey xryKey = XryKey.fromDisplayName(pair.getKey()); XryNamespace xryNamespace = XryNamespace.NONE; if (XryNamespace.contains(nameSpace)) { xryNamespace = XryNamespace.fromDisplayName(nameSpace); diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYDeviceGenInfoFileParser.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYDeviceGenInfoFileParser.java index 4f21b57595..d13e27ffc4 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYDeviceGenInfoFileParser.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYDeviceGenInfoFileParser.java @@ -137,7 +137,9 @@ final class XRYDeviceGenInfoFileParser implements XRYFileParser { //need two XRY Key Value pairs per blackboard attribute. for (int i = 1; i < xryLines.length; i += 2) { if (!XRYKeyValuePair.isPair(xryLines[i])) { - //log + logger.log(Level.WARNING, String.format("[XRY DSP] Expected a key value " + + "pair on this line (in brackets) [ %s ], but one was not detected." + + " Discarding...", xryLines[i])); continue; } @@ -149,7 +151,9 @@ final class XRYDeviceGenInfoFileParser implements XRYFileParser { XRYKeyValuePair secondPair = XRYKeyValuePair.from(xryLines[i + 1]); attribute = getBlackboardAttribute(firstPair, secondPair); } else { - //log + logger.log(Level.WARNING, String.format("[XRY DSP] Expected a key value " + + "pair on this line (in brackets) [ %s ], but one was not detected." + + " Discarding...", xryLines[i+1])); } if (attribute.isPresent()) { diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYKeyValuePair.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYKeyValuePair.java index 0befc4bbf7..0111268922 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYKeyValuePair.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYKeyValuePair.java @@ -19,7 +19,13 @@ package org.sleuthkit.autopsy.datasourceprocessors.xry; /** - * A XRY Key Value pair. + * A XRY Key Value pair. XRY Key Value pairs make up the body of XRY entities. + * + * Example: + * + * Attribute: Device Name + * Time: 3/20/2012 12:06:30 AM + * Status: Read */ class XRYKeyValuePair { From e2e94e792ef96c824312862b4e023569b7242958 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Fri, 13 Dec 2019 17:00:18 -0500 Subject: [PATCH 12/22] Adjusted the parsers given the new dump data --- .../xry/XRYCallsFileParser.java | 24 +++++++++++-------- .../xry/XRYContactsFileParser.java | 2 ++ .../xry/XRYFileParserFactory.java | 1 + .../xry/XRYWebBookmarksFileParser.java | 1 + 4 files changed, 18 insertions(+), 10 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java index 6b01a665a5..c1b7dcd09d 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java @@ -56,16 +56,20 @@ final class XRYCallsFileParser extends AbstractSingleKeyValueParser { * blackboard attribute types, if any. */ private enum XryKey { - NUMBER("number", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER), NAME_MATCHED("name (matched)", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_NAME), TIME("time", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME), DIRECTION("direction", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DIRECTION), CALL_TYPE("call type", null), + NUMBER("number", null), TEL("tel", null), + TO("to", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_TO), + FROM("from", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_FROM), + DELETED("deleted", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_ISDELETED), DURATION("duration", null), STORAGE("storage", null), INDEX("index", null), - NAME("name", null); + TYPE("type", null), + NAME("name", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_NAME); private final String name; private final BlackboardAttribute.ATTRIBUTE_TYPE type; @@ -105,8 +109,9 @@ final class XRYCallsFileParser extends AbstractSingleKeyValueParser { * @return */ public static XryKey fromDisplayName(String key) { + String normalizedKey = key.trim().toLowerCase(); for (XryKey keyChoice : XryKey.values()) { - if (key.equals(keyChoice.name)) { + if (normalizedKey.equals(keyChoice.name)) { return keyChoice; } } @@ -138,14 +143,12 @@ final class XRYCallsFileParser extends AbstractSingleKeyValueParser { * @return */ public static boolean contains(String xryNamespace) { - String normalizedNamespace = xryNamespace.trim().toLowerCase(); - for (XryNamespace keyChoice : XryNamespace.values()) { - if (normalizedNamespace.equals(keyChoice.name)) { - return true; - } + try { + XryNamespace.fromDisplayName(xryNamespace); + return true; + } catch (IllegalArgumentException ex) { + return false; } - - return false; } /** @@ -192,6 +195,7 @@ final class XRYCallsFileParser extends AbstractSingleKeyValueParser { switch (xryKey) { case TEL: + case NUMBER: //Apply the namespace switch (xryNamespace) { case FROM: diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYContactsFileParser.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYContactsFileParser.java index 976caabbfe..c5436e7d96 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYContactsFileParser.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYContactsFileParser.java @@ -42,9 +42,11 @@ final class XRYContactsFileParser extends AbstractSingleKeyValueParser { put("name", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_NAME); put("tel", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER); put("mobile", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_MOBILE); + put("home", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_HOME); put("related application", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PROG_NAME); put("address home", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_LOCATION); put("email home", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_EMAIL_HOME); + put("deleted", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_ISDELETED); //Ignoring or need more information to decide. put("storage", null); diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYFileParserFactory.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYFileParserFactory.java index 06492de07b..f7a28d6bff 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYFileParserFactory.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYFileParserFactory.java @@ -46,6 +46,7 @@ final class XRYFileParserFactory { case "calls": return new XRYCallsFileParser(); case "contacts/contacts": + case "contacts": return new XRYContactsFileParser(); case "device/general information": return new XRYDeviceGenInfoFileParser(); diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYWebBookmarksFileParser.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYWebBookmarksFileParser.java index d4edc3b2da..aff0b8a5ad 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYWebBookmarksFileParser.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYWebBookmarksFileParser.java @@ -38,6 +38,7 @@ final class XRYWebBookmarksFileParser extends AbstractSingleKeyValueParser { { put("web address", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_URL); put("domain", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DOMAIN); + put("application", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PROG_NAME); } }; From 84fb5277228b985b2b873a08cc6fac65a1d21cb6 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Mon, 16 Dec 2019 11:04:17 -0500 Subject: [PATCH 13/22] Cleaned up some implementation and comments. Made DeviceGenInfo use the template parser --- ...r.java => AbstractSingleEntityParser.java} | 55 +++----- .../xry/XRYCallsFileParser.java | 29 ++-- .../xry/XRYContactsFileParser.java | 44 +++--- .../xry/XRYDeviceGenInfoFileParser.java | 129 ++++-------------- .../xry/XRYWebBookmarksFileParser.java | 21 ++- 5 files changed, 104 insertions(+), 174 deletions(-) rename Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/{AbstractSingleKeyValueParser.java => AbstractSingleEntityParser.java} (70%) diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/AbstractSingleKeyValueParser.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/AbstractSingleEntityParser.java similarity index 70% rename from Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/AbstractSingleKeyValueParser.java rename to Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/AbstractSingleEntityParser.java index 88fd334522..870475124f 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/AbstractSingleKeyValueParser.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/AbstractSingleEntityParser.java @@ -22,23 +22,19 @@ import java.io.IOException; import java.nio.file.Path; import java.util.ArrayList; import java.util.List; -import java.util.Optional; import java.util.logging.Level; import org.sleuthkit.autopsy.coreutils.Logger; -import org.sleuthkit.datamodel.BlackboardAttribute; import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.TskCoreException; /** - * Template parse method for reports that make blackboard attributes from a - * single key value pair. - * - * This parse implementation will create 1 artifact per XRY entity. + * Template parse method for reports that make artifacts from a single XRY + * Entity. */ -abstract class AbstractSingleKeyValueParser implements XRYFileParser { - - private static final Logger logger = Logger.getLogger(AbstractSingleKeyValueParser.class.getName()); +abstract class AbstractSingleEntityParser implements XRYFileParser { + private static final Logger logger = Logger.getLogger(AbstractSingleEntityParser.class.getName()); + protected static final String PARSER_NAME = "XRY DSP"; @Override @@ -50,14 +46,13 @@ abstract class AbstractSingleKeyValueParser implements XRYFileParser { String xryEntity = reader.nextEntity(); String[] xryLines = xryEntity.split("\n"); - List attributes = new ArrayList<>(); + List keyValuePairs = new ArrayList<>(); //First line of the entity is the title, the entity will always be non-empty. logger.log(Level.INFO, String.format("[XRY DSP] Processing [ %s ]", xryLines[0])); String namespace = ""; //Process each line, searching for a key value pair or a namespace. - //If neither are found, an error message is logged. for (int i = 1; i < xryLines.length; i++) { String xryLine = xryLines[i]; @@ -69,8 +64,7 @@ abstract class AbstractSingleKeyValueParser implements XRYFileParser { continue; } - //Find the XRY key on this line. Assume key is the value between - //the start of the line and the first delimiter. + //Check if this line resembles a Key Value pair. if(!XRYKeyValuePair.isPair(xryLine)) { logger.log(Level.WARNING, String.format("[XRY DSP] Expected a key value " + "pair on this line (in brackets) [ %s ], but one was not detected.", @@ -80,6 +74,7 @@ abstract class AbstractSingleKeyValueParser implements XRYFileParser { XRYKeyValuePair pair = XRYKeyValuePair.from(xryLine, namespace); + //Verify the implementation recognizes the key. if (!canProcess(pair)) { logger.log(Level.WARNING, String.format("[XRY DSP] The following key, " + "value pair (in brackets) [ %s ] was not recognized. Discarding...", @@ -93,27 +88,19 @@ abstract class AbstractSingleKeyValueParser implements XRYFileParser { pair)); continue; } - - //Create the attribute, if any. - Optional attribute = getBlackboardAttribute(namespace, pair); - if(attribute.isPresent()) { - attributes.add(attribute.get()); - } + + keyValuePairs.add(pair); } - - //Only create artifacts with non-empty attributes. - if (!attributes.isEmpty()) { - makeArtifact(attributes, parent); + + if(!keyValuePairs.isEmpty()) { + makeArtifact(keyValuePairs, parent); } } } /** * Determines if the XRY key value pair can be processed by the - * XRY report parser. - * - * @param pair - * @return + * implementation. */ abstract boolean canProcess(XRYKeyValuePair pair); @@ -135,18 +122,8 @@ abstract class AbstractSingleKeyValueParser implements XRYFileParser { abstract boolean isNamespace(String nameSpace); /** - * Creates an attribute from the extracted key value pair. - * - * @param nameSpace The namespace of this key value pair. - * It will have been verified with isNamespace, otherwise it will be empty. - * @param pair The XRYKeyValuePair to process - * @return The corresponding blackboard attribute, if any. + * Makes an artifact from the parsed key value pairs. */ - abstract Optional getBlackboardAttribute(String nameSpace, XRYKeyValuePair pair); - - /** - * Makes an artifact from the parsed attributes. - */ - abstract void makeArtifact(List attributes, Content parent) throws TskCoreException; + abstract void makeArtifact(List keyValuePairs, Content parent) throws TskCoreException; } diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java index c1b7dcd09d..0ccde6975c 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java @@ -27,6 +27,7 @@ import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; import java.time.temporal.TemporalAccessor; import java.time.temporal.TemporalQueries; +import java.util.ArrayList; import java.util.List; import java.util.Optional; import java.util.logging.Level; @@ -39,7 +40,7 @@ import org.sleuthkit.datamodel.TskCoreException; /** * Parses XRY Calls files and creates artifacts. */ -final class XRYCallsFileParser extends AbstractSingleKeyValueParser { +final class XRYCallsFileParser extends AbstractSingleEntityParser { private static final Logger logger = Logger.getLogger(XRYCallsFileParser.class.getName()); @@ -186,11 +187,25 @@ final class XRYCallsFileParser extends AbstractSingleKeyValueParser { } @Override - Optional getBlackboardAttribute(String nameSpace, XRYKeyValuePair pair) { + void makeArtifact(List keyValuePairs, Content parent) throws TskCoreException { + List attributes = new ArrayList<>(); + for(XRYKeyValuePair pair : keyValuePairs) { + Optional attribute = getBlackboardAttribute(pair); + if(attribute.isPresent()) { + attributes.add(attribute.get()); + } + } + if(!attributes.isEmpty()) { + BlackboardArtifact artifact = parent.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_CALLLOG); + artifact.addAttributes(attributes); + } + } + + Optional getBlackboardAttribute(XRYKeyValuePair pair) { XryKey xryKey = XryKey.fromDisplayName(pair.getKey()); XryNamespace xryNamespace = XryNamespace.NONE; - if (XryNamespace.contains(nameSpace)) { - xryNamespace = XryNamespace.fromDisplayName(nameSpace); + if (XryNamespace.contains(pair.getNamespace())) { + xryNamespace = XryNamespace.fromDisplayName(pair.getNamespace()); } switch (xryKey) { @@ -239,12 +254,6 @@ final class XRYCallsFileParser extends AbstractSingleKeyValueParser { } } - @Override - void makeArtifact(List attributes, Content parent) throws TskCoreException { - BlackboardArtifact artifact = parent.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_CALLLOG); - artifact.addAttributes(attributes); - } - /** * Removes the locale from the date time value. * diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYContactsFileParser.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYContactsFileParser.java index c5436e7d96..9122e3e5b7 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYContactsFileParser.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYContactsFileParser.java @@ -18,6 +18,7 @@ */ package org.sleuthkit.autopsy.datasourceprocessors.xry; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -32,7 +33,7 @@ import org.sleuthkit.datamodel.TskCoreException; /** * Parses XRY Contacts-Contacts files and creates artifacts. */ -final class XRYContactsFileParser extends AbstractSingleKeyValueParser { +final class XRYContactsFileParser extends AbstractSingleEntityParser { private static final Logger logger = Logger.getLogger(XRYContactsFileParser.class.getName()); @@ -69,29 +70,32 @@ final class XRYContactsFileParser extends AbstractSingleKeyValueParser { return false; } - @Override - Optional getBlackboardAttribute(String nameSpace, XRYKeyValuePair pair) { + Optional getBlackboardAttribute(XRYKeyValuePair pair) { String normalizedKey = pair.getKey().toLowerCase(); - if(XRY_KEYS.containsKey(normalizedKey)) { - BlackboardAttribute.ATTRIBUTE_TYPE attrType = XRY_KEYS.get(normalizedKey); - if(attrType != null) { - return Optional.of(new BlackboardAttribute(attrType, PARSER_NAME, pair.getValue())); - } - - logger.log(Level.WARNING, String.format("[XRY DSP] Key value pair " - + "(in brackets) [ %s ] was recognized but we need " - + "more data or time to finish implementation. Discarding... ", - pair)); - return Optional.empty(); + BlackboardAttribute.ATTRIBUTE_TYPE attrType = XRY_KEYS.get(normalizedKey); + if(attrType != null) { + return Optional.of(new BlackboardAttribute(attrType, PARSER_NAME, pair.getValue())); } - - throw new IllegalArgumentException(String.format("Key [ %s ] passed the " - + "isKey() test but was not matched.", pair.getKey())); + + logger.log(Level.WARNING, String.format("[XRY DSP] Key value pair " + + "(in brackets) [ %s ] was recognized but we need " + + "more data or time to finish implementation. Discarding... ", + pair)); + return Optional.empty(); } @Override - void makeArtifact(List attributes, Content parent) throws TskCoreException { - BlackboardArtifact artifact = parent.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_CONTACT); - artifact.addAttributes(attributes); + void makeArtifact(List keyValuePairs, Content parent) throws TskCoreException { + List attributes = new ArrayList<>(); + for(XRYKeyValuePair pair : keyValuePairs) { + Optional attribute = getBlackboardAttribute(pair); + if(attribute.isPresent()) { + attributes.add(attribute.get()); + } + } + if(!attributes.isEmpty()) { + BlackboardArtifact artifact = parent.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_CONTACT); + artifact.addAttributes(attributes); + } } } diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYDeviceGenInfoFileParser.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYDeviceGenInfoFileParser.java index d13e27ffc4..b98236045e 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYDeviceGenInfoFileParser.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYDeviceGenInfoFileParser.java @@ -18,8 +18,6 @@ */ package org.sleuthkit.autopsy.datasourceprocessors.xry; -import java.io.IOException; -import java.nio.file.Path; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -35,13 +33,10 @@ import org.sleuthkit.datamodel.TskCoreException; /** * Parses XRY Device-General Information files and creates artifacts. */ -final class XRYDeviceGenInfoFileParser implements XRYFileParser { +final class XRYDeviceGenInfoFileParser extends AbstractSingleEntityParser { private static final Logger logger = Logger.getLogger(XRYDeviceGenInfoFileParser.class.getName()); - //Human readable name of this parser. - private static final String PARSER_NAME = "XRY DSP"; - //All known XRY keys for Device Gen Info reports. private static final String ATTRIBUTE_KEY = "attribute"; private static final String DATA_KEY = "data"; @@ -77,102 +72,45 @@ final class XRYDeviceGenInfoFileParser implements XRYFileParser { put("revision", null); } }; - - /** - * Device-General Information reports generally have 2 key value pairs for - * every blackboard attribute. The two only known keys are "Data" and - * "Attribute", where data is some generic information that the Attribute - * key describes. - * - * Example: - * - * Data: Nokia XYZ - * Attribute: Device Name - * - * This parse implementation assumes that the data field does not span - * multiple lines. If the data does span multiple lines, then on the next - * iteration it will log an exception proclaiming a failed attempt to make a - * 'Data' and 'Attribute' pair. - * - * @param reader The XRYFileReader that reads XRY entities from the - * Device-General Information report. - * @param parent The parent Content to create artifacts from. - * @throws IOException If an I/O error is encountered during report reading - * @throws TskCoreException If an error during artifact creation is - * encountered. - */ + + @Override - public void parse(XRYFileReader reader, Content parent) throws IOException, TskCoreException { - Path reportPath = reader.getReportPath(); - logger.log(Level.INFO, String.format("[XRY DSP] Processing report at [ %s ]", reportPath.toString())); - - while (reader.hasNextEntity()) { - String xryEntity = reader.nextEntity(); - //Extract attributes from this entity. - List attributes = getBlackboardAttributes(xryEntity); - if (!attributes.isEmpty()) { - //Save the artifact. - BlackboardArtifact artifact = parent.newArtifact( - BlackboardArtifact.ARTIFACT_TYPE.TSK_DEVICE_INFO); - artifact.addAttributes(attributes); - } - } + boolean canProcess(XRYKeyValuePair pair) { + String key = pair.getKey().trim().toLowerCase(); + return key.equals(DATA_KEY) || key.equals(ATTRIBUTE_KEY); } - /** - * Parses the XRY entity, extracts key value pairs and creates blackboard - * attributes from these key value pairs. - * - * @param xryEntity XRY entity to parse - * @return A collection of attributes from the XRY entity. - */ - private List getBlackboardAttributes(String xryEntity) { - String[] xryLines = xryEntity.split("\n"); - - //First line of the entity is the title, the entity will always be non-empty. - logger.log(Level.INFO, String.format("[XRY DSP] Processing [ %s ]", xryLines[0])); + @Override + boolean isNamespace(String nameSpace) { + //No known namespaces + return false; + } + + @Override + void makeArtifact(List keyValuePairs, Content parent) throws TskCoreException { List attributes = new ArrayList<>(); - - //Iterate two lines at a time. For Device-General Information, we generally - //need two XRY Key Value pairs per blackboard attribute. - for (int i = 1; i < xryLines.length; i += 2) { - if (!XRYKeyValuePair.isPair(xryLines[i])) { - logger.log(Level.WARNING, String.format("[XRY DSP] Expected a key value " - + "pair on this line (in brackets) [ %s ], but one was not detected." - + " Discarding...", xryLines[i])); - continue; - } - - XRYKeyValuePair firstPair = XRYKeyValuePair.from(xryLines[i]); - Optional attribute = Optional.empty(); - if (i + 1 == xryLines.length) { - attribute = getBlackboardAttribute(firstPair); - } else if (XRYKeyValuePair.isPair(xryLines[i + 1])) { - XRYKeyValuePair secondPair = XRYKeyValuePair.from(xryLines[i + 1]); - attribute = getBlackboardAttribute(firstPair, secondPair); + for(int i = 0; i < keyValuePairs.size(); i++) { + Optional attribute; + if(i + 1 == keyValuePairs.size()) { + attribute = getBlackboardAttribute(keyValuePairs.get(i)); } else { - logger.log(Level.WARNING, String.format("[XRY DSP] Expected a key value " - + "pair on this line (in brackets) [ %s ], but one was not detected." - + " Discarding...", xryLines[i+1])); + attribute = getBlackboardAttribute(keyValuePairs.get(i), keyValuePairs.get(i+1)); } - - if (attribute.isPresent()) { + if(attribute.isPresent()) { attributes.add(attribute.get()); } } - return attributes; + if(!attributes.isEmpty()) { + BlackboardArtifact artifact = parent.newArtifact( + BlackboardArtifact.ARTIFACT_TYPE.TSK_DEVICE_INFO); + artifact.addAttributes(attributes); + } } /** * Creates the appropriate blackboard attribute given a single XRY Key Value - * pair. It is assumed that the only 'Data' keys can appear by themselves in - * Device-Gen Info reports. If a Data key is by itself, then it's assumed to - * be a TSK_PATH attribute. - * - * A WARNING will be logged if this input is not a Data key. - * - * @param pair KeyValuePair to - * @return + * pair. It is assumed that only 'Data' keys can appear by themselves. + * If a Data key is by itself, its value most closely resembles a TSK_PATH attribute. */ private Optional getBlackboardAttribute(XRYKeyValuePair pair) { if (pair.hasKey(DATA_KEY)) { @@ -188,17 +126,10 @@ final class XRYDeviceGenInfoFileParser implements XRYFileParser { } /** - * Creates the appropriate blackboard attribute given the XRY Key Value - * pairs. If the attribute value is recognized but has no corresponding + * Creates the appropriate blackboard attribute given two XRY Key Value + * pairs. The expectation is that one pair is the 'Data' key and the other is + * an 'Attribute' key. If the attribute value is recognized but has no corresponding * Blackboard attribute type, the Optional will be empty. - * - * A WARNING message will be logged for all recognized attribute values that - * don't have a type. More data is needed to make a decision about the - * appropriate type. - * - * @param firstPair - * @param secondPair - * @return */ private Optional getBlackboardAttribute(XRYKeyValuePair firstPair, XRYKeyValuePair secondPair) { String attributeValue; diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYWebBookmarksFileParser.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYWebBookmarksFileParser.java index aff0b8a5ad..30cb171c22 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYWebBookmarksFileParser.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYWebBookmarksFileParser.java @@ -18,6 +18,7 @@ */ package org.sleuthkit.autopsy.datasourceprocessors.xry; +import java.util.ArrayList; import java.util.Map; import java.util.HashMap; import java.util.List; @@ -30,7 +31,7 @@ import org.sleuthkit.datamodel.TskCoreException; /** * Parses XRY Web-Bookmark files and creates artifacts. */ -final class XRYWebBookmarksFileParser extends AbstractSingleKeyValueParser { +final class XRYWebBookmarksFileParser extends AbstractSingleEntityParser { //All known XRY keys for web bookmarks. private static final Map KEY_TO_TYPE @@ -54,8 +55,7 @@ final class XRYWebBookmarksFileParser extends AbstractSingleKeyValueParser { return false; } - @Override - Optional getBlackboardAttribute(String nameSpace, XRYKeyValuePair pair) { + Optional getBlackboardAttribute(XRYKeyValuePair pair) { String normalizedKey = pair.getKey().toLowerCase(); return Optional.of(new BlackboardAttribute( KEY_TO_TYPE.get(normalizedKey), @@ -63,8 +63,17 @@ final class XRYWebBookmarksFileParser extends AbstractSingleKeyValueParser { } @Override - void makeArtifact(List attributes, Content parent) throws TskCoreException { - BlackboardArtifact artifact = parent.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_BOOKMARK); - artifact.addAttributes(attributes); + void makeArtifact(List keyValuePairs, Content parent) throws TskCoreException { + List attributes = new ArrayList<>(); + for(XRYKeyValuePair pair : keyValuePairs) { + Optional attribute = getBlackboardAttribute(pair); + if(attribute.isPresent()) { + attributes.add(attribute.get()); + } + } + if(!attributes.isEmpty()) { + BlackboardArtifact artifact = parent.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_BOOKMARK); + artifact.addAttributes(attributes); + } } } From 02c7f636568a19041788b3bd977ae200c03bb17c Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Mon, 16 Dec 2019 11:28:57 -0500 Subject: [PATCH 14/22] Addressed review comments and fixed a bug --- .../xry/XRYDeviceGenInfoFileParser.java | 2 +- .../datasourceprocessors/xry/XRYKeyValuePair.java | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYDeviceGenInfoFileParser.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYDeviceGenInfoFileParser.java index b98236045e..81aca1cc8e 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYDeviceGenInfoFileParser.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYDeviceGenInfoFileParser.java @@ -89,7 +89,7 @@ final class XRYDeviceGenInfoFileParser extends AbstractSingleEntityParser { @Override void makeArtifact(List keyValuePairs, Content parent) throws TskCoreException { List attributes = new ArrayList<>(); - for(int i = 0; i < keyValuePairs.size(); i++) { + for(int i = 0; i < keyValuePairs.size(); i+=2) { Optional attribute; if(i + 1 == keyValuePairs.size()) { attribute = getBlackboardAttribute(keyValuePairs.get(i)); diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYKeyValuePair.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYKeyValuePair.java index 0111268922..fbea9b4e70 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYKeyValuePair.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYKeyValuePair.java @@ -35,7 +35,7 @@ class XRYKeyValuePair { private final String value; private final String namespace; - public XRYKeyValuePair(String key, String value, String namespace) { + XRYKeyValuePair(String key, String value, String namespace) { this.key = key.trim(); this.value = value.trim(); this.namespace = namespace.trim(); @@ -49,7 +49,7 @@ class XRYKeyValuePair { * * @param targetKey Key name to test. */ - public boolean hasKey(String targetKey) { + boolean hasKey(String targetKey) { String normalizedKey = key.toLowerCase(); String normalizedTargetKey = targetKey.trim().toLowerCase(); return normalizedKey.equals(normalizedTargetKey); @@ -58,21 +58,21 @@ class XRYKeyValuePair { /** * Retrieves the value contained within this pair. */ - public String getValue() { + String getValue() { return value; } /** * Retrieves the key contained within this pair. */ - public String getKey() { + String getKey() { return key; } /** * Retrieves the namespace contained within this pair. */ - public String getNamespace() { + String getNamespace() { return namespace; } From ea12a5be994b72fcb9c5bd62f1e40ba324768428 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Mon, 16 Dec 2019 12:18:29 -0500 Subject: [PATCH 15/22] Various comment fixes and demoted some log messages to INFO --- .../xry/AbstractSingleEntityParser.java | 1 + .../xry/XRYCallsFileParser.java | 16 ++-------------- .../xry/XRYContactsFileParser.java | 4 ++-- .../xry/XRYDeviceGenInfoFileParser.java | 12 +++++++----- .../xry/XRYKeyValuePair.java | 2 +- .../xry/XRYMessagesFileParser.java | 2 +- .../xry/XRYWebBookmarksFileParser.java | 8 ++++---- 7 files changed, 18 insertions(+), 27 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/AbstractSingleEntityParser.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/AbstractSingleEntityParser.java index 870475124f..cdd84fa06e 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/AbstractSingleEntityParser.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/AbstractSingleEntityParser.java @@ -82,6 +82,7 @@ abstract class AbstractSingleEntityParser implements XRYFileParser { continue; } + //Empty values are meaningless for blackboard attributes. if (pair.getValue().isEmpty()) { logger.log(Level.WARNING, String.format("[XRY DSP] The following key value pair" + "(in brackets) [ %s ] was recognized, but the value was empty. Discarding...", diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java index 0ccde6975c..6bc90740bf 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java @@ -86,9 +86,6 @@ final class XRYCallsFileParser extends AbstractSingleEntityParser { /** * Indicates if the display name of the XRY key is a recognized type. - * - * @param xryKey - * @return */ public static boolean contains(String key) { try { @@ -105,9 +102,6 @@ final class XRYCallsFileParser extends AbstractSingleEntityParser { * It is assumed that XRY key string is recognized. Otherwise, an * IllegalArgumentException is thrown. Test all membership with * contains() before hand. - * - * @param xryKey - * @return */ public static XryKey fromDisplayName(String key) { String normalizedKey = key.trim().toLowerCase(); @@ -139,9 +133,6 @@ final class XRYCallsFileParser extends AbstractSingleEntityParser { /** * Indicates if the display name of the XRY namespace is a recognized * type. - * - * @param xryNamespace - * @return */ public static boolean contains(String xryNamespace) { try { @@ -159,9 +150,6 @@ final class XRYCallsFileParser extends AbstractSingleEntityParser { * It is assumed that XRY namespace string is recognized. Otherwise, an * IllegalArgumentException is thrown. Test all membership with * contains() before hand. - * - * @param xryNamespace - * @return */ public static XryNamespace fromDisplayName(String xryNamespace) { String normalizedNamespace = xryNamespace.trim().toLowerCase(); @@ -201,7 +189,7 @@ final class XRYCallsFileParser extends AbstractSingleEntityParser { } } - Optional getBlackboardAttribute(XRYKeyValuePair pair) { + private Optional getBlackboardAttribute(XRYKeyValuePair pair) { XryKey xryKey = XryKey.fromDisplayName(pair.getKey()); XryNamespace xryNamespace = XryNamespace.NONE; if (XryNamespace.contains(pair.getNamespace())) { @@ -246,7 +234,7 @@ final class XRYCallsFileParser extends AbstractSingleEntityParser { PARSER_NAME, pair.getValue())); } - logger.log(Level.WARNING, String.format("[XRY DSP] Key value pair " + logger.log(Level.INFO, String.format("[XRY DSP] Key value pair " + "(in brackets) [ %s ] was recognized but " + "more data or time is needed to finish implementation. Discarding... ", pair)); diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYContactsFileParser.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYContactsFileParser.java index 9122e3e5b7..3024718c81 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYContactsFileParser.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYContactsFileParser.java @@ -70,14 +70,14 @@ final class XRYContactsFileParser extends AbstractSingleEntityParser { return false; } - Optional getBlackboardAttribute(XRYKeyValuePair pair) { + private Optional getBlackboardAttribute(XRYKeyValuePair pair) { String normalizedKey = pair.getKey().toLowerCase(); BlackboardAttribute.ATTRIBUTE_TYPE attrType = XRY_KEYS.get(normalizedKey); if(attrType != null) { return Optional.of(new BlackboardAttribute(attrType, PARSER_NAME, pair.getValue())); } - logger.log(Level.WARNING, String.format("[XRY DSP] Key value pair " + logger.log(Level.INFO, String.format("[XRY DSP] Key value pair " + "(in brackets) [ %s ] was recognized but we need " + "more data or time to finish implementation. Discarding... ", pair)); diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYDeviceGenInfoFileParser.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYDeviceGenInfoFileParser.java index 81aca1cc8e..14d8d54660 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYDeviceGenInfoFileParser.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYDeviceGenInfoFileParser.java @@ -42,9 +42,11 @@ final class XRYDeviceGenInfoFileParser extends AbstractSingleEntityParser { private static final String DATA_KEY = "data"; //All of the known XRY Attribute values for device gen info. The value of the - //attribute keys are actionable for this parser. See parse() header for more - //details. - private static final Map KEY_TO_TYPE + //attribute keys are actionable for this parser. + //Ex: + // Data: Nokia + // Attribute: Device Type + private static final Map XRY_ATTRIBUTE_VALUES = new HashMap() { { put("device name", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DEVICE_NAME); @@ -148,14 +150,14 @@ final class XRYDeviceGenInfoFileParser extends AbstractSingleEntityParser { } String normalizedAttributeValue = attributeValue.toLowerCase(); - if (!KEY_TO_TYPE.containsKey(normalizedAttributeValue)) { + if (!XRY_ATTRIBUTE_VALUES.containsKey(normalizedAttributeValue)) { logger.log(Level.WARNING, String.format("[XRY DSP] Key value pair " + "(in brackets) [ %s : %s ] was not recognized. Discarding... ", attributeValue, dataValue)); return Optional.empty(); } - BlackboardAttribute.ATTRIBUTE_TYPE attrType = KEY_TO_TYPE.get(normalizedAttributeValue); + BlackboardAttribute.ATTRIBUTE_TYPE attrType = XRY_ATTRIBUTE_VALUES.get(normalizedAttributeValue); if (attrType == null) { logger.log(Level.WARNING, String.format("[XRY DSP] Key value pair " + "(in brackets) [ %s : %s ] was recognized but we need " diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYKeyValuePair.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYKeyValuePair.java index fbea9b4e70..a14f571060 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYKeyValuePair.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYKeyValuePair.java @@ -23,7 +23,7 @@ package org.sleuthkit.autopsy.datasourceprocessors.xry; * * Example: * - * Attribute: Device Name + * Attribute: Device Name * Time: 3/20/2012 12:06:30 AM * Status: Read */ diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYMessagesFileParser.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYMessagesFileParser.java index ed29c178a6..74242ddca5 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYMessagesFileParser.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYMessagesFileParser.java @@ -626,7 +626,7 @@ final class XRYMessagesFileParser implements XRYFileParser { PARSER_NAME, pair.getValue())); } - logger.log(Level.WARNING, String.format("[XRY DSP] Key value pair " + logger.log(Level.INFO, String.format("[XRY DSP] Key value pair " + "(in brackets) [ %s ] was recognized but " + "more data or time is needed to finish implementation. Discarding... ", pair)); diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYWebBookmarksFileParser.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYWebBookmarksFileParser.java index 30cb171c22..658ba46af3 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYWebBookmarksFileParser.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYWebBookmarksFileParser.java @@ -34,7 +34,7 @@ import org.sleuthkit.datamodel.TskCoreException; final class XRYWebBookmarksFileParser extends AbstractSingleEntityParser { //All known XRY keys for web bookmarks. - private static final Map KEY_TO_TYPE + private static final Map XRY_KEYS = new HashMap() { { put("web address", BlackboardAttribute.ATTRIBUTE_TYPE.TSK_URL); @@ -46,7 +46,7 @@ final class XRYWebBookmarksFileParser extends AbstractSingleEntityParser { @Override boolean canProcess(XRYKeyValuePair pair) { String normalizedKey = pair.getKey().toLowerCase(); - return KEY_TO_TYPE.containsKey(normalizedKey); + return XRY_KEYS.containsKey(normalizedKey); } @Override @@ -55,10 +55,10 @@ final class XRYWebBookmarksFileParser extends AbstractSingleEntityParser { return false; } - Optional getBlackboardAttribute(XRYKeyValuePair pair) { + private Optional getBlackboardAttribute(XRYKeyValuePair pair) { String normalizedKey = pair.getKey().toLowerCase(); return Optional.of(new BlackboardAttribute( - KEY_TO_TYPE.get(normalizedKey), + XRY_KEYS.get(normalizedKey), PARSER_NAME, pair.getValue())); } From af9a497ccbc00f7a4beef4b910358b62a9c68ce1 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Mon, 16 Dec 2019 12:23:10 -0500 Subject: [PATCH 16/22] Added some comments --- .../autopsy/datasourceprocessors/xry/XRYCallsFileParser.java | 4 ++++ .../datasourceprocessors/xry/XRYContactsFileParser.java | 4 ++++ .../datasourceprocessors/xry/XRYWebBookmarksFileParser.java | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java index 6bc90740bf..ef31f5f417 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYCallsFileParser.java @@ -189,6 +189,10 @@ final class XRYCallsFileParser extends AbstractSingleEntityParser { } } + /** + * Creates the appropriate blackboard attribute given a single XRY Key Value + * pair, if any. Most XRY keys are mapped to an attribute type in the enum above. + */ private Optional getBlackboardAttribute(XRYKeyValuePair pair) { XryKey xryKey = XryKey.fromDisplayName(pair.getKey()); XryNamespace xryNamespace = XryNamespace.NONE; diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYContactsFileParser.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYContactsFileParser.java index 3024718c81..b2dae939e3 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYContactsFileParser.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYContactsFileParser.java @@ -70,6 +70,10 @@ final class XRYContactsFileParser extends AbstractSingleEntityParser { return false; } + /** + * Creates the appropriate blackboard attribute given a single XRY Key Value + * pair. + */ private Optional getBlackboardAttribute(XRYKeyValuePair pair) { String normalizedKey = pair.getKey().toLowerCase(); BlackboardAttribute.ATTRIBUTE_TYPE attrType = XRY_KEYS.get(normalizedKey); diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYWebBookmarksFileParser.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYWebBookmarksFileParser.java index 658ba46af3..9678db0274 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYWebBookmarksFileParser.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYWebBookmarksFileParser.java @@ -55,6 +55,10 @@ final class XRYWebBookmarksFileParser extends AbstractSingleEntityParser { return false; } + /** + * Creates the appropriate blackboard attribute given a single XRY Key Value + * pair. + */ private Optional getBlackboardAttribute(XRYKeyValuePair pair) { String normalizedKey = pair.getKey().toLowerCase(); return Optional.of(new BlackboardAttribute( From ae7273f403d89acc47e40027a0e0a6c238fd797a Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Mon, 16 Dec 2019 12:42:46 -0500 Subject: [PATCH 17/22] Demoted message to warning --- .../autopsy/datasourceprocessors/xry/XRYReportProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYReportProcessor.java b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYReportProcessor.java index 98ee693101..a67ce772f4 100755 --- a/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYReportProcessor.java +++ b/Core/src/org/sleuthkit/autopsy/datasourceprocessors/xry/XRYReportProcessor.java @@ -54,7 +54,7 @@ final class XRYReportProcessor { XRYFileParser parser = XRYFileParserFactory.get(reportType); parser.parse(xryFileReader, parent); } else { - logger.log(Level.SEVERE, String.format("[XRY DSP] XRY File (in brackets) " + logger.log(Level.WARNING, String.format("[XRY DSP] XRY File (in brackets) " + "[ %s ] was found, but no parser to support its report type exists. " + "Report type is [ %s ]", xryFileReader.getReportPath().toString(), reportType)); } From 7693255d17126fca5d281022913edf0da694f90d Mon Sep 17 00:00:00 2001 From: Kelly Kelly Date: Mon, 16 Dec 2019 15:07:30 -0500 Subject: [PATCH 18/22] Fixed selection issues --- .../sleuthkit/autopsy/geolocation/KdTree.java | 2 -- .../autopsy/geolocation/MapPanel.java | 36 ++++++++++++++----- 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/geolocation/KdTree.java b/Core/src/org/sleuthkit/autopsy/geolocation/KdTree.java index 656abc9489..3a2f305083 100755 --- a/Core/src/org/sleuthkit/autopsy/geolocation/KdTree.java +++ b/Core/src/org/sleuthkit/autopsy/geolocation/KdTree.java @@ -417,8 +417,6 @@ public class KdTree implements Iterable { } Double nodeDistance = node.id.euclideanDistance(value); if (nodeDistance.compareTo(lastDistance) < 0) { - if (results.size() == K && lastNode != null) - results.remove(lastNode); results.add(node); } else if (nodeDistance.equals(lastDistance)) { results.add(node); diff --git a/Core/src/org/sleuthkit/autopsy/geolocation/MapPanel.java b/Core/src/org/sleuthkit/autopsy/geolocation/MapPanel.java index 7be6ac25a8..555caa781b 100755 --- a/Core/src/org/sleuthkit/autopsy/geolocation/MapPanel.java +++ b/Core/src/org/sleuthkit/autopsy/geolocation/MapPanel.java @@ -70,6 +70,7 @@ import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; import org.sleuthkit.autopsy.geolocation.datamodel.GeoLocationDataException; import org.sleuthkit.datamodel.TskCoreException; import javax.imageio.ImageIO; +import javax.swing.SwingUtilities; import org.jxmapviewer.viewer.DefaultWaypointRenderer; /** @@ -204,13 +205,14 @@ final public class MapPanel extends javax.swing.JPanel { Iterator iterator = waypointTree.iterator(); while (iterator.hasNext()) { MapWaypoint point = iterator.next(); - if (point != currentlySelectedWaypoint) { +// if (point != currentlySelectedWaypoint) { set.add(point); - } +// } } // Add the currentlySelectedWaypoint to the end so that // it will be painted last. if (currentlySelectedWaypoint != null) { + set.remove(currentlySelectedWaypoint); set.add(currentlySelectedWaypoint); } } @@ -342,7 +344,11 @@ final public class MapPanel extends javax.swing.JPanel { */ private void showPopupMenu(Point point) { try { - MapWaypoint waypoint = findClosestWaypoint(point); + List waypoints = findClosestWaypoint(point); + MapWaypoint waypoint = null; + if(waypoints.size() > 0) { + waypoint = waypoints.get(0); + } showPopupMenu(waypoint, point); // Change the details popup to the currently selected point only if // it the popup is currently visible @@ -410,6 +416,7 @@ final public class MapPanel extends javax.swing.JPanel { currentPopup = popupFactory.getPopup(this, detailPane, popupLocation.x, popupLocation.y); currentPopup.show(); + mapViewer.revalidate(); mapViewer.repaint(); } } @@ -437,7 +444,7 @@ final public class MapPanel extends javax.swing.JPanel { * @return A waypoint that is within 10 pixels of the given point, or null * if none was found. */ - private MapWaypoint findClosestWaypoint(Point mouseClickPoint) { + private List findClosestWaypoint(Point mouseClickPoint) { if (waypointTree == null) { return null; } @@ -446,7 +453,7 @@ final public class MapPanel extends javax.swing.JPanel { GeoPosition geopos = mapViewer.getTileFactory().pixelToGeo(mouseClickPoint, mapViewer.getZoom()); // Get the 5 nearest neightbors to the point - Collection waypoints = waypointTree.nearestNeighbourSearch(20, MapWaypoint.getDummyWaypoint(geopos)); + Collection waypoints = waypointTree.nearestNeighbourSearch(10, MapWaypoint.getDummyWaypoint(geopos)); if (waypoints == null || waypoints.isEmpty()) { return null; @@ -456,6 +463,7 @@ final public class MapPanel extends javax.swing.JPanel { // These maybe the points closest to lat/log was clicked but // that doesn't mean they are close in terms of pixles. + List closestPoints = new ArrayList<>(); while (iterator.hasNext()) { MapWaypoint nextWaypoint = iterator.next(); @@ -466,11 +474,11 @@ final public class MapPanel extends javax.swing.JPanel { (int) point.getY() - rect.y); if (converted_gp_pt.distance(mouseClickPoint) < 10) { - return nextWaypoint; + closestPoints.add(nextWaypoint); } } - return null; + return closestPoints; } /** @@ -629,8 +637,14 @@ final public class MapPanel extends javax.swing.JPanel { }//GEN-LAST:event_mapViewerMouseMoved private void mapViewerMouseClicked(java.awt.event.MouseEvent evt) {//GEN-FIRST:event_mapViewerMouseClicked - if(!evt.isPopupTrigger() && (evt.getButton() == MouseEvent.BUTTON1)) { - currentlySelectedWaypoint = findClosestWaypoint(evt.getPoint()); + if(!evt.isPopupTrigger() && SwingUtilities.isLeftMouseButton(evt)) { + List waypoints = findClosestWaypoint(evt.getPoint()); + if(waypoints.size() > 0) { + currentlySelectedWaypoint = waypoints.get(0); + } + + +// currentlySelectedWaypoint = findClosestWaypoint(evt.getPoint()); showDetailsPopup(); } }//GEN-LAST:event_mapViewerMouseClicked @@ -667,6 +681,10 @@ final public class MapPanel extends javax.swing.JPanel { int y = (int)point.getY(); BufferedImage image = (waypoint == currentlySelectedWaypoint ? selectedWaypointImage: defaultWaypointImage); + + if(waypoint == currentlySelectedWaypoint) { + System.out.println("Paint selected waypoint"); + } (gd.create()).drawImage(image, x -image.getWidth() / 2, y -image.getHeight(), null); } From 31ce04a7920cdfdba2ad24d2593504590d64a4a0 Mon Sep 17 00:00:00 2001 From: Kelly Kelly Date: Mon, 16 Dec 2019 15:18:57 -0500 Subject: [PATCH 19/22] Removed print statement --- Core/src/org/sleuthkit/autopsy/geolocation/MapPanel.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/geolocation/MapPanel.java b/Core/src/org/sleuthkit/autopsy/geolocation/MapPanel.java index 555caa781b..c6174dac01 100755 --- a/Core/src/org/sleuthkit/autopsy/geolocation/MapPanel.java +++ b/Core/src/org/sleuthkit/autopsy/geolocation/MapPanel.java @@ -681,10 +681,6 @@ final public class MapPanel extends javax.swing.JPanel { int y = (int)point.getY(); BufferedImage image = (waypoint == currentlySelectedWaypoint ? selectedWaypointImage: defaultWaypointImage); - - if(waypoint == currentlySelectedWaypoint) { - System.out.println("Paint selected waypoint"); - } (gd.create()).drawImage(image, x -image.getWidth() / 2, y -image.getHeight(), null); } From f126cbba8fa2fef8c20d6be96ef4dbfb8a232b2a Mon Sep 17 00:00:00 2001 From: Kelly Kelly Date: Mon, 16 Dec 2019 15:30:35 -0500 Subject: [PATCH 20/22] Fixed missing label value --- .../autopsy/geolocation/datamodel/LastKnownWaypoint.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/geolocation/datamodel/LastKnownWaypoint.java b/Core/src/org/sleuthkit/autopsy/geolocation/datamodel/LastKnownWaypoint.java index 7bf85874ff..483dfd4689 100755 --- a/Core/src/org/sleuthkit/autopsy/geolocation/datamodel/LastKnownWaypoint.java +++ b/Core/src/org/sleuthkit/autopsy/geolocation/datamodel/LastKnownWaypoint.java @@ -71,7 +71,7 @@ final class LastKnownWaypoint extends Waypoint { "LastKnownWaypoint_Label=Last Known Location",}) private static String getLabelFromArtifact(Map attributeMap) throws GeoLocationDataException { BlackboardAttribute attribute = attributeMap.get(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_NAME); - String label = attribute.getDisplayString(); + String label = attribute != null ? attribute.getDisplayString() : Bundle.LastKnownWaypoint_Label(); if (label == null || label.isEmpty()) { label = Bundle.LastKnownWaypoint_Label(); From 5c01064e2b36730a58830900739367a59ea31905 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Mon, 16 Dec 2019 16:30:18 -0500 Subject: [PATCH 21/22] Fix runaway heap use in EventsModel.java --- Core/src/org/sleuthkit/autopsy/timeline/EventsModel.java | 1 - 1 file changed, 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/timeline/EventsModel.java b/Core/src/org/sleuthkit/autopsy/timeline/EventsModel.java index c75c7c814e..146bc156ed 100755 --- a/Core/src/org/sleuthkit/autopsy/timeline/EventsModel.java +++ b/Core/src/org/sleuthkit/autopsy/timeline/EventsModel.java @@ -209,7 +209,6 @@ public final class EventsModel { * data source data in the case database. */ synchronized private void populateDataSourcesCache() throws TskCoreException { - datasourceIDsToNamesMap.clear(); SleuthkitCase skCase = currentCase.getSleuthkitCase(); for (DataSource ds : skCase.getDataSources()) { datasourceIDsToNamesMap.putIfAbsent(ds.getId(), ds.getName()); From ef54899ac31775c2228e955c51eaff7d725a70d8 Mon Sep 17 00:00:00 2001 From: Kelly Kelly Date: Tue, 17 Dec 2019 11:29:22 -0500 Subject: [PATCH 22/22] Removed some commented out code --- Core/src/org/sleuthkit/autopsy/geolocation/MapPanel.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/geolocation/MapPanel.java b/Core/src/org/sleuthkit/autopsy/geolocation/MapPanel.java index c6174dac01..a90dac796b 100755 --- a/Core/src/org/sleuthkit/autopsy/geolocation/MapPanel.java +++ b/Core/src/org/sleuthkit/autopsy/geolocation/MapPanel.java @@ -205,9 +205,7 @@ final public class MapPanel extends javax.swing.JPanel { Iterator iterator = waypointTree.iterator(); while (iterator.hasNext()) { MapWaypoint point = iterator.next(); -// if (point != currentlySelectedWaypoint) { - set.add(point); -// } + set.add(point); } // Add the currentlySelectedWaypoint to the end so that // it will be painted last.