From 2f8bcf2a7a6d4b3cf3e3707c99a5249beadeee31 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Wed, 27 Feb 2019 13:28:25 -0500 Subject: [PATCH] VCard support added. --- thunderbirdparser/ivy.xml | 3 + .../nbproject/project.properties | 3 + thunderbirdparser/nbproject/project.xml | 12 + .../ThunderbirdMboxFileIngestModule.java | 197 +++++-- .../thunderbirdparser/VcardParser.java | 543 ++++++++++++++++++ 5 files changed, 708 insertions(+), 50 deletions(-) create mode 100755 thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/VcardParser.java diff --git a/thunderbirdparser/ivy.xml b/thunderbirdparser/ivy.xml index 4bf1f47c46..b66def4489 100644 --- a/thunderbirdparser/ivy.xml +++ b/thunderbirdparser/ivy.xml @@ -7,8 +7,11 @@ + + + diff --git a/thunderbirdparser/nbproject/project.properties b/thunderbirdparser/nbproject/project.properties index ceee1f4265..af5e0794ae 100644 --- a/thunderbirdparser/nbproject/project.properties +++ b/thunderbirdparser/nbproject/project.properties @@ -1,7 +1,10 @@ +file.reference.commons-lang3-3.8.1.jar=release/modules/ext/commons-lang3-3.8.1.jar file.reference.apache-mime4j-core-0.8.0.jar=release/modules/ext/apache-mime4j-core-0.8.0-SNAPSHOT.jar file.reference.apache-mime4j-dom-0.8.0.jar=release/modules/ext/apache-mime4j-dom-0.8.0-SNAPSHOT.jar file.reference.apache-mime4j-mbox-iterator-0.8.0.jar=release/modules/ext/apache-mime4j-mbox-iterator-0.8.0-SNAPSHOT.jar file.reference.java-libpst-1.0-SNAPSHOT.jar=release/modules/ext/java-libpst-1.0-SNAPSHOT.jar +file.reference.ez-vcard-0.10.5.jar=release/modules/ext/ez-vcard-0.10.5.jar +file.reference.vinnie-2.0.2.jar=release/modules/ext/vinnie-2.0.2.jar javac.source=1.8 javac.compilerargs=-Xlint -Xlint:-serial license.file=../LICENSE-2.0.txt diff --git a/thunderbirdparser/nbproject/project.xml b/thunderbirdparser/nbproject/project.xml index 666a84dff7..c8ecd28927 100644 --- a/thunderbirdparser/nbproject/project.xml +++ b/thunderbirdparser/nbproject/project.xml @@ -50,6 +50,10 @@ + + ext/commons-lang3-3.8.1.jar + release/modules/ext/commons-lang3-3.8.1.jar + ext/apache-mime4j-core-0.8.0-SNAPSHOT.jar release/modules/ext/apache-mime4j-core-0.8.0-SNAPSHOT.jar @@ -66,6 +70,14 @@ ext/apache-mime4j-mbox-iterator-0.8.0-SNAPSHOT.jar release/modules/ext/apache-mime4j-mbox-iterator-0.8.0-SNAPSHOT.jar + + ext/ez-vcard-0.10.5.jar + release/modules/ext/ez-vcard-0.10.5.jar + + + ext/vinnie-2.0.2.jar + release/modules/ext/vinnie-2.0.2.jar + diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java index bfa1542ce1..c29a26a64f 100644 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2011-2014 Basis Technology Corp. + * Copyright 2011-2019 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -53,24 +53,30 @@ import org.sleuthkit.datamodel.BlackboardAttribute; import org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE; import org.sleuthkit.datamodel.DerivedFile; import org.sleuthkit.datamodel.Relationship; +import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskData; import org.sleuthkit.datamodel.TskDataException; import org.sleuthkit.datamodel.TskException; /** - * File-level ingest module that detects MBOX files based on signature. - * Understands Thunderbird folder layout to provide additional structure and - * metadata. + * File-level ingest module that detects MBOX, PST, and vCard files based on + * signature. Understands Thunderbird folder layout to provide additional + * structure and metadata. */ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { - private static final Logger logger = Logger.getLogger(ThunderbirdMboxFileIngestModule.class.getName()); - private IngestServices services = IngestServices.getInstance(); + private final IngestServices services = IngestServices.getInstance(); private FileManager fileManager; private IngestJobContext context; private Blackboard blackboard; + + private Case currentCase; + private SleuthkitCase tskCase; + /** + * Empty constructor. + */ ThunderbirdMboxFileIngestModule() { } @@ -79,6 +85,8 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { public void startUp(IngestJobContext context) throws IngestModuleException { this.context = context; try { + currentCase = Case.getCurrentCaseThrows(); + tskCase = currentCase.getSleuthkitCase(); fileManager = Case.getCurrentCaseThrows().getServices().getFileManager(); } catch (NoCurrentCaseException ex) { logger.log(Level.SEVERE, "Exception while getting open case.", ex); @@ -132,6 +140,10 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { if (PstParser.isPstFile(abstractFile)) { return processPst(abstractFile); } + + if (VcardParser.isVcardFile(abstractFile)) { + return processVcard(abstractFile); + } return ProcessResult.OK; } @@ -177,13 +189,8 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { PstParser.ParseResult result = parser.parse(file, abstractFile.getId()); if (result == PstParser.ParseResult.OK) { - try { - // parse success: Process email and add artifacts - processEmails(parser.getResults(), abstractFile); - } catch (NoCurrentCaseException ex) { - logger.log(Level.SEVERE, "Exception while getting open case.", ex); //NON-NLS - return ProcessResult.ERROR; - } + // parse success: Process email and add artifacts + processEmails(parser.getResults(), abstractFile); } else if (result == PstParser.ParseResult.ENCRYPT) { // encrypted pst: Add encrypted file artifact @@ -279,12 +286,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { MboxParser parser = new MboxParser(services, emailFolder); List emails = parser.parse(file, abstractFile.getId()); - try { - processEmails(emails, abstractFile); - } catch (NoCurrentCaseException ex) { - logger.log(Level.SEVERE, "Exception while getting open case.", ex); //NON-NLS - return ProcessResult.ERROR; - } + processEmails(emails, abstractFile); if (file.delete() == false) { logger.log(Level.INFO, "Failed to delete temp file: {0}", file.getName()); //NON-NLS @@ -299,6 +301,63 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { return ProcessResult.OK; } + + /** + * Parse and extract data from a vCard file. + * + * @param abstractFile The content to be processed. + * + * @return 'ERROR' whenever a NoCurrentCaseException is encountered; + * otherwise 'OK'. + */ + @Messages({ + "# {0} - file name", + "# {1} - file ID", + "ThunderbirdMboxFileIngestModule.errorMessage.outOfDiskSpace=Out of disk space. Cannot copy '{0}' (id={1}) to parse." + }) + private ProcessResult processVcard(AbstractFile abstractFile) { + String fileName; + try { + fileName = getTempPath() + File.separator + abstractFile.getName() + + "-" + String.valueOf(abstractFile.getId()); + } catch (NoCurrentCaseException ex) { + logger.log(Level.SEVERE, "Exception while getting open case.", ex); //NON-NLS + return ProcessResult.ERROR; + } + File file = new File(fileName); + + long freeSpace = services.getFreeDiskSpace(); + if ((freeSpace != IngestMonitor.DISK_FREE_SPACE_UNKNOWN) && (abstractFile.getSize() >= freeSpace)) { + logger.log(Level.WARNING, String.format("Not enough disk space to write file '%s' (id=%d) to disk.", + abstractFile.getName(), abstractFile.getId())); //NON-NLS + IngestMessage msg = IngestMessage.createErrorMessage(EmailParserModuleFactory.getModuleName(), EmailParserModuleFactory.getModuleName(), + Bundle.ThunderbirdMboxFileIngestModule_errorMessage_outOfDiskSpace(abstractFile.getName(), abstractFile.getId())); + services.postMessage(msg); + return ProcessResult.OK; + } + + try { + ContentUtils.writeToFile(abstractFile, file, context::fileIngestIsCancelled); + } catch (IOException ex) { + logger.log(Level.WARNING, String.format("Failed writing the vCard file '%s' (id=%d) to disk.", + abstractFile.getName(), abstractFile.getId()), ex); //NON-NLS + return ProcessResult.OK; + } + + try { + VcardParser parser = new VcardParser(currentCase, context); + parser.parse(file, abstractFile); + } catch (IOException | NoCurrentCaseException ex) { + logger.log(Level.WARNING, String.format("Exception while parsing the file '%s' (id=%d).", file.getName(), abstractFile.getId()), ex); //NON-NLS + return ProcessResult.OK; + } + + if (file.delete() == false) { + logger.log(Level.INFO, "Failed to delete temp file: {0}", file.getName()); //NON-NLS + } + + return ProcessResult.OK; + } /** * Get a path to a temporary folder. @@ -318,8 +377,9 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { /** * Get a module output folder. - * + * * @throws NoCurrentCaseException if there is no open case. + * * @return the module output folder */ static String getModuleOutputPath() throws NoCurrentCaseException { @@ -349,15 +409,14 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { * * @param emails * @param abstractFile - * @throws NoCurrentCaseException if there is no open case. */ - private void processEmails(List emails, AbstractFile abstractFile) throws NoCurrentCaseException { + private void processEmails(List emails, AbstractFile abstractFile) { List derivedFiles = new ArrayList<>(); for (EmailMessage email : emails) { - BlackboardArtifact msgArtifact = addArtifact(email, abstractFile); + BlackboardArtifact msgArtifact = addEmailArtifact(email, abstractFile); if ((msgArtifact != null) && (email.hasAttachment())) { derivedFiles.addAll(handleAttachments(email.getAttachments(), abstractFile, msgArtifact )); @@ -423,7 +482,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { Pattern p = Pattern.compile("\\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\\.[A-Z]{2,4}\\b", Pattern.CASE_INSENSITIVE); Matcher m = p.matcher(input); - Set emailAddresses = new HashSet(); + Set emailAddresses = new HashSet<>(); while (m.find()) { emailAddresses.add( m.group()); } @@ -431,14 +490,15 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { } /** - * Add a blackboard artifact for the given email message. + * Add a blackboard artifact for the given e-mail message. * - * @param email - * @param abstractFile - * @throws NoCurrentCaseException if there is no open case. + * @param email The e-mail message. + * @param abstractFile The associated file. + * + * @return The generated e-mail message artifact. */ @Messages({"ThunderbirdMboxFileIngestModule.addArtifact.indexError.message=Failed to index email message detected artifact for keyword search."}) - private BlackboardArtifact addArtifact(EmailMessage email, AbstractFile abstractFile) throws NoCurrentCaseException { + private BlackboardArtifact addEmailArtifact(EmailMessage email, AbstractFile abstractFile) { BlackboardArtifact bbart = null; List bbattributes = new ArrayList<>(); String to = email.getRecipients(); @@ -460,19 +520,17 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { AccountFileInstance senderAccountInstance = null; - Case openCase = Case.getCurrentCaseThrows(); - if (senderAddressList.size() == 1) { senderAddress = senderAddressList.get(0); try { - senderAccountInstance = openCase.getSleuthkitCase().getCommunicationsManager().createAccountFileInstance(Account.Type.EMAIL, senderAddress, EmailParserModuleFactory.getModuleName(), abstractFile); + senderAccountInstance = currentCase.getSleuthkitCase().getCommunicationsManager().createAccountFileInstance(Account.Type.EMAIL, senderAddress, EmailParserModuleFactory.getModuleName(), abstractFile); } catch(TskCoreException ex) { logger.log(Level.WARNING, "Failed to create account for email address " + senderAddress, ex); //NON-NLS } } else { - logger.log(Level.WARNING, "Failed to find sender address, from = "+ from); //NON-NLS + logger.log(Level.WARNING, "Failed to find sender address, from = {0}", from); //NON-NLS } List recipientAddresses = new ArrayList<>(); @@ -484,7 +542,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { recipientAddresses.forEach((addr) -> { try { AccountFileInstance recipientAccountInstance = - openCase.getSleuthkitCase().getCommunicationsManager().createAccountFileInstance(Account.Type.EMAIL, addr, + currentCase.getSleuthkitCase().getCommunicationsManager().createAccountFileInstance(Account.Type.EMAIL, addr, EmailParserModuleFactory.getModuleName(), abstractFile); recipientAccountInstances.add(recipientAccountInstance); } @@ -493,25 +551,25 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { } }); - addEmailAttribute(headers, ATTRIBUTE_TYPE.TSK_HEADERS, bbattributes); - addEmailAttribute(from, ATTRIBUTE_TYPE.TSK_EMAIL_FROM, bbattributes); - addEmailAttribute(to, ATTRIBUTE_TYPE.TSK_EMAIL_TO, bbattributes); - addEmailAttribute(subject, ATTRIBUTE_TYPE.TSK_SUBJECT, bbattributes); + addArtifactAttribute(headers, ATTRIBUTE_TYPE.TSK_HEADERS, bbattributes); + addArtifactAttribute(from, ATTRIBUTE_TYPE.TSK_EMAIL_FROM, bbattributes); + addArtifactAttribute(to, ATTRIBUTE_TYPE.TSK_EMAIL_TO, bbattributes); + addArtifactAttribute(subject, ATTRIBUTE_TYPE.TSK_SUBJECT, bbattributes); - addEmailAttribute(dateL, ATTRIBUTE_TYPE.TSK_DATETIME_RCVD, bbattributes); - addEmailAttribute(dateL, ATTRIBUTE_TYPE.TSK_DATETIME_SENT, bbattributes); + addArtifactAttribute(dateL, ATTRIBUTE_TYPE.TSK_DATETIME_RCVD, bbattributes); + addArtifactAttribute(dateL, ATTRIBUTE_TYPE.TSK_DATETIME_SENT, bbattributes); - addEmailAttribute(body, ATTRIBUTE_TYPE.TSK_EMAIL_CONTENT_PLAIN, bbattributes); + addArtifactAttribute(body, ATTRIBUTE_TYPE.TSK_EMAIL_CONTENT_PLAIN, bbattributes); - addEmailAttribute(((id < 0L) ? NbBundle.getMessage(this.getClass(), "ThunderbirdMboxFileIngestModule.notAvail") : String.valueOf(id)), + addArtifactAttribute(((id < 0L) ? NbBundle.getMessage(this.getClass(), "ThunderbirdMboxFileIngestModule.notAvail") : String.valueOf(id)), ATTRIBUTE_TYPE.TSK_MSG_ID, bbattributes); - addEmailAttribute(((localPath.isEmpty() == false) ? localPath : "/foo/bar"), + addArtifactAttribute(((localPath.isEmpty() == false) ? localPath : "/foo/bar"), ATTRIBUTE_TYPE.TSK_PATH, bbattributes); - addEmailAttribute(cc, ATTRIBUTE_TYPE.TSK_EMAIL_CC, bbattributes); - addEmailAttribute(bodyHTML, ATTRIBUTE_TYPE.TSK_EMAIL_CONTENT_HTML, bbattributes); - addEmailAttribute(rtf, ATTRIBUTE_TYPE.TSK_EMAIL_CONTENT_RTF, bbattributes); + addArtifactAttribute(cc, ATTRIBUTE_TYPE.TSK_EMAIL_CC, bbattributes); + addArtifactAttribute(bodyHTML, ATTRIBUTE_TYPE.TSK_EMAIL_CONTENT_HTML, bbattributes); + addArtifactAttribute(rtf, ATTRIBUTE_TYPE.TSK_EMAIL_CONTENT_RTF, bbattributes); try { @@ -520,7 +578,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { bbart.addAttributes(bbattributes); // Add account relationships - openCase.getSleuthkitCase().getCommunicationsManager().addRelationships(senderAccountInstance, recipientAccountInstances, bbart,Relationship.Type.MESSAGE, dateL); + currentCase.getSleuthkitCase().getCommunicationsManager().addRelationships(senderAccountInstance, recipientAccountInstances, bbart,Relationship.Type.MESSAGE, dateL); try { // index the artifact for keyword search @@ -535,23 +593,62 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { return bbart; } - - private void addEmailAttribute(String stringVal, ATTRIBUTE_TYPE attrType, Collection bbattributes) { + + /** + * Add an attribute of a specified type to a supplied Collection. + * + * @param stringVal The attribute value. + * @param attrType The type of attribute to be added. + * @param bbattributes The Collection to which the attribute will be added. + */ + static void addArtifactAttribute(String stringVal, BlackboardAttribute.Type attrType, Collection bbattributes) { if (stringVal.isEmpty() == false) { bbattributes.add(new BlackboardAttribute(attrType, EmailParserModuleFactory.getModuleName(), stringVal)); } } - private void addEmailAttribute(long longVal, ATTRIBUTE_TYPE attrType, Collection bbattributes) { + + /** + * Add an attribute of a specified type to a supplied Collection. + * + * @param stringVal The attribute value. + * @param attrType The type of attribute to be added. + * @param bbattributes The Collection to which the attribute will be added. + */ + static void addArtifactAttribute(String stringVal, ATTRIBUTE_TYPE attrType, Collection bbattributes) { + if (stringVal.isEmpty() == false) { + bbattributes.add(new BlackboardAttribute(attrType, EmailParserModuleFactory.getModuleName(), stringVal)); + } + } + + /** + * Add an attribute of a specified type to a supplied Collection. + * + * @param longVal The attribute value. + * @param attrType The type of attribute to be added. + * @param bbattributes The Collection to which the attribute will be added. + */ + static void addArtifactAttribute(long longVal, ATTRIBUTE_TYPE attrType, Collection bbattributes) { if (longVal > 0) { bbattributes.add(new BlackboardAttribute(attrType, EmailParserModuleFactory.getModuleName(), longVal)); } } + /** + * Post an error message for the user. + * + * @param subj The error subject. + * @param details The error details. + */ void postErrorMessage(String subj, String details) { IngestMessage ingestMessage = IngestMessage.createErrorMessage(EmailParserModuleFactory.getModuleVersion(), subj, details); services.postMessage(ingestMessage); } + /** + * Get the IngestServices object. + * + * @return The IngestServices object. + */ IngestServices getServices() { return services; } diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/VcardParser.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/VcardParser.java new file mode 100755 index 0000000000..6f533f54f3 --- /dev/null +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/VcardParser.java @@ -0,0 +1,543 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2019 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.thunderbirdparser; + +import ezvcard.Ezvcard; +import ezvcard.VCard; +import ezvcard.parameter.EmailType; +import ezvcard.parameter.TelephoneType; +import ezvcard.property.Email; +import ezvcard.property.Organization; +import ezvcard.property.Photo; +import ezvcard.property.Telephone; +import ezvcard.property.Url; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.logging.Level; +import org.apache.commons.lang3.StringUtils; +import org.openide.util.NbBundle; +import org.sleuthkit.autopsy.casemodule.Case; +import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; +import org.sleuthkit.autopsy.casemodule.services.Blackboard; +import org.sleuthkit.autopsy.casemodule.services.FileManager; +import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; +import org.sleuthkit.autopsy.ingest.IngestJobContext; +import org.sleuthkit.autopsy.ingest.IngestServices; +import org.sleuthkit.autopsy.ingest.ModuleContentEvent; +import org.sleuthkit.autopsy.ingest.ModuleDataEvent; +import static org.sleuthkit.autopsy.thunderbirdparser.ThunderbirdMboxFileIngestModule.getRelModuleOutputPath; +import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.datamodel.Account; +import org.sleuthkit.datamodel.AccountFileInstance; +import org.sleuthkit.datamodel.BlackboardArtifact; +import org.sleuthkit.datamodel.BlackboardAttribute; +import org.sleuthkit.datamodel.Content; +import org.sleuthkit.datamodel.DataSource; +import org.sleuthkit.datamodel.Relationship; +import org.sleuthkit.datamodel.SleuthkitCase; +import org.sleuthkit.datamodel.TskCoreException; +import org.sleuthkit.datamodel.TskData; +import org.sleuthkit.datamodel.TskDataException; +import org.sleuthkit.datamodel.TskException; + +/** + * A parser that can extract information from a vCard file and create the + * appropriate artifacts. + */ +final class VcardParser { + private static final String VCARD_HEADER = "BEGIN:VCARD"; + private static final long MIN_FILE_SIZE = 22; + + private static final String PHOTO_TYPE_BMP = "bmp"; + private static final String PHOTO_TYPE_GIF = "gif"; + private static final String PHOTO_TYPE_JPEG = "jpeg"; + private static final String PHOTO_TYPE_PNG = "png"; + private static final Map photoTypeExtensions; + static { + photoTypeExtensions = new HashMap<>(); + photoTypeExtensions.put(PHOTO_TYPE_BMP, ".bmp"); + photoTypeExtensions.put(PHOTO_TYPE_GIF, ".gif"); + photoTypeExtensions.put(PHOTO_TYPE_JPEG, ".jpg"); + photoTypeExtensions.put(PHOTO_TYPE_PNG, ".png"); + } + + private static final Logger logger = Logger.getLogger(VcardParser.class.getName()); + + private final IngestServices services = IngestServices.getInstance(); + private final FileManager fileManager; + private final IngestJobContext context; + private final Blackboard blackboard; + private final Case currentCase; + private final SleuthkitCase tskCase; + + /** + * Create a VcardParser object. + */ + VcardParser(Case currentCase, IngestJobContext context) { + this.context = context; + this.currentCase = currentCase; + tskCase = currentCase.getSleuthkitCase(); + blackboard = currentCase.getServices().getBlackboard(); + fileManager = currentCase.getServices().getFileManager(); + } + + /** + * Is the supplied content a vCard file? + * + * @param content The content to check. + * + * @return True if the supplied content is a vCard file; otherwise false. + */ + static boolean isVcardFile(Content content) { + try { + if (content.getSize() > MIN_FILE_SIZE) { + byte[] buffer = new byte[VCARD_HEADER.length()]; + int byteRead = content.read(buffer, 0, VCARD_HEADER.length()); + if (byteRead > 0) { + String header = new String(buffer); + return header.equalsIgnoreCase(VCARD_HEADER); + } + } + } catch (TskException ex) { + logger.log(Level.WARNING, String.format("Exception while detecting if the file '%s' (id=%d) is a vCard file.", + content.getName(), content.getId())); //NON-NLS + } + + return false; + } + + /** + * Parse the VCard file and compile its data in a VCard object. The + * corresponding artifacts will be created. + * + * @param vcardFile The VCard file to be parsed. + * @param abstractFile The abstract file with which to associate artifacts. + * + * @throws IOException If there is an issue parsing the VCard + * file. + * @throws NoCurrentCaseException If there is no open case. + */ + void parse(File vcardFile, AbstractFile abstractFile) throws IOException, NoCurrentCaseException { + VCard vcard = Ezvcard.parse(vcardFile).first(); + addContactArtifact(vcard, abstractFile); + } + + + + /** + * Add a blackboard artifact for the given contact. + * + * @param vcard The VCard that contains the contact information. + * @param abstractFile The file associated with the data. + * + * @throws NoCurrentCaseException if there is no open case. + * + * @return The generated contact artifact. + */ + @NbBundle.Messages({"VcardParser.addContactArtifact.indexError=Failed to index the contact artifact for keyword search."}) + private BlackboardArtifact addContactArtifact(VCard vcard, AbstractFile abstractFile) throws NoCurrentCaseException { + List attributes = new ArrayList<>(); + List accountInstances = new ArrayList<>(); + + extractPhotos(vcard, abstractFile); + + ThunderbirdMboxFileIngestModule.addArtifactAttribute(vcard.getFormattedName().getValue(), BlackboardAttribute.ATTRIBUTE_TYPE.TSK_NAME_PERSON, attributes); + + for (Telephone telephone : vcard.getTelephoneNumbers()) { + addPhoneAttributes(telephone, abstractFile, attributes); + addPhoneAccountInstances(telephone, abstractFile, accountInstances); + } + + for (Email email : vcard.getEmails()) { + addEmailAttributes(email, abstractFile, attributes); + addEmailAccountInstances(email, abstractFile, accountInstances); + } + + for (Url url : vcard.getUrls()) { + ThunderbirdMboxFileIngestModule.addArtifactAttribute(url.getValue(), BlackboardAttribute.ATTRIBUTE_TYPE.TSK_URL, attributes); + } + + for (Organization organization : vcard.getOrganizations()) { + List values = organization.getValues(); + if (values.isEmpty() == false) { + ThunderbirdMboxFileIngestModule.addArtifactAttribute(values.get(0), BlackboardAttribute.ATTRIBUTE_TYPE.TSK_ORGANIZATION, attributes); + } + } + + AccountFileInstance deviceAccountInstance = addDeviceAccountInstance(abstractFile); + + BlackboardArtifact artifact = null; + org.sleuthkit.datamodel.Blackboard tskBlackboard = tskCase.getBlackboard(); + try { + // Create artifact if it doesn't already exist. + if (!tskBlackboard.artifactExists(abstractFile, BlackboardArtifact.ARTIFACT_TYPE.TSK_CONTACT, attributes)) { + artifact = abstractFile.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_CONTACT); + artifact.addAttributes(attributes); + List blackboardArtifacts = new ArrayList<>(); + blackboardArtifacts.add(artifact); + + // Add account relationships. + if (deviceAccountInstance != null) { + try { + currentCase.getSleuthkitCase().getCommunicationsManager().addRelationships( + deviceAccountInstance, accountInstances, artifact, Relationship.Type.CONTACT, abstractFile.getCrtime()); + } catch (TskDataException ex) { + logger.log(Level.SEVERE, String.format("Failed to create phone and e-mail account relationships (fileName='%s'; fileId=%d; accountId=%d).", + abstractFile.getName(), abstractFile.getId(), deviceAccountInstance.getAccount().getAccountID()), ex); //NON-NLS + } + } + + // Index the artifact for keyword search. + try { + blackboard.indexArtifact(artifact); + } catch (Blackboard.BlackboardException ex) { + logger.log(Level.SEVERE, "Unable to index blackboard artifact " + artifact.getArtifactID(), ex); //NON-NLS + MessageNotifyUtil.Notify.error(Bundle.VcardParser_addContactArtifact_indexError(), artifact.getDisplayName()); + } + + // Fire event to notify UI of this new artifact. + IngestServices.getInstance().fireModuleDataEvent(new ModuleDataEvent( + EmailParserModuleFactory.getModuleName(), BlackboardArtifact.ARTIFACT_TYPE.TSK_CONTACT, + blackboardArtifacts)); + } + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, String.format("Failed to create contact artifact for vCard file '%s' (id=%d).", + abstractFile.getName(), abstractFile.getId()), ex); //NON-NLS + } + + return artifact; + } + + /** + * Extract photos from a given VCard and add them as derived files. + * + * @param vcard The VCard from which to extract the photos. + * @param abstractFile The file associated with the data. + * + * @throws NoCurrentCaseException if there is no open case. + */ + private void extractPhotos(VCard vcard, AbstractFile abstractFile) throws NoCurrentCaseException { + String parentFileName = getUniqueName(abstractFile); + // Skip files that already have been extracted. + try { + String outputPath = getOutputFolderPath(parentFileName); + if (new File(outputPath).exists()) { + List vcardPhotos = vcard.getPhotos(); + List derivedFilesCreated = new ArrayList<>(); + for (int i=0; i < vcardPhotos.size(); i++) { + Photo photo = vcardPhotos.get(i); + + if (photo.getUrl() != null) { + // Skip this photo since its data is not embedded. + continue; + } + + String type = photo.getType(); + if (type == null) { + // Skip this photo since no type is defined. + continue; + } + + type = type.toLowerCase(); + if (type.startsWith("image/")) { + type = type.substring(6); + } + String extension = photoTypeExtensions.get(type); + + byte[] data = photo.getData(); + String extractedFileName = String.format("photo_%d%s", i, extension == null ? "" : extension); + String extractedFilePath = Paths.get(outputPath, extractedFileName).toString(); + try { + writeExtractedImage(extractedFilePath, data); + derivedFilesCreated.add(fileManager.addDerivedFile(extractedFileName, getFileRelativePath(parentFileName, extractedFileName), data.length, + abstractFile.getCtime(), abstractFile.getCrtime(), abstractFile.getAtime(), abstractFile.getAtime(), + true, abstractFile, null, EmailParserModuleFactory.getModuleName(), null, null, TskData.EncodingType.NONE)); + } catch (IOException | TskCoreException ex) { + logger.log(Level.WARNING, String.format("Could not write image to '%s' (id=%d).", extractedFilePath, abstractFile.getId()), ex); //NON-NLS + } + } + if (!derivedFilesCreated.isEmpty()) { + services.fireModuleContentEvent(new ModuleContentEvent(abstractFile)); + context.addFilesToJob(derivedFilesCreated); + } + } + else { + logger.log(Level.INFO, String.format("Skipping photo extraction for file '%s' (id=%d), because it has already been processed.", + abstractFile.getName(), abstractFile.getId())); //NON-NLS + } + } catch (SecurityException ex) { + logger.log(Level.WARNING, String.format("Could not create extraction folder for '%s' (id=%d).", parentFileName, abstractFile.getId())); + } + } + + /** + * Writes image to the module output location. + * + * @param outputPath Path where images is written. + * @param data Byte representation of the data to be written to the + * specified location. + */ + private void writeExtractedImage(String outputPath, byte[] data) throws IOException { + File outputFile = new File(outputPath); + FileOutputStream outputStream = new FileOutputStream(outputFile); + outputStream.write(data); + } + + /** + * Creates a unique name for a file by concatentating the file name and the + * file object id. + * + * @param file The file. + * + * @return The unique file name. + */ + private String getUniqueName(AbstractFile file) { + return file.getName() + "_" + file.getId(); + } + + /** + * Gets the relative path to the file. The path is relative to the case + * folder. + * + * @param fileName Name of the the file for which the path is to be + * generated. + * + * @return The relative file path. + */ + private String getFileRelativePath(String parentFileName, String fileName) throws NoCurrentCaseException { + // Used explicit FWD slashes to maintain DB consistency across operating systems. + return "/" + getRelModuleOutputPath() + "/" + parentFileName + "/" + fileName; //NON-NLS + } + + /** + * Gets path to the output folder for file extraction. If the path does not + * exist, it is created. + * + * @param parentFileName Name of the abstract file being processed. + * + * @throws NoCurrentCaseException if there is no open case. + * + * @return Path to the file extraction folder for a given abstract file. + */ + private String getOutputFolderPath(String parentFileName) throws NoCurrentCaseException { + String outputFolderPath = ThunderbirdMboxFileIngestModule.getModuleOutputPath() + File.separator + parentFileName; + File outputFilePath = new File(outputFolderPath); + if (!outputFilePath.exists()) { + outputFilePath.mkdirs(); + } + return outputFolderPath; + } + + /** + * Generate phone attributes for a given VCard Telephone object. + * + * @param telephone The VCard Telephone from which to generate attributes. + * @param abstractFile The VCard file. + * @param attributes The Collection to which generated attributes will be + * added. + */ + private void addPhoneAttributes(Telephone telephone, AbstractFile abstractFile, Collection attributes) { + String telephoneText = telephone.getText(); + if (telephoneText == null || telephoneText.isEmpty()) { + return; + } + + // Add phone number to collection for later creation of TSK_CONTACT. + List telephoneTypes = telephone.getTypes(); + if (telephoneTypes.isEmpty()) { + ThunderbirdMboxFileIngestModule.addArtifactAttribute(telephone.getText(), BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PHONE_NUMBER, attributes); + } else { + for (TelephoneType type : telephoneTypes) { + /* + * Unfortunately, if the types are lower-case, they don't + * get separated correctly into individual TelephoneTypes by + * ez-vcard. Therefore, we must read them manually + * ourselves. + */ + List splitTelephoneTypes = Arrays.asList( + type.getValue().toUpperCase().replaceAll("\\s+","").split(",")); + + for (String splitType : splitTelephoneTypes) { + String attributeTypeName = "TSK_PHONE_" + splitType; + try { + BlackboardAttribute.Type attributeType = tskCase.getAttributeType(attributeTypeName); + if (attributeType == null) { + // Add this attribute type to the case database. + attributeType = tskCase.addArtifactAttributeType(attributeTypeName, + BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING, + String.format("Phone (%s)", StringUtils.capitalize(splitType.toLowerCase()))); + } + ThunderbirdMboxFileIngestModule.addArtifactAttribute(telephone.getText(), attributeType, attributes); + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, String.format("Unable to retrieve attribute type '%s' for file '%s' (id=%d).", attributeTypeName, abstractFile.getName(), abstractFile.getId()), ex); + } catch (TskDataException ex) { + logger.log(Level.SEVERE, String.format("Unable to add custom attribute type '%s' for file '%s' (id=%d).", attributeTypeName, abstractFile.getName(), abstractFile.getId()), ex); + } + } + } + } + } + + /** + * Generate e-mail attributes for a given VCard Email object. + * + * @param email The VCard Email from which to generate attributes. + * @param abstractFile The VCard file. + * @param attributes The Collection to which generated attributes will be + * added. + */ + private void addEmailAttributes(Email email, AbstractFile abstractFile, Collection attributes) { + String emailValue = email.getValue(); + if (emailValue == null || emailValue.isEmpty()) { + return; + } + + // Add phone number to collection for later creation of TSK_CONTACT. + List emailTypes = email.getTypes(); + if (emailTypes.isEmpty()) { + ThunderbirdMboxFileIngestModule.addArtifactAttribute(email.getValue(), BlackboardAttribute.ATTRIBUTE_TYPE.TSK_EMAIL, attributes); + } else { + for (EmailType type : emailTypes) { + /* + * Unfortunately, if the types are lower-case, they don't + * get separated correctly into individual EmailTypes by + * ez-vcard. Therefore, we must read them manually + * ourselves. + */ + List splitEmailTypes = Arrays.asList( + type.getValue().toUpperCase().replaceAll("\\s+","").split(",")); + + for (String splitType : splitEmailTypes) { + String attributeTypeName = "TSK_EMAIL_" + splitType; + try { + BlackboardAttribute.Type attributeType = tskCase.getAttributeType(attributeTypeName); + if (attributeType == null) { + // Add this attribute type to the case database. + attributeType = tskCase.addArtifactAttributeType(attributeTypeName, + BlackboardAttribute.TSK_BLACKBOARD_ATTRIBUTE_VALUE_TYPE.STRING, + String.format("Email (%s)", StringUtils.capitalize(splitType.toLowerCase()))); + } + ThunderbirdMboxFileIngestModule.addArtifactAttribute(email.getValue(), attributeType, attributes); + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, String.format("Unable to retrieve attribute type '%s' for file '%s' (id=%d).", attributeTypeName, abstractFile.getName(), abstractFile.getId()), ex); + } catch (TskDataException ex) { + logger.log(Level.SEVERE, String.format("Unable to add custom attribute type '%s' for file '%s' (id=%d).", attributeTypeName, abstractFile.getName(), abstractFile.getId()), ex); + } + } + } + } + } + + /** + * Generate account instances for a given VCard Telephone object. + * + * @param telephone The VCard Telephone from which to generate + * account instances. + * @param abstractFile The VCard file. + * @param accountInstances The Collection to which generated account + * instances will be added. + */ + private void addPhoneAccountInstances(Telephone telephone, AbstractFile abstractFile, Collection accountInstances) { + String telephoneText = telephone.getText(); + if (telephoneText == null || telephoneText.isEmpty()) { + return; + } + + // Add phone number as a TSK_ACCOUNT. + try { + AccountFileInstance phoneAccountInstance = tskCase.getCommunicationsManager().createAccountFileInstance(Account.Type.PHONE, + telephoneText, EmailParserModuleFactory.getModuleName(), abstractFile); + accountInstances.add(phoneAccountInstance); + } + catch(TskCoreException ex) { + logger.log(Level.WARNING, String.format( + "Failed to create account for phone number '%s' (content='%s'; id=%d).", + telephoneText, abstractFile.getName(), abstractFile.getId()), ex); //NON-NLS + } + } + + /** + * Generate account instances for a given VCard Email object. + * + * @param telephone The VCard Email from which to generate account + * instances. + * @param abstractFile The VCard file. + * @param accountInstances The Collection to which generated account + * instances will be added. + */ + private void addEmailAccountInstances(Email email, AbstractFile abstractFile, Collection accountInstances) { + String emailValue = email.getValue(); + if (emailValue == null || emailValue.isEmpty()) { + return; + } + + // Add e-mail as a TSK_ACCOUNT. + try { + AccountFileInstance emailAccountInstance = tskCase.getCommunicationsManager().createAccountFileInstance(Account.Type.EMAIL, + emailValue, EmailParserModuleFactory.getModuleName(), abstractFile); + accountInstances.add(emailAccountInstance); + } + catch(TskCoreException ex) { + logger.log(Level.WARNING, String.format( + "Failed to create account for e-mail address '%s' (content='%s'; id=%d).", + emailValue, abstractFile.getName(), abstractFile.getId()), ex); //NON-NLS + } + } + + /** + * Generate device account instance for a given file. + * + * @param abstractFile The VCard file. + * + * @return The generated device account instance. + */ + private AccountFileInstance addDeviceAccountInstance(AbstractFile abstractFile) { + // Add 'DEVICE' TSK_ACCOUNT. + AccountFileInstance deviceAccountInstance = null; + String deviceId = null; + try { + long dataSourceObjId = abstractFile.getDataSourceObjectId(); + DataSource dataSource = tskCase.getDataSource(dataSourceObjId); + deviceId = dataSource.getDeviceId(); + deviceAccountInstance = tskCase.getCommunicationsManager().createAccountFileInstance(Account.Type.DEVICE, + deviceId, EmailParserModuleFactory.getModuleName(), abstractFile); + } + catch (TskCoreException ex) { + logger.log(Level.WARNING, String.format( + "Failed to create device account for '%s' (content='%s'; id=%d).", + deviceId, abstractFile.getName(), abstractFile.getId()), ex); //NON-NLS + } + catch (TskDataException ex) { + logger.log(Level.WARNING, String.format( + "Failed to get the data source from the case database (id=%d).", + abstractFile.getId()), ex); //NON-NLS + } + + return deviceAccountInstance; + } +}