From b2e452b9b67298a8c33fd5fb05b41c36ee0815e5 Mon Sep 17 00:00:00 2001 From: Kelly Kelly Date: Tue, 26 Jan 2021 13:08:06 -0500 Subject: [PATCH 01/25] Inital commit of changes --- .../thunderbirdparser/EmailMessage.java | 14 +++++++- .../MimeJ4MessageParser.java | 33 ++++++++++++------- 2 files changed, 34 insertions(+), 13 deletions(-) diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/EmailMessage.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/EmailMessage.java index 09a6637e6e..5e70a5eae2 100644 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/EmailMessage.java +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/EmailMessage.java @@ -43,7 +43,7 @@ class EmailMessage { private String localPath = ""; private boolean hasAttachment = false; private long sentDate = 0L; - private List attachments = new ArrayList<>(); + private final List attachments = new ArrayList<>(); private long id = -1L; private String messageID = ""; private String inReplyToID = ""; @@ -410,4 +410,16 @@ class EmailMessage { } } + + static class AttachedEmailMessage extends Attachment { + private final EmailMessage emailMessage; + + AttachedEmailMessage(EmailMessage emailMessage) { + this.emailMessage = emailMessage; + } + + EmailMessage getEmailMessage() { + return emailMessage; + } + } } diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/MimeJ4MessageParser.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/MimeJ4MessageParser.java index ecefe871f1..036cdaaf8b 100755 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/MimeJ4MessageParser.java +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/MimeJ4MessageParser.java @@ -26,10 +26,10 @@ import java.util.ArrayList; import java.util.List; import java.util.UUID; import java.util.logging.Level; -import org.apache.james.mime4j.dom.BinaryBody; import org.apache.james.mime4j.dom.Body; import org.apache.james.mime4j.dom.Entity; import org.apache.james.mime4j.dom.Message; +import org.apache.james.mime4j.dom.MessageWriter; import org.apache.james.mime4j.dom.Multipart; import org.apache.james.mime4j.dom.SingleBody; import org.apache.james.mime4j.dom.TextBody; @@ -39,6 +39,7 @@ import org.apache.james.mime4j.dom.address.MailboxList; import org.apache.james.mime4j.dom.field.ContentDispositionField; import org.apache.james.mime4j.dom.field.ContentTypeField; import org.apache.james.mime4j.message.DefaultMessageBuilder; +import org.apache.james.mime4j.message.DefaultMessageWriter; import org.apache.james.mime4j.stream.Field; import org.apache.james.mime4j.stream.MimeConfig; import org.openide.util.NbBundle; @@ -293,7 +294,7 @@ class MimeJ4MessageParser implements AutoCloseable{ * @param e */ @NbBundle.Messages({"MimeJ4MessageParser.handleAttch.noOpenCase.errMsg=Exception while getting open case."}) - private static void handleAttachment(EmailMessage email, Entity e, long fileID, int index) { + private void handleAttachment(EmailMessage email, Entity e, long fileID, int index) { String outputDirPath; String relModuleOutputPath; try { @@ -322,22 +323,30 @@ class MimeJ4MessageParser implements AutoCloseable{ String outPath = outputDirPath + uniqueFilename; Body body = e.getBody(); - if (body instanceof SingleBody) { + if (body != null) { long fileLength; try (EncodedFileOutputStream fos = new EncodedFileOutputStream(new FileOutputStream(outPath), TskData.EncodingType.XOR1)) { - ((SingleBody) body).writeTo(fos); + + EmailMessage.Attachment attach; + MessageWriter msgWriter = new DefaultMessageWriter(); + + if(body instanceof Message) { + msgWriter.writeMessage((Message)body, fos); + attach = new EmailMessage.AttachedEmailMessage(extractEmail((Message)body, email.getLocalPath(), fileID)); + } else { + msgWriter.writeBody(body, fos); + attach = new EmailMessage.Attachment(); + } fileLength = fos.getBytesWritten(); + attach.setName(filename); + attach.setLocalPath(relModuleOutputPath + uniqueFilename); + attach.setSize(fileLength); + attach.setEncodingType(TskData.EncodingType.XOR1); + email.addAttachment(attach); + } catch (IOException ex) { logger.log(Level.WARNING, "Failed to create file output stream for: " + outPath, ex); //NON-NLS - return; } - - EmailMessage.Attachment attach = new EmailMessage.Attachment(); - attach.setName(filename); - attach.setLocalPath(relModuleOutputPath + uniqueFilename); - attach.setSize(fileLength); - attach.setEncodingType(TskData.EncodingType.XOR1); - email.addAttachment(attach); } From 47d58bfb19236b5697799b1286378e624186d6a7 Mon Sep 17 00:00:00 2001 From: Eugene Livis Date: Tue, 26 Jan 2021 13:56:33 -0500 Subject: [PATCH 02/25] Using different defaults depending on the platform --- .../sleuthkit/autopsy/core/UserPreferences.java | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/core/UserPreferences.java b/Core/src/org/sleuthkit/autopsy/core/UserPreferences.java index 0d3fd0198a..537c25c827 100644 --- a/Core/src/org/sleuthkit/autopsy/core/UserPreferences.java +++ b/Core/src/org/sleuthkit/autopsy/core/UserPreferences.java @@ -28,6 +28,7 @@ import org.openide.util.NbPreferences; import org.python.icu.util.TimeZone; import org.sleuthkit.autopsy.machinesettings.UserMachinePreferences; import org.sleuthkit.autopsy.coreutils.ModuleSettings; +import org.sleuthkit.autopsy.coreutils.PlatformUtil; import org.sleuthkit.autopsy.coreutils.TextConverterException; import org.sleuthkit.autopsy.coreutils.Version; import org.sleuthkit.datamodel.CaseDbConnectionInfo; @@ -84,7 +85,8 @@ public final class UserPreferences { private static final boolean DISPLAY_TRANSLATED_NAMES_DEFAULT = true; public static final String EXTERNAL_HEX_EDITOR_PATH = "ExternalHexEditorPath"; public static final String SOLR_MAX_JVM_SIZE = "SolrMaxJVMSize"; - private static final int DEFAULT_SOLR_HEAP_SIZE_MB = 2048; + private static final int DEFAULT_SOLR_HEAP_SIZE_MB_64BIT_PLATFORM = 2048; + private static final int DEFAULT_SOLR_HEAP_SIZE_MB_32BIT_PLATFORM = 512; public static final String RESULTS_TABLE_PAGE_SIZE = "ResultsTablePageSize"; private static final String GEO_TILE_OPTION = "GeolocationTileOption"; private static final String GEO_OSM_TILE_ZIP_PATH = "GeolocationOsmZipPath"; @@ -534,12 +536,17 @@ public final class UserPreferences { } /** - * Get the maximum JVM heap size (in MB) for the embedded Solr server. + * Get the maximum JVM heap size (in MB) for the embedded Solr server. The returned value + * depends on the platform (64bit vs 32bit). * - * @return Saved value or default (2 GB) + * @return Saved value or default (2 GB for 64bit platforms, 512MB for 32bit) */ public static int getMaxSolrVMSize() { - return preferences.getInt(SOLR_MAX_JVM_SIZE, DEFAULT_SOLR_HEAP_SIZE_MB); + if (PlatformUtil.is64BitJVM()) { + return preferences.getInt(SOLR_MAX_JVM_SIZE, DEFAULT_SOLR_HEAP_SIZE_MB_64BIT_PLATFORM); + } else { + return preferences.getInt(SOLR_MAX_JVM_SIZE, DEFAULT_SOLR_HEAP_SIZE_MB_32BIT_PLATFORM); + } } /** From 149fd3a9321dc8ce1c170b94ccb030b571eee0dc Mon Sep 17 00:00:00 2001 From: rcordovano Date: Tue, 26 Jan 2021 16:08:33 -0500 Subject: [PATCH 03/25] Revert to older versions of a couple of files in release-4.18.0 --- .../autopsy/datamodel/FileTypes.java | 23 ------------------- .../hashdatabase/HashDbIngestModule.java | 17 +++----------- 2 files changed, 3 insertions(+), 37 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/FileTypes.java b/Core/src/org/sleuthkit/autopsy/datamodel/FileTypes.java index 563f9c8cee..aa1d0a04ec 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/FileTypes.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/FileTypes.java @@ -20,7 +20,6 @@ package org.sleuthkit.autopsy.datamodel; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.List; import java.util.Observable; import java.util.Observer; @@ -35,8 +34,6 @@ import org.openide.util.Lookup; import org.openide.util.NbBundle; import org.openide.util.lookup.Lookups; import org.sleuthkit.autopsy.coreutils.Logger; -import org.sleuthkit.datamodel.AnalysisResult; -import org.sleuthkit.datamodel.AnalysisResultAdded; import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardAttribute; import org.sleuthkit.datamodel.Content; @@ -46,7 +43,6 @@ import org.sleuthkit.datamodel.Directory; import org.sleuthkit.datamodel.File; import org.sleuthkit.datamodel.LayoutFile; import org.sleuthkit.datamodel.LocalFile; -import org.sleuthkit.datamodel.Score; import org.sleuthkit.datamodel.SlackFile; import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.SleuthkitItemVisitor; @@ -389,11 +385,6 @@ public final class FileTypes implements AutopsyVisitableItem { return content.newArtifact(artifactTypeID); } - @Override - public AnalysisResultAdded newAnalysisResult(BlackboardArtifact.Type artifactType, Score score, String conclusion, String configuration, String justification, Collection attributesList) throws TskCoreException { - return content.newAnalysisResult(artifactType, score, conclusion, configuration, justification, attributesList); - } - @Override public BlackboardArtifact newArtifact(BlackboardArtifact.ARTIFACT_TYPE type) throws TskCoreException { return content.newArtifact(type); @@ -434,15 +425,6 @@ public final class FileTypes implements AutopsyVisitableItem { return content.getAllArtifacts(); } - @Override - public List getAllAnalysisResults() throws TskCoreException { - return content.getAllAnalysisResults(); - } - - public List getAnalysisResults(BlackboardArtifact.Type artifactType) throws TskCoreException { - return content.getAnalysisResults(artifactType); - } - @Override public Set getHashSetNames() throws TskCoreException { return content.getHashSetNames(); @@ -467,10 +449,5 @@ public final class FileTypes implements AutopsyVisitableItem { public long getAllArtifactsCount() throws TskCoreException { return content.getAllArtifactsCount(); } - - @Override - public Score getAggregateScore() throws TskCoreException { - return content.getAggregateScore(); - } } } diff --git a/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/HashDbIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/HashDbIngestModule.java index 74e019d8cb..19b9ff28b7 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/HashDbIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/HashDbIngestModule.java @@ -41,8 +41,6 @@ import org.sleuthkit.autopsy.ingest.IngestModuleReferenceCounter; import org.sleuthkit.autopsy.ingest.IngestServices; import org.sleuthkit.autopsy.modules.hashdatabase.HashDbManager.HashDb; import org.sleuthkit.datamodel.AbstractFile; -import org.sleuthkit.datamodel.AnalysisResult; -import org.sleuthkit.datamodel.AnalysisResultAdded; import org.sleuthkit.datamodel.Blackboard; import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE; @@ -50,7 +48,6 @@ import org.sleuthkit.datamodel.BlackboardAttribute; import org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE; import org.sleuthkit.datamodel.HashHitInfo; import org.sleuthkit.datamodel.HashUtility; -import org.sleuthkit.datamodel.Score; import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskData; @@ -519,7 +516,7 @@ public class HashDbIngestModule implements FileIngestModule { private void postHashSetHitToBlackboard(AbstractFile abstractFile, String md5Hash, String hashSetName, String comment, boolean showInboxMessage) { try { String moduleName = HashLookupModuleFactory.getModuleName(); - //BlackboardArtifact badFile = abstractFile.newArtifact(ARTIFACT_TYPE.TSK_HASHSET_HIT); + BlackboardArtifact badFile = abstractFile.newArtifact(ARTIFACT_TYPE.TSK_HASHSET_HIT); Collection attributes = new ArrayList<>(); //TODO Revisit usage of deprecated constructor as per TSK-583 //BlackboardAttribute att2 = new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_SET_NAME.getTypeID(), MODULE_NAME, "Known Bad", hashSetName); @@ -527,22 +524,14 @@ public class HashDbIngestModule implements FileIngestModule { attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_HASH_MD5, moduleName, md5Hash)); attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_COMMENT, moduleName, comment)); - SleuthkitCase.CaseDbTransaction trans = this.skCase.beginTransaction(); - - AnalysisResultAdded resultAdded = blackboard.newAnalysisResult(new BlackboardArtifact.Type(ARTIFACT_TYPE.TSK_HASHSET_HIT), abstractFile.getId(), abstractFile.getDataSourceObjectId(), new Score(Score.Significance.MEDIUM, Score.Confidence.HIGH), moduleName, comment, hashSetName, attributes, trans); - AnalysisResult badFile = resultAdded.getAnalysisResult(); - trans.commit(); + badFile.addAttributes(attributes); try { /* * post the artifact which will index the artifact for keyword * search, and fire an event to notify UI of this new artifact */ - - - blackboard.postArtifact(badFile, moduleName); - } catch (Blackboard.BlackboardException ex) { logger.log(Level.SEVERE, "Unable to index blackboard artifact " + badFile.getArtifactID(), ex); //NON-NLS MessageNotifyUtil.Notify.error( @@ -585,7 +574,7 @@ public class HashDbIngestModule implements FileIngestModule { abstractFile.getName() + md5Hash, badFile)); } - } catch (TskException | Blackboard.BlackboardException ex) { + } catch (TskException ex) { logger.log(Level.WARNING, "Error creating blackboard artifact", ex); //NON-NLS } } From 4974955ffa37f3a4d53587cd6cdb6960d32ee7ed Mon Sep 17 00:00:00 2001 From: Kelly Kelly Date: Tue, 26 Jan 2021 16:08:33 -0500 Subject: [PATCH 04/25] Modified code to support nested email messages --- .../ThunderbirdMboxFileIngestModule.java | 39 ++++++++++++++----- 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java index f97bde95fb..bec7347055 100644 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java @@ -48,6 +48,7 @@ import org.sleuthkit.autopsy.ingest.IngestModule.ProcessResult; import org.sleuthkit.autopsy.ingest.IngestMonitor; import org.sleuthkit.autopsy.ingest.IngestServices; import org.sleuthkit.autopsy.ingest.ModuleContentEvent; +import org.sleuthkit.autopsy.thunderbirdparser.EmailMessage.AttachedEmailMessage; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.Account; import org.sleuthkit.datamodel.AccountFileInstance; @@ -450,12 +451,15 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { List derivedFiles = new ArrayList<>(); AccountFileInstanceCache accountFileInstanceCache = new AccountFileInstanceCache(abstractFile, currentCase); - BlackboardArtifact msgArtifact = addEmailArtifact(message, abstractFile, accountFileInstanceCache); +// BlackboardArtifact msgArtifact = addEmailArtifact(message, abstractFile, accountFileInstanceCache); + + createEmailArtifact(message, abstractFile, accountFileInstanceCache, derivedFiles); + accountFileInstanceCache.clear(); - if ((msgArtifact != null) && (message.hasAttachment())) { - derivedFiles.addAll(handleAttachments(message.getAttachments(), abstractFile, msgArtifact)); - } +// if ((msgArtifact != null) && (message.hasAttachment())) { +// derivedFiles.addAll(handleAttachments(message.getAttachments(), abstractFile, msgArtifact)); +// } if (derivedFiles.isEmpty() == false) { for (AbstractFile derived : derivedFiles) { @@ -564,11 +568,13 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { } } - BlackboardArtifact msgArtifact = addEmailArtifact(current, abstractFile, accountFileInstanceCache); - - if ((msgArtifact != null) && (current.hasAttachment())) { - derivedFiles.addAll(handleAttachments(current.getAttachments(), abstractFile, msgArtifact )); - } +// BlackboardArtifact msgArtifact = addEmailArtifact(current, abstractFile, accountFileInstanceCache); +// +// if ((msgArtifact != null) && (current.hasAttachment())) { +// derivedFiles.addAll(handleAttachments(current.getAttachments(), abstractFile, msgArtifact )); +// } + + createEmailArtifact(current, abstractFile, accountFileInstanceCache, derivedFiles); } if (derivedFiles.isEmpty() == false) { @@ -581,6 +587,21 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { } context.addFilesToJob(derivedFiles); } + + void createEmailArtifact(EmailMessage email, AbstractFile abstractFile, AccountFileInstanceCache accountFileInstanceCache, List derivedFiles) { + BlackboardArtifact msgArtifact = addEmailArtifact(email, abstractFile, accountFileInstanceCache); + + if ((msgArtifact != null) && (email.hasAttachment())) { + derivedFiles.addAll(handleAttachments(email.getAttachments(), abstractFile, msgArtifact )); + + for (EmailMessage.Attachment attach : email.getAttachments()) { + if(attach instanceof AttachedEmailMessage) { + createEmailArtifact(((AttachedEmailMessage) attach).getEmailMessage(), abstractFile, accountFileInstanceCache, derivedFiles); + } + } + } + } + /** * Add the given attachments as derived files and reschedule them for * ingest. From 2103ce95f3a2f925024f9ddb10f1ccf1d248a749 Mon Sep 17 00:00:00 2001 From: Kelly Kelly Date: Wed, 27 Jan 2021 17:13:28 -0500 Subject: [PATCH 05/25] Cleaned up EmailParser code --- .../MimeJ4MessageParser.java | 5 +- .../ThunderbirdMboxFileIngestModule.java | 338 +++++++++--------- 2 files changed, 162 insertions(+), 181 deletions(-) diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/MimeJ4MessageParser.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/MimeJ4MessageParser.java index 036cdaaf8b..ba5b24842f 100755 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/MimeJ4MessageParser.java +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/MimeJ4MessageParser.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2019 Basis Technology Corp. + * Copyright 2019-2020 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -31,7 +31,6 @@ import org.apache.james.mime4j.dom.Entity; import org.apache.james.mime4j.dom.Message; import org.apache.james.mime4j.dom.MessageWriter; import org.apache.james.mime4j.dom.Multipart; -import org.apache.james.mime4j.dom.SingleBody; import org.apache.james.mime4j.dom.TextBody; import org.apache.james.mime4j.dom.address.AddressList; import org.apache.james.mime4j.dom.address.Mailbox; @@ -348,8 +347,6 @@ class MimeJ4MessageParser implements AutoCloseable{ logger.log(Level.WARNING, "Failed to create file output stream for: " + outPath, ex); //NON-NLS } } - - } /** diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java index bec7347055..1c165ef535 100644 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java @@ -73,13 +73,14 @@ import org.sleuthkit.datamodel.blackboardutils.attributes.MessageAttachments.Fil * structure and metadata. */ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { + private static final Logger logger = Logger.getLogger(ThunderbirdMboxFileIngestModule.class.getName()); private final IngestServices services = IngestServices.getInstance(); private FileManager fileManager; private IngestJobContext context; private Blackboard blackboard; private CommunicationArtifactsHelper communicationArtifactsHelper; - + private static final int MBOX_SIZE_TO_SPLIT = 1048576000; private Case currentCase; @@ -90,7 +91,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { } @Override - @Messages ({"ThunderbirdMboxFileIngestModule.noOpenCase.errMsg=Exception while getting open case."}) + @Messages({"ThunderbirdMboxFileIngestModule.noOpenCase.errMsg=Exception while getting open case."}) public void startUp(IngestJobContext context) throws IngestModuleException { this.context = context; try { @@ -113,8 +114,8 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { } //skip unalloc - if ((abstractFile.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS)) || - (abstractFile.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.SLACK))) { + if ((abstractFile.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS)) + || (abstractFile.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.SLACK))) { return ProcessResult.OK; } @@ -125,7 +126,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { // check its signature boolean isMbox = false; boolean isEMLFile = false; - + try { byte[] t = new byte[64]; if (abstractFile.getSize() > 64) { @@ -138,15 +139,15 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { } catch (TskException ex) { logger.log(Level.WARNING, null, ex); } - + boolean isPstFile = PstParser.isPstFile(abstractFile); boolean isVcardFile = VcardParser.isVcardFile(abstractFile); - + if (context.fileIngestIsCancelled()) { return ProcessResult.OK; } - - if (isMbox || isEMLFile || isPstFile || isVcardFile ) { + + if (isMbox || isEMLFile || isPstFile || isVcardFile) { try { communicationArtifactsHelper = new CommunicationArtifactsHelper(currentCase.getSleuthkitCase(), EmailParserModuleFactory.getModuleName(), abstractFile, Account.Type.EMAIL); @@ -159,7 +160,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { if (isMbox) { return processMBox(abstractFile); } - + if (isEMLFile) { return processEMLFile(abstractFile); } @@ -167,11 +168,11 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { if (isPstFile) { return processPst(abstractFile); } - + if (isVcardFile) { return processVcard(abstractFile); } - + return ProcessResult.OK; } @@ -187,7 +188,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { String fileName; try { fileName = getTempPath() + File.separator + abstractFile.getName() - + "-" + String.valueOf(abstractFile.getId()); + + "-" + String.valueOf(abstractFile.getId()); } catch (NoCurrentCaseException ex) { logger.log(Level.SEVERE, "Exception while getting open case.", ex); //NON-NLS return ProcessResult.ERROR; @@ -204,8 +205,8 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { services.postMessage(msg); return ProcessResult.OK; } - - try (PstParser parser = new PstParser(services)){ + + try (PstParser parser = new PstParser(services)) { try { ContentUtils.writeToFile(abstractFile, file, context::fileIngestIsCancelled); } catch (IOException ex) { @@ -215,7 +216,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { PstParser.ParseResult result = parser.open(file, abstractFile.getId()); - switch( result) { + switch (result) { case OK: Iterator pstMsgIterator = parser.getEmailMessageIterator(); if (pstMsgIterator != null) { @@ -239,20 +240,20 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { // encrypted pst: Add encrypted file artifact try { - BlackboardArtifact artifact = abstractFile.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_ENCRYPTION_DETECTED); - artifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_NAME, EmailParserModuleFactory.getModuleName(), NbBundle.getMessage(this.getClass(), "ThunderbirdMboxFileIngestModule.encryptionFileLevel"))); + BlackboardArtifact artifact = abstractFile.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_ENCRYPTION_DETECTED); + artifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_NAME, EmailParserModuleFactory.getModuleName(), NbBundle.getMessage(this.getClass(), "ThunderbirdMboxFileIngestModule.encryptionFileLevel"))); - try { - // index the artifact for keyword search - blackboard.postArtifact(artifact, EmailParserModuleFactory.getModuleName()); - } catch (Blackboard.BlackboardException ex) { - MessageNotifyUtil.Notify.error(Bundle.ThunderbirdMboxFileIngestModule_processPst_indexError_message(), artifact.getDisplayName()); - logger.log(Level.SEVERE, "Unable to index blackboard artifact " + artifact.getArtifactID(), ex); //NON-NLS - } - } catch (TskCoreException ex) { - logger.log(Level.INFO, "Failed to add encryption attribute to file: {0}", abstractFile.getName()); //NON-NLS + try { + // index the artifact for keyword search + blackboard.postArtifact(artifact, EmailParserModuleFactory.getModuleName()); + } catch (Blackboard.BlackboardException ex) { + MessageNotifyUtil.Notify.error(Bundle.ThunderbirdMboxFileIngestModule_processPst_indexError_message(), artifact.getDisplayName()); + logger.log(Level.SEVERE, "Unable to index blackboard artifact " + artifact.getArtifactID(), ex); //NON-NLS } - break; + } catch (TskCoreException ex) { + logger.log(Level.INFO, "Failed to add encryption attribute to file: {0}", abstractFile.getName()); //NON-NLS + } + break; default: // parsing error: log message postErrorMessage( @@ -263,8 +264,8 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { logger.log(Level.INFO, "PSTParser failed to parse {0}", abstractFile.getName()); //NON-NLS return ProcessResult.ERROR; } - } catch(Exception ex) { - logger.log(Level.WARNING, String.format("Failed to close temp pst file %s", file.getAbsolutePath())); + } catch (Exception ex) { + logger.log(Level.WARNING, String.format("Failed to close temp pst file %s", file.getAbsolutePath())); } finally { file.delete(); } @@ -295,7 +296,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { String fileName; try { fileName = getTempPath() + File.separator + abstractFile.getName() - + "-" + String.valueOf(abstractFile.getId()); + + "-" + String.valueOf(abstractFile.getId()); } catch (NoCurrentCaseException ex) { logger.log(Level.SEVERE, "Exception while getting open case.", ex); //NON-NLS return ProcessResult.ERROR; @@ -314,7 +315,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { } if (abstractFile.getSize() < MBOX_SIZE_TO_SPLIT) { - + try { ContentUtils.writeToFile(abstractFile, file, context::fileIngestIsCancelled); } catch (IOException ex) { @@ -322,25 +323,25 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { return ProcessResult.OK; } - try{ + try { processMboxFile(file, abstractFile, emailFolder); if (context.fileIngestIsCancelled()) { return ProcessResult.OK; } - }finally { + } finally { file.delete(); } } else { - List mboxSplitOffsets = new ArrayList<>(); - try{ + List mboxSplitOffsets = new ArrayList<>(); + try { mboxSplitOffsets = findMboxSplitOffset(abstractFile, file); } catch (IOException ex) { logger.log(Level.WARNING, String.format("Failed finding split offsets for mbox file {0}.", fileName), ex); //NON-NLS return ProcessResult.OK; } - long startingOffset = 0; + long startingOffset = 0; for (Long mboxSplitOffset : mboxSplitOffsets) { File splitFile = new File(fileName + "-" + mboxSplitOffset); try { @@ -349,55 +350,54 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { logger.log(Level.WARNING, "Failed writing split mbox file to disk.", ex); //NON-NLS return ProcessResult.OK; } - try{ + try { processMboxFile(splitFile, abstractFile, emailFolder); - startingOffset = mboxSplitOffset; + startingOffset = mboxSplitOffset; } finally { splitFile.delete(); } - + if (context.fileIngestIsCancelled()) { return ProcessResult.OK; } } - } - + } + return ProcessResult.OK; } - + private List findMboxSplitOffset(AbstractFile abstractFile, File file) throws IOException { - + List mboxSplitOffset = new ArrayList<>(); - + byte[] buffer = new byte[7]; ReadContentInputStream in = new ReadContentInputStream(abstractFile); - in.skip(MBOX_SIZE_TO_SPLIT); + in.skip(MBOX_SIZE_TO_SPLIT); int len = in.read(buffer); while (len != -1) { len = in.read(buffer); - if (buffer[0] == 13 && buffer[1] == 10 && buffer[2] == 70 && buffer[3] == 114 && - buffer[4] == 111 && buffer[5] == 109 && buffer[6] == 32) { - mboxSplitOffset.add(in.getCurPosition() - 5 ); - in.skip(MBOX_SIZE_TO_SPLIT); + if (buffer[0] == 13 && buffer[1] == 10 && buffer[2] == 70 && buffer[3] == 114 + && buffer[4] == 111 && buffer[5] == 109 && buffer[6] == 32) { + mboxSplitOffset.add(in.getCurPosition() - 5); + in.skip(MBOX_SIZE_TO_SPLIT); } } - + return mboxSplitOffset; - + } - - + private void processMboxFile(File file, AbstractFile abstractFile, String emailFolder) { - - try(MboxParser emailIterator = MboxParser.getEmailIterator( emailFolder, file, abstractFile.getId())) { + + try (MboxParser emailIterator = MboxParser.getEmailIterator(emailFolder, file, abstractFile.getId())) { List emails = new ArrayList<>(); - if(emailIterator != null) { - while(emailIterator.hasNext()) { + if (emailIterator != null) { + while (emailIterator.hasNext()) { if (context.fileIngestIsCancelled()) { return; } EmailMessage emailMessage = emailIterator.next(); - if(emailMessage != null) { + if (emailMessage != null) { emails.add(emailMessage); } } @@ -409,13 +409,13 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { abstractFile.getName()), errors); } } - processEmails(emails, MboxParser.getEmailIterator( emailFolder, file, abstractFile.getId()), abstractFile); - } catch(Exception ex) { + processEmails(emails, MboxParser.getEmailIterator(emailFolder, file, abstractFile.getId()), abstractFile); + } catch (Exception ex) { logger.log(Level.WARNING, String.format("Failed to close mbox temp file %s", file.getAbsolutePath())); } } - + /** * Parse and extract data from a vCard file. * @@ -439,8 +439,8 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { } return ProcessResult.OK; } - - private ProcessResult processEMLFile(AbstractFile abstractFile) { + + private ProcessResult processEMLFile(AbstractFile abstractFile) { try { EmailMessage message = EMLParser.parse(abstractFile); @@ -451,16 +451,9 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { List derivedFiles = new ArrayList<>(); AccountFileInstanceCache accountFileInstanceCache = new AccountFileInstanceCache(abstractFile, currentCase); -// BlackboardArtifact msgArtifact = addEmailArtifact(message, abstractFile, accountFileInstanceCache); - createEmailArtifact(message, abstractFile, accountFileInstanceCache, derivedFiles); - accountFileInstanceCache.clear(); -// if ((msgArtifact != null) && (message.hasAttachment())) { -// derivedFiles.addAll(handleAttachments(message.getAttachments(), abstractFile, msgArtifact)); -// } - if (derivedFiles.isEmpty() == false) { for (AbstractFile derived : derivedFiles) { services.fireModuleContentEvent(new ModuleContentEvent(derived)); @@ -497,7 +490,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { /** * Get a module output folder. - * + * * @throws NoCurrentCaseException if there is no open case. * * @return the module output folder @@ -531,49 +524,42 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { * @param fullMessageIterator * @param abstractFile */ - private void processEmails(List partialEmailsForThreading, Iterator fullMessageIterator, + private void processEmails(List partialEmailsForThreading, Iterator fullMessageIterator, AbstractFile abstractFile) { - + // Create cache for accounts AccountFileInstanceCache accountFileInstanceCache = new AccountFileInstanceCache(abstractFile, currentCase); - + // Putting try/catch around this to catch any exception and still allow // the creation of the artifacts to continue. - try{ + try { EmailMessageThreader.threadMessages(partialEmailsForThreading); - } catch(Exception ex) { + } catch (Exception ex) { logger.log(Level.WARNING, String.format("Exception thrown parsing emails from %s", abstractFile.getName()), ex); } - + List derivedFiles = new ArrayList<>(); int msgCnt = 0; - while(fullMessageIterator.hasNext()) { + while (fullMessageIterator.hasNext()) { if (context.fileIngestIsCancelled()) { return; } - + EmailMessage current = fullMessageIterator.next(); - - if(current == null) { + + if (current == null) { continue; } - if(partialEmailsForThreading.size() > msgCnt) { + if (partialEmailsForThreading.size() > msgCnt) { EmailMessage threaded = partialEmailsForThreading.get(msgCnt++); - - if(threaded.getMessageID().equals(current.getMessageID()) && - threaded.getSubject().equals(current.getSubject())) { + + if (threaded.getMessageID().equals(current.getMessageID()) + && threaded.getSubject().equals(current.getSubject())) { current.setMessageThreadID(threaded.getMessageThreadID()); } } - -// BlackboardArtifact msgArtifact = addEmailArtifact(current, abstractFile, accountFileInstanceCache); -// -// if ((msgArtifact != null) && (current.hasAttachment())) { -// derivedFiles.addAll(handleAttachments(current.getAttachments(), abstractFile, msgArtifact )); -// } - createEmailArtifact(current, abstractFile, accountFileInstanceCache, derivedFiles); } @@ -587,21 +573,21 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { } context.addFilesToJob(derivedFiles); } - + void createEmailArtifact(EmailMessage email, AbstractFile abstractFile, AccountFileInstanceCache accountFileInstanceCache, List derivedFiles) { BlackboardArtifact msgArtifact = addEmailArtifact(email, abstractFile, accountFileInstanceCache); - - if ((msgArtifact != null) && (email.hasAttachment())) { - derivedFiles.addAll(handleAttachments(email.getAttachments(), abstractFile, msgArtifact )); - + + if ((msgArtifact != null) && (email.hasAttachment())) { + derivedFiles.addAll(handleAttachments(email.getAttachments(), abstractFile, msgArtifact)); + for (EmailMessage.Attachment attach : email.getAttachments()) { - if(attach instanceof AttachedEmailMessage) { + if (attach instanceof AttachedEmailMessage) { createEmailArtifact(((AttachedEmailMessage) attach).getEmailMessage(), abstractFile, accountFileInstanceCache, derivedFiles); } } } } - + /** * Add the given attachments as derived files and reschedule them for * ingest. @@ -613,8 +599,8 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { * @return List of attachments */ @NbBundle.Messages({ - "ThunderbirdMboxFileIngestModule.handleAttch.addAttachmentsErrorMsg=Failed to add attachments to email message." -}) + "ThunderbirdMboxFileIngestModule.handleAttch.addAttachmentsErrorMsg=Failed to add attachments to email message." + }) private List handleAttachments(List attachments, AbstractFile abstractFile, BlackboardArtifact messageArtifact) { List files = new ArrayList<>(); List fileAttachments = new ArrayList<>(); @@ -632,11 +618,11 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { DerivedFile df = fileManager.addDerivedFile(filename, relPath, size, cTime, crTime, aTime, mTime, true, abstractFile, "", EmailParserModuleFactory.getModuleName(), EmailParserModuleFactory.getModuleVersion(), "", encodingType); - + associateAttachmentWithMesssge(messageArtifact, df); - + files.add(df); - + fileAttachments.add(new FileAttachment(df)); } catch (TskCoreException ex) { postErrorMessage( @@ -647,17 +633,16 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { logger.log(Level.INFO, "", ex); } } - - + try { communicationArtifactsHelper.addAttachments(messageArtifact, new MessageAttachments(fileAttachments, Collections.emptyList())); } catch (TskCoreException ex) { - postErrorMessage( - NbBundle.getMessage(this.getClass(), "ThunderbirdMboxFileIngestModule.handleAttch.addAttachmentsErrorMsg"), - ""); - logger.log(Level.INFO, "Failed to add attachments to email message.", ex); + postErrorMessage( + NbBundle.getMessage(this.getClass(), "ThunderbirdMboxFileIngestModule.handleAttch.addAttachmentsErrorMsg"), + ""); + logger.log(Level.INFO, "Failed to add attachments to email message.", ex); } - + return files; } @@ -673,32 +658,33 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { bba.addAttributes(attributes); //write out to bb return bba; } - + /** - * Finds and returns a set of unique email addresses found in the input string + * Finds and returns a set of unique email addresses found in the input + * string * * @param input - input string, like the To/CC line from an email header - * + * * @return Set: set of email addresses found in the input string */ private Set findEmailAddresess(String input) { Pattern p = Pattern.compile("\\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\\.[A-Z]{2,4}\\b", - Pattern.CASE_INSENSITIVE); + Pattern.CASE_INSENSITIVE); Matcher m = p.matcher(input); Set emailAddresses = new HashSet<>(); while (m.find()) { - emailAddresses.add( m.group()); + emailAddresses.add(m.group()); } return emailAddresses; } - + /** * Add a blackboard artifact for the given e-mail message. * - * @param email The e-mail message. - * @param abstractFile The associated file. - * @param accountFileInstanceCache The current cache of account instances. - * + * @param email The e-mail message. + * @param abstractFile The associated file. + * @param accountFileInstanceCache The current cache of account instances. + * * @return The generated e-mail message artifact. */ @Messages({"ThunderbirdMboxFileIngestModule.addArtifact.indexError.message=Failed to index email message detected artifact for keyword search."}) @@ -722,35 +708,33 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { List senderAddressList = new ArrayList<>(); String senderAddress; senderAddressList.addAll(findEmailAddresess(from)); - + if (context.fileIngestIsCancelled()) { return null; } - + AccountFileInstance senderAccountInstance = null; if (senderAddressList.size() == 1) { senderAddress = senderAddressList.get(0); try { senderAccountInstance = accountFileInstanceCache.getAccountInstance(senderAddress); + } catch (TskCoreException ex) { + logger.log(Level.WARNING, "Failed to create account for email address " + senderAddress, ex); //NON-NLS } - catch(TskCoreException ex) { - logger.log(Level.WARNING, "Failed to create account for email address " + senderAddress, ex); //NON-NLS - } + } else { + logger.log(Level.WARNING, "Failed to find sender address, from = {0}", from); //NON-NLS } - else { - logger.log(Level.WARNING, "Failed to find sender address, from = {0}", from); //NON-NLS - } - + if (context.fileIngestIsCancelled()) { return null; } - + List recipientAddresses = new ArrayList<>(); recipientAddresses.addAll(findEmailAddresess(to)); recipientAddresses.addAll(findEmailAddresess(cc)); recipientAddresses.addAll(findEmailAddresess(bcc)); - + List recipientAccountInstances = new ArrayList<>(); for (String addr : recipientAddresses) { if (context.fileIngestIsCancelled()) { @@ -759,56 +743,54 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { try { AccountFileInstance recipientAccountInstance = accountFileInstanceCache.getAccountInstance(addr); recipientAccountInstances.add(recipientAccountInstance); - } - catch(TskCoreException ex) { + } catch (TskCoreException ex) { logger.log(Level.WARNING, "Failed to create account for email address " + addr, ex); //NON-NLS } } - + addArtifactAttribute(headers, ATTRIBUTE_TYPE.TSK_HEADERS, bbattributes); addArtifactAttribute(from, ATTRIBUTE_TYPE.TSK_EMAIL_FROM, bbattributes); addArtifactAttribute(to, ATTRIBUTE_TYPE.TSK_EMAIL_TO, bbattributes); addArtifactAttribute(subject, ATTRIBUTE_TYPE.TSK_SUBJECT, bbattributes); - + addArtifactAttribute(dateL, ATTRIBUTE_TYPE.TSK_DATETIME_RCVD, bbattributes); addArtifactAttribute(dateL, ATTRIBUTE_TYPE.TSK_DATETIME_SENT, bbattributes); - + addArtifactAttribute(body, ATTRIBUTE_TYPE.TSK_EMAIL_CONTENT_PLAIN, bbattributes); - - addArtifactAttribute(((id < 0L) ? NbBundle.getMessage(this.getClass(), "ThunderbirdMboxFileIngestModule.notAvail") : String.valueOf(id)), + + addArtifactAttribute(((id < 0L) ? NbBundle.getMessage(this.getClass(), "ThunderbirdMboxFileIngestModule.notAvail") : String.valueOf(id)), ATTRIBUTE_TYPE.TSK_MSG_ID, bbattributes); - - addArtifactAttribute(((localPath.isEmpty() == false) ? localPath : ""), + + addArtifactAttribute(((localPath.isEmpty() == false) ? localPath : ""), ATTRIBUTE_TYPE.TSK_PATH, bbattributes); - + addArtifactAttribute(cc, ATTRIBUTE_TYPE.TSK_EMAIL_CC, bbattributes); addArtifactAttribute(bodyHTML, ATTRIBUTE_TYPE.TSK_EMAIL_CONTENT_HTML, bbattributes); addArtifactAttribute(rtf, ATTRIBUTE_TYPE.TSK_EMAIL_CONTENT_RTF, bbattributes); addArtifactAttribute(threadID, ATTRIBUTE_TYPE.TSK_THREAD_ID, bbattributes); - - + try { if (context.fileIngestIsCancelled()) { return null; } - + bbart = abstractFile.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_EMAIL_MSG); bbart.addAttributes(bbattributes); if (context.fileIngestIsCancelled()) { return null; } - + // Add account relationships - currentCase.getSleuthkitCase().getCommunicationsManager().addRelationships(senderAccountInstance, recipientAccountInstances, bbart,Relationship.Type.MESSAGE, dateL); + currentCase.getSleuthkitCase().getCommunicationsManager().addRelationships(senderAccountInstance, recipientAccountInstances, bbart, Relationship.Type.MESSAGE, dateL); if (context.fileIngestIsCancelled()) { return null; } - + try { // index the artifact for keyword search - blackboard.postArtifact(bbart, EmailParserModuleFactory.getModuleName()); + blackboard.postArtifact(bbart, EmailParserModuleFactory.getModuleName()); } catch (Blackboard.BlackboardException ex) { logger.log(Level.SEVERE, "Unable to index blackboard artifact " + bbart.getArtifactID(), ex); //NON-NLS MessageNotifyUtil.Notify.error(Bundle.ThunderbirdMboxFileIngestModule_addArtifact_indexError_message(), bbart.getDisplayName()); @@ -819,11 +801,11 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { return bbart; } - + /** * Add an attribute of a specified type to a supplied Collection. - * - * @param stringVal The attribute value. + * + * @param stringVal The attribute value. * @param attrType The type of attribute to be added. * @param bbattributes The Collection to which the attribute will be added. */ @@ -835,7 +817,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { /** * Add an attribute of a specified type to a supplied Collection. - * + * * @param stringVal The attribute value. * @param attrType The type of attribute to be added. * @param bbattributes The Collection to which the attribute will be added. @@ -845,10 +827,10 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { bbattributes.add(new BlackboardAttribute(attrType, EmailParserModuleFactory.getModuleName(), stringVal)); } } - + /** * Add an attribute of a specified type to a supplied Collection. - * + * * @param longVal The attribute value. * @param attrType The type of attribute to be added. * @param bbattributes The Collection to which the attribute will be added. @@ -858,49 +840,51 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { bbattributes.add(new BlackboardAttribute(attrType, EmailParserModuleFactory.getModuleName(), longVal)); } } - + /** - * Cache for storing AccountFileInstance. - * The idea is that emails will be used multiple times in a file and - * we shouldn't do a database lookup each time. + * Cache for storing AccountFileInstance. The idea is that emails will be + * used multiple times in a file and we shouldn't do a database lookup each + * time. */ static private class AccountFileInstanceCache { + private final Map cacheMap; private final AbstractFile file; private final Case currentCase; - + /** * Create a new cache. Caches are linked to a specific file. + * * @param file - * @param currentCase + * @param currentCase */ AccountFileInstanceCache(AbstractFile file, Case currentCase) { - cacheMap= new HashMap<>(); + cacheMap = new HashMap<>(); this.file = file; this.currentCase = currentCase; } - + /** * Get the account file instance from the cache or the database. - * + * * @param email The email for this account. - * + * * @return The corresponding AccountFileInstance - * - * @throws TskCoreException + * + * @throws TskCoreException */ AccountFileInstance getAccountInstance(String email) throws TskCoreException { if (cacheMap.containsKey(email)) { return cacheMap.get(email); } - - AccountFileInstance accountInstance = - currentCase.getSleuthkitCase().getCommunicationsManager().createAccountFileInstance(Account.Type.EMAIL, email, - EmailParserModuleFactory.getModuleName(), file); + + AccountFileInstance accountInstance + = currentCase.getSleuthkitCase().getCommunicationsManager().createAccountFileInstance(Account.Type.EMAIL, email, + EmailParserModuleFactory.getModuleName(), file); cacheMap.put(email, accountInstance); return accountInstance; } - + /** * Clears the cache. */ @@ -908,10 +892,10 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { cacheMap.clear(); } } - + /** * Post an error message for the user. - * + * * @param subj The error subject. * @param details The error details. */ @@ -922,7 +906,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { /** * Get the IngestServices object. - * + * * @return The IngestServices object. */ IngestServices getServices() { @@ -933,5 +917,5 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { public void shutDown() { // nothing to shut down } - + } From eb25cba6f39455ff7fe30d0b1cdb4368263e2f66 Mon Sep 17 00:00:00 2001 From: Brian Carrier Date: Mon, 1 Feb 2021 09:41:00 -0500 Subject: [PATCH 06/25] Update Project Vic tag names to latest - fixes bug with latest LE Bundle Module --- .../imagegallery/ImageGalleryController.java | 13 +---- .../imagegallery/ImageGalleryService.java | 54 +++++++++++-------- .../imagegallery/actions/OpenAction.java | 4 ++ .../datamodel/CategoryManager.java | 14 +++-- .../imagegallery/datamodel/DrawableDB.java | 6 +-- 5 files changed, 47 insertions(+), 44 deletions(-) diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java index 95509c4c16..694e7b1c08 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java @@ -95,8 +95,7 @@ public final class ImageGalleryController { private static final Logger logger = Logger.getLogger(ImageGalleryController.class.getName()); private static final Set INGEST_JOB_EVENTS_OF_INTEREST = EnumSet.of(IngestManager.IngestJobEvent.DATA_SOURCE_ANALYSIS_STARTED, IngestManager.IngestJobEvent.DATA_SOURCE_ANALYSIS_COMPLETED); private static final Set INGEST_MODULE_EVENTS_OF_INTEREST = EnumSet.of(IngestManager.IngestModuleEvent.DATA_ADDED, IngestManager.IngestModuleEvent.FILE_DONE); - - private static String DEFAULT_TAG_SET_NAME = "Project VIC"; + /* * The file limit for image gallery. If the selected data source (or all * data sources, if that option is selected) has more than this many files @@ -738,7 +737,7 @@ public final class ImageGalleryController { List tagSetList = getCaseDatabase().getTaggingManager().getTagSets(); if (tagSetList != null && !tagSetList.isEmpty()) { for (TagSet set : tagSetList) { - if (set.getName().equals(getCategoryTagSetName())) { + if (set.getName().equals(ImageGalleryService.PROJECT_VIC_TAG_SET_NAME)) { return set; } } @@ -749,14 +748,6 @@ public final class ImageGalleryController { } } - /** - * Returns the name of the category tag set. - * - * @return Tagset name - */ - static String getCategoryTagSetName() { - return DEFAULT_TAG_SET_NAME; - } /** * A listener for ingest module application events. diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryService.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryService.java index 176b6e1a44..4a8664eb6f 100755 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryService.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryService.java @@ -44,20 +44,32 @@ import org.sleuthkit.datamodel.TskData; }) public class ImageGalleryService implements AutopsyService { - private static final String CATEGORY_ONE_NAME = "Child Exploitation (Illegal)"; - private static final String CATEGORY_TWO_NAME = "Child Exploitation (Non-Illegal/Age Difficult)"; - private static final String CATEGORY_THREE_NAME = "CGI/Animation (Child Exploitive)"; - private static final String CATEGORY_FOUR_NAME = "Exemplar/Comparison (Internal Use Only)"; - private static final String CATEGORY_FIVE_NAME = "Non-pertinent"; - - private static final List DEFAULT_CATEGORY_DEFINITION = new ArrayList<>(); + /* Image Gallery has its own definition of Project VIC tag names because + * these will be used if the Project Vic module is not installed. These will + * get added when a case is opened if the tag set is not already defined. + * + * The following list of names must be kept in sync with the CountryManager + * code in the ProjectVic module. + * + * Autopsy Core Tag code and TSK DataModel upgrade code also have a + * references to the "Projet VIC" set name. Be careful changing any of these names. + */ + static String PROJECT_VIC_TAG_SET_NAME = "Project VIC"; + private static final String PV_US_CAT0 = "Non-Pertinent"; + private static final String PV_US_CAT1 = "Child Abuse Material - (CAM)"; + private static final String PV_US_CAT2 = "Child Exploitive (Non-CAM) Age Difficult"; + private static final String PV_US_CAT3 = "CGI/Animation - Child Exploitive"; + private static final String PV_US_CAT4 = "Comparison Images"; + + private static final List PROJECT_VIC_US_CATEGORIES = new ArrayList<>(); static { - DEFAULT_CATEGORY_DEFINITION.add(new TagNameDefinition(CATEGORY_ONE_NAME, "", TagName.HTML_COLOR.RED, TskData.FileKnown.BAD)); - DEFAULT_CATEGORY_DEFINITION.add(new TagNameDefinition(CATEGORY_TWO_NAME, "", TagName.HTML_COLOR.LIME, TskData.FileKnown.BAD)); - DEFAULT_CATEGORY_DEFINITION.add(new TagNameDefinition(CATEGORY_THREE_NAME, "", TagName.HTML_COLOR.YELLOW, TskData.FileKnown.BAD)); - DEFAULT_CATEGORY_DEFINITION.add(new TagNameDefinition(CATEGORY_FOUR_NAME, "", TagName.HTML_COLOR.PURPLE, TskData.FileKnown.UNKNOWN)); - DEFAULT_CATEGORY_DEFINITION.add(new TagNameDefinition(CATEGORY_FIVE_NAME, "", TagName.HTML_COLOR.FUCHSIA, TskData.FileKnown.UNKNOWN)); + // NOTE: The colors here are what will be shown in the border + PROJECT_VIC_US_CATEGORIES.add(new TagNameDefinition(PV_US_CAT0, "", TagName.HTML_COLOR.GREEN, TskData.FileKnown.UNKNOWN)); + PROJECT_VIC_US_CATEGORIES.add(new TagNameDefinition(PV_US_CAT1, "", TagName.HTML_COLOR.RED, TskData.FileKnown.BAD)); + PROJECT_VIC_US_CATEGORIES.add(new TagNameDefinition(PV_US_CAT2, "", TagName.HTML_COLOR.YELLOW, TskData.FileKnown.BAD)); + PROJECT_VIC_US_CATEGORIES.add(new TagNameDefinition(PV_US_CAT3, "", TagName.HTML_COLOR.FUCHSIA, TskData.FileKnown.BAD)); + PROJECT_VIC_US_CATEGORIES.add(new TagNameDefinition(PV_US_CAT4, "", TagName.HTML_COLOR.BLUE, TskData.FileKnown.UNKNOWN)); } @Override @@ -91,17 +103,17 @@ public class ImageGalleryService implements AutopsyService { // Check to see if the Project VIC tag set exists, if not create a // tag set using the default tags. - boolean addDefaultTagSet = true; + boolean addProjVicTagSet = true; List tagSets = context.getCase().getServices().getTagsManager().getAllTagSets(); for (TagSet set : tagSets) { - if (set.getName().equals(ImageGalleryController.getCategoryTagSetName())) { - addDefaultTagSet = false; + if (set.getName().equals(PROJECT_VIC_TAG_SET_NAME)) { + addProjVicTagSet = false; break; } } - if (addDefaultTagSet) { - addDefaultTagSet(context.getCase()); + if (addProjVicTagSet) { + addProjetVicTagSet(context.getCase()); } ImageGalleryController.createController(context.getCase()); @@ -134,13 +146,11 @@ public class ImageGalleryService implements AutopsyService { * * @throws TskCoreException */ - private void addDefaultTagSet(Case currentCase) throws TskCoreException { + private void addProjetVicTagSet(Case currentCase) throws TskCoreException { List tagNames = new ArrayList<>(); - for (TagNameDefinition def : DEFAULT_CATEGORY_DEFINITION) { + for (TagNameDefinition def : PROJECT_VIC_US_CATEGORIES) { tagNames.add(currentCase.getSleuthkitCase().addOrUpdateTagName(def.getDisplayName(), def.getDescription(), def.getColor(), def.getKnownStatus())); } - - currentCase.getServices().getTagsManager().addTagSet(ImageGalleryController.getCategoryTagSetName(), tagNames); + currentCase.getServices().getTagsManager().addTagSet(PROJECT_VIC_TAG_SET_NAME, tagNames); } - } diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/actions/OpenAction.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/actions/OpenAction.java index 2df0ee2ed1..92ea324105 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/actions/OpenAction.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/actions/OpenAction.java @@ -157,7 +157,10 @@ public final class OpenAction extends CallableSystemAction { } Platform.runLater(() -> { ImageGalleryController controller; + // @@@ This call gets a lock. We shouldn't do this in the UI.... controller = ImageGalleryController.getController(currentCase); + + // Display an error if we could not get the controller and return if (controller == null) { Alert errorDIalog = new Alert(Alert.AlertType.ERROR); errorDIalog.initModality(Modality.APPLICATION_MODAL); @@ -174,6 +177,7 @@ public final class OpenAction extends CallableSystemAction { return; } + // Make sure the user is aware of Single vs Multi-user behaviors if (currentCase.getCaseType() == Case.CaseType.MULTI_USER_CASE && ImageGalleryPreferences.isMultiUserCaseInfoDialogDisabled() == false) { Alert dialog = new Alert(Alert.AlertType.INFORMATION); diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/CategoryManager.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/CategoryManager.java index 8419fb40fe..4464a60fc7 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/CategoryManager.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/CategoryManager.java @@ -99,9 +99,9 @@ public class CategoryManager { } /** - * get the number of file with the given {@link DhsImageCategory} + * get the number of file with the given tag * - * @param cat get the number of files with Category = cat + * @param tagName get the number of files with Category = tagName * * @return the number of files with the given Category */ @@ -110,20 +110,18 @@ public class CategoryManager { } /** - * increment the cached value for the number of files with the given - * {@link DhsImageCategory} + * increment the cached value for the number of files with the given tag * - * @param cat the Category to increment + * @param tagName the Category to increment */ synchronized public void incrementCategoryCount(TagName tagName) { categoryCounts.getUnchecked(tagName).increment(); } /** - * decrement the cached value for the number of files with the given - * DhsImageCategory + * decrement the cached value for the number of files with the given tag * - * @param cat the Category to decrement + * @param tagName the Category to decrement */ synchronized public void decrementCategoryCount(TagName tagName) { categoryCounts.getUnchecked(tagName).decrement(); diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java index 3aa39bd2fd..ab4a2cbe6a 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java @@ -207,19 +207,19 @@ public final class DrawableDB { */ UNKNOWN, /** - * Analyis (an ingest job or image gallery database rebuild) for the + * Analysis (an ingest job or image gallery database rebuild) for the * data source is in progress. */ IN_PROGRESS, /** - * Analyis (an ingest job or image gallery database rebuild) for the + * Analysis (an ingest job or image gallery database rebuild) for the * data source has been completed and at least one file in the data * source has a MIME type (ingest filters may have been applied, so some * files may not have been typed). */ COMPLETE, /** - * Analyis (an ingest job or image gallery database rebuild) for the + * Analysis (an ingest job or image gallery database rebuild) for the * data source has been completed, but the files for the data source * were not assigned a MIME type (file typing was not enabled). */ From 89a2a46c54ea12db0bc8fc09e97cee7f2a2477fd Mon Sep 17 00:00:00 2001 From: Brian Carrier Date: Mon, 1 Feb 2021 10:19:33 -0500 Subject: [PATCH 07/25] refactoring while debugging TagSet issue. No logic/public API changes --- .../services/TagNameDefinition.java | 26 +++++++++--------- .../casemodule/services/TagSetDefinition.java | 2 +- .../casemodule/services/TagsManager.java | 27 +++++++++++-------- .../autopsy/datamodel/DhsImageCategory.java | 1 + 4 files changed, 31 insertions(+), 25 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/services/TagNameDefinition.java b/Core/src/org/sleuthkit/autopsy/casemodule/services/TagNameDefinition.java index 17cff6d900..ff07662e64 100755 --- a/Core/src/org/sleuthkit/autopsy/casemodule/services/TagNameDefinition.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/services/TagNameDefinition.java @@ -61,19 +61,19 @@ final public class TagNameDefinition implements Comparable { private final TskData.FileKnown knownStatus; private static final List STANDARD_TAGS_DEFINITIONS = new ArrayList<>(); - private static final List OLD_CATEGORY_TAG_NAMES = new ArrayList<>(); + private static final List PROJECT_VIC_NAMES_NO_LONGER_USED = new ArrayList<>(); static { STANDARD_TAGS_DEFINITIONS.add(new TagNameDefinition(Bundle.TagNameDefinition_predefTagNames_bookmark_text(), "", TagName.HTML_COLOR.NONE, TskData.FileKnown.UNKNOWN)); STANDARD_TAGS_DEFINITIONS.add(new TagNameDefinition(Bundle.TagNameDefinition_predefTagNames_followUp_text(), "", TagName.HTML_COLOR.NONE, TskData.FileKnown.UNKNOWN)); STANDARD_TAGS_DEFINITIONS.add(new TagNameDefinition(Bundle.TagNameDefinition_predefTagNames_notableItem_text(), "", TagName.HTML_COLOR.NONE, TskData.FileKnown.BAD)); - OLD_CATEGORY_TAG_NAMES.add("CAT-1: Child Exploitation (Illegal)"); - OLD_CATEGORY_TAG_NAMES.add("CAT-2: Child Exploitation (Non-Illegal/Age Difficult)"); - OLD_CATEGORY_TAG_NAMES.add("CAT-3: CGI/Animation (Child Exploitive)"); - OLD_CATEGORY_TAG_NAMES.add("CAT-4: Exemplar/Comparison (Internal Use Only)"); - OLD_CATEGORY_TAG_NAMES.add("CAT-5: Non-pertinent"); - OLD_CATEGORY_TAG_NAMES.add("CAT-0: Uncategorized"); + PROJECT_VIC_NAMES_NO_LONGER_USED.add("CAT-1: Child Exploitation (Illegal)"); + PROJECT_VIC_NAMES_NO_LONGER_USED.add("CAT-2: Child Exploitation (Non-Illegal/Age Difficult)"); + PROJECT_VIC_NAMES_NO_LONGER_USED.add("CAT-3: CGI/Animation (Child Exploitive)"); + PROJECT_VIC_NAMES_NO_LONGER_USED.add("CAT-4: Exemplar/Comparison (Internal Use Only)"); + PROJECT_VIC_NAMES_NO_LONGER_USED.add("CAT-5: Non-pertinent"); + PROJECT_VIC_NAMES_NO_LONGER_USED.add("CAT-0: Uncategorized"); } /** @@ -259,7 +259,7 @@ final public class TagNameDefinition implements Comparable { */ static synchronized Set getTagNameDefinitions() { if (needsVersionUpdate()) { - updateTagDefinitions(); + updatePropertyFile(); } String tagsProperty = ModuleSettings.getConfigSetting(TAGS_SETTINGS_NAME, TAG_NAMES_SETTING_KEY); @@ -311,7 +311,7 @@ final public class TagNameDefinition implements Comparable { /** * Updates the Tag Definition file to the current format. */ - private static void updateTagDefinitions() { + private static void updatePropertyFile() { Integer version = getPropertyFileVersion(); List definitions = new ArrayList<>(); @@ -355,18 +355,18 @@ final public class TagNameDefinition implements Comparable { } // Remove the standard and Project VIC tags from the list - List tagStrings = new ArrayList<>(); + List tagStringsToKeep = new ArrayList<>(); List standardTags = getStandardTagNames(); for (TagNameDefinition def : definitions) { if (!standardTags.contains(def.getDisplayName()) - && !OLD_CATEGORY_TAG_NAMES.contains(def.getDisplayName())) { - tagStrings.add(def.toSettingsFormat()); + && !PROJECT_VIC_NAMES_NO_LONGER_USED.contains(def.getDisplayName())) { + tagStringsToKeep.add(def.toSettingsFormat()); } } // Write out the version and the new tag list. ModuleSettings.setConfigSetting(TAGS_SETTINGS_NAME, TAG_SETTING_VERSION_KEY, Integer.toString(TAG_SETTINGS_VERSION)); - ModuleSettings.setConfigSetting(TAGS_SETTINGS_NAME, TAG_NAMES_SETTING_KEY, String.join(";", tagStrings)); + ModuleSettings.setConfigSetting(TAGS_SETTINGS_NAME, TAG_NAMES_SETTING_KEY, String.join(";", tagStringsToKeep)); } /** diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/services/TagSetDefinition.java b/Core/src/org/sleuthkit/autopsy/casemodule/services/TagSetDefinition.java index ad11becdff..6303e2736e 100755 --- a/Core/src/org/sleuthkit/autopsy/casemodule/services/TagSetDefinition.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/services/TagSetDefinition.java @@ -88,7 +88,7 @@ final public class TagSetDefinition { } /** - * Returns a list of the defined TagSet objects. + * Returns a list of configured TagSets (from the user's config folder) * * @return A list of TagSetDefinition objects or empty list if none were * found. diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/services/TagsManager.java b/Core/src/org/sleuthkit/autopsy/casemodule/services/TagsManager.java index 1a9b2e792d..4bb813a779 100755 --- a/Core/src/org/sleuthkit/autopsy/casemodule/services/TagsManager.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/services/TagsManager.java @@ -55,7 +55,9 @@ public class TagsManager implements Closeable { private static final Logger LOGGER = Logger.getLogger(TagsManager.class.getName()); private final SleuthkitCase caseDb; - private static String DEFAULT_TAG_SET_NAME = "Project VIC"; + // NOTE: This name is also hard coded in Image Gallery and Projet Vic module. + // They need to stay in sync + private static String PROJECT_VIC_TAG_SET_NAME = "Project VIC"; private static final Object lock = new Object(); @@ -196,7 +198,7 @@ public class TagsManager implements Closeable { try { List tagSetList = Case.getCurrentCaseThrows().getSleuthkitCase().getTaggingManager().getTagSets(); for (TagSet tagSet : tagSetList) { - if (tagSet.getName().equals(DEFAULT_TAG_SET_NAME)) { + if (tagSet.getName().equals(PROJECT_VIC_TAG_SET_NAME)) { for (TagName tagName : tagSet.getTagNames()) { tagList.add(tagName.getDisplayName()); } @@ -237,7 +239,7 @@ public class TagsManager implements Closeable { } /** - * Creates a new TagSetDefinition file. + * Creates a new TagSetDefinition file that will be used for future cases * * @param tagSetDef The tag set definition. * @@ -258,23 +260,26 @@ public class TagsManager implements Closeable { TagsManager(SleuthkitCase caseDb) { this.caseDb = caseDb; - // Add standard tags and the Project VIC default tag set and tags. + // Add standard tags and any configured tag sets. TaggingManager taggingMgr = caseDb.getTaggingManager(); try { - List setList = taggingMgr.getTagSets(); - if (setList.isEmpty()) { + List tagSetsInCase = taggingMgr.getTagSets(); + if (tagSetsInCase.isEmpty()) { + + // add the standard tag names for (TagNameDefinition def : TagNameDefinition.getStandardTagNameDefinitions()) { caseDb.addOrUpdateTagName(def.getDisplayName(), def.getDescription(), def.getColor(), def.getKnownStatus()); } - //Assume new case and add tag sets + + //Assume new case and add all tag sets for (TagSetDefinition setDef : TagSetDefinition.readTagSetDefinitions()) { - List tagNameList = new ArrayList<>(); + List tagNamesInSet = new ArrayList<>(); for (TagNameDefinition tagNameDef : setDef.getTagNameDefinitions()) { - tagNameList.add(caseDb.addOrUpdateTagName(tagNameDef.getDisplayName(), tagNameDef.getDescription(), tagNameDef.getColor(), tagNameDef.getKnownStatus())); + tagNamesInSet.add(caseDb.addOrUpdateTagName(tagNameDef.getDisplayName(), tagNameDef.getDescription(), tagNameDef.getColor(), tagNameDef.getKnownStatus())); } - if (!tagNameList.isEmpty()) { - taggingMgr.addTagSet(setDef.getName(), tagNameList); + if (!tagNamesInSet.isEmpty()) { + taggingMgr.addTagSet(setDef.getName(), tagNamesInSet); } } } diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/DhsImageCategory.java b/Core/src/org/sleuthkit/autopsy/datamodel/DhsImageCategory.java index 50974519c7..4d7f87389b 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/DhsImageCategory.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/DhsImageCategory.java @@ -30,6 +30,7 @@ import org.openide.util.NbBundle; /** * Enum to represent the six categories in the DHS image categorization scheme. + * NOTE: This appears to not be used anywhere anymore after the ImageGallery refactoring */ @NbBundle.Messages({ "Category.one=CAT-1: Child Exploitation (Illegal)", From 4ea715e2350e2927196c78d10a1780df42429254 Mon Sep 17 00:00:00 2001 From: Kelly Kelly Date: Mon, 1 Feb 2021 17:07:56 -0500 Subject: [PATCH 08/25] Fixed WEB_COOKIE time attributes --- .../autopsy/recentactivity/Chromium.java | 4 ++-- .../autopsy/recentactivity/Extract.java | 18 ++++++++++++++---- .../autopsy/recentactivity/ExtractEdge.java | 4 ++-- .../autopsy/recentactivity/ExtractSafari.java | 4 ++-- .../autopsy/recentactivity/Firefox.java | 4 ++-- 5 files changed, 22 insertions(+), 12 deletions(-) diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Chromium.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Chromium.java index ef91cee067..17f09c81b1 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Chromium.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Chromium.java @@ -2,7 +2,7 @@ * * Autopsy Forensic Browser * - * Copyright 2012-2020 Basis Technology Corp. + * Copyright 2012-2021 Basis Technology Corp. * * Copyright 2012 42six Solutions. * @@ -480,7 +480,7 @@ class Chromium extends Extract { bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL, RecentActivityExtracterModuleFactory.getModuleName(), ((result.get("host_key").toString() != null) ? result.get("host_key").toString() : ""))); //NON-NLS - bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME, + bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_ACCESSED, RecentActivityExtracterModuleFactory.getModuleName(), (Long.valueOf(result.get("last_access_utc").toString()) / 1000000) - Long.valueOf("11644473600"))); //NON-NLS diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Extract.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Extract.java index ebf483b5d0..fa23d8564f 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Extract.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Extract.java @@ -2,7 +2,7 @@ * * Autopsy Forensic Browser * - * Copyright 2012-2019 Basis Technology Corp. + * Copyright 2012-2021 Basis Technology Corp. * * Copyright 2012 42six Solutions. * Contact: aebadirad 42six com @@ -325,17 +325,27 @@ abstract class Extract { * @return List of BlackboarAttributes for the passed in attributes */ protected Collection createCookieAttributes(String url, - Long creationTime, String name, String value, String programName, String domain) { + Long creationTime, Long accessTime, Long endTime, String name, String value, String programName, String domain) { Collection bbattributes = new ArrayList<>(); bbattributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_URL, RecentActivityExtracterModuleFactory.getModuleName(), (url != null) ? url : "")); //NON-NLS - if (creationTime != null) { - bbattributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME, + if (creationTime != null && creationTime != 0) { + bbattributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME_CREATED, RecentActivityExtracterModuleFactory.getModuleName(), creationTime)); } + + if (accessTime != null && accessTime != 0) { + bbattributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME_ACCESSED, + RecentActivityExtracterModuleFactory.getModuleName(), accessTime)); + } + + if(endTime != null && endTime != 0) { + bbattributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME_END, + RecentActivityExtracterModuleFactory.getModuleName(), endTime)); + } bbattributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_NAME, RecentActivityExtracterModuleFactory.getModuleName(), diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractEdge.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractEdge.java index 133764f427..7ed9cce966 100755 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractEdge.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractEdge.java @@ -2,7 +2,7 @@ * * Autopsy Forensic Browser * - * Copyright 2019-2020 Basis Technology Corp. + * Copyright 2019-2021 Basis Technology Corp. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -649,7 +649,7 @@ final class ExtractEdge extends Extract { String url = flipDomain(domain); BlackboardArtifact bbart = origFile.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_COOKIE); - bbart.addAttributes(createCookieAttributes(url, ftime, name, value, this.getName(), NetworkUtils.extractDomain(url))); + bbart.addAttributes(createCookieAttributes(url, null, ftime, null, name, value, this.getName(), NetworkUtils.extractDomain(url))); return bbart; } diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractSafari.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractSafari.java index c11e4fde22..d0e6f7fe85 100755 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractSafari.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractSafari.java @@ -2,7 +2,7 @@ * * Autopsy Forensic Browser * - * Copyright 2019 Basis Technology Corp. + * Copyright 2019-2021 Basis Technology Corp. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -548,7 +548,7 @@ final class ExtractSafari extends Extract { Cookie cookie = iter.next(); BlackboardArtifact bbart = origFile.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_COOKIE); - bbart.addAttributes(createCookieAttributes(cookie.getURL(), cookie.getCreationDate(), cookie.getName(), cookie.getValue(), this.getName(), NetworkUtils.extractDomain(cookie.getURL()))); + bbart.addAttributes(createCookieAttributes(cookie.getURL(), cookie.getCreationDate(), null, cookie.getExpirationDate(), cookie.getName(), cookie.getValue(), this.getName(), NetworkUtils.extractDomain(cookie.getURL()))); bbartifacts.add(bbart); } } diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Firefox.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Firefox.java index e0bf5c10b8..79800ff5e2 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Firefox.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Firefox.java @@ -2,7 +2,7 @@ * * Autopsy Forensic Browser * - * Copyright 2012-2020 Basis Technology Corp. + * Copyright 2012-2021 Basis Technology Corp. * * Copyright 2012 42six Solutions. * Contact: aebadirad 42six com @@ -395,7 +395,7 @@ class Firefox extends Extract { bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL, RecentActivityExtracterModuleFactory.getModuleName(), ((host != null) ? host : ""))); //NON-NLS - bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME, + bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_ACCESSED, RecentActivityExtracterModuleFactory.getModuleName(), (Long.valueOf(result.get("lastAccessed").toString())))); //NON-NLS bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_NAME, From b3f6dc0e1df6670fae17137f4c5768a5c966b630 Mon Sep 17 00:00:00 2001 From: Eugene Livis Date: Mon, 1 Feb 2021 17:21:31 -0500 Subject: [PATCH 09/25] Bug fixes, improved logging, added synchronization --- .../autopsy/keywordsearch/Bundle.properties | 2 +- .../keywordsearch/Bundle.properties-MERGED | 6 +- .../autopsy/keywordsearch/Server.java | 72 ++++++------------- .../keywordsearch/SolrSearchService.java | 16 ++++- 4 files changed, 41 insertions(+), 55 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties index b4e2850586..d2ab5f76c7 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties @@ -187,7 +187,7 @@ Server.query.exception.msg=Error running query: {0} Server.query2.exception.msg=Error running query: {0} Server.queryTerms.exception.msg=Error running terms query: {0} Server.connect.exception.msg=Failed to connect to Solr server: {0} -Server.openCore.exception.msg=Keyword search service not yet running +Server.openCore.exception.msg=Local keyword search service not yet running Server.openCore.exception.cantOpen.msg=Could not create or open index Server.openCore.exception.noIndexDir.msg=Index directory could not be created or is missing Server.request.exception.exception.msg=Could not issue Solr request diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED index 88f0d4628a..02244f9477 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED @@ -37,7 +37,7 @@ KeywordSearchResultFactory.createNodeForKey.noResultsFound.text=No results found KeywordSearchResultFactory.query.exception.msg=Could not perform the query OpenIDE-Module-Display-Category=Ingest Module -OpenIDE-Module-Long-Description=Keyword Search ingest module.\n\nThe module indexes files found in the disk image at ingest time.\nIt then periodically runs the search on the indexed files using one or more keyword lists (containing pure words and/or regular expressions) and posts results.\n\n\The module also contains additional tools integrated in the main GUI, such as keyword list configuration, keyword search bar in the top-right corner, extracted text viewer and search results viewer showing highlighted keywords found. +OpenIDE-Module-Long-Description=Keyword Search ingest module.\n\nThe module indexes files found in the disk image at ingest time.\nIt then periodically runs the search on the indexed files using one or more keyword lists (containing pure words and/or regular expressions) and posts results.\n\nThe module also contains additional tools integrated in the main GUI, such as keyword list configuration, keyword search bar in the top-right corner, extracted text viewer and search results viewer showing highlighted keywords found. OpenIDE-Module-Name=KeywordSearch OptionsCategory_Name_KeywordSearchOptions=Keyword Search OptionsCategory_Keywords_KeywordSearchOptions=Keyword Search @@ -340,10 +340,14 @@ SolrSearch.checkingForLatestIndex.msg=Looking for text index with latest Solr an SolrSearch.complete.msg=Text index successfully opened SolrSearch.creatingNewIndex.msg=Creating new text index SolrSearch.findingIndexes.msg=Looking for existing text index directories +# {0} - futureVersion +# {1} - currentVersion +SolrSearch.futureIndexVersion.msg=The text index for the case is for Solr {0}. This version of Autopsy is compatible with Solr {1}. SolrSearch.indentifyingIndex.msg=Identifying text index to use SolrSearch.lookingForMetadata.msg=Looking for text index metadata file SolrSearch.openCore.msg=Opening text index. For large cases this may take several minutes. SolrSearch.readingIndexes.msg=Reading text index metadata file +SolrSearch.unableToFindIndex.msg=Unable to find index that can be used for this case # {0} - index folder path SolrSearchService.exceptionMessage.failedToDeleteIndexFiles=Failed to delete text index files at {0} SolrSearchService.exceptionMessage.noCurrentSolrCore=IndexMetadata did not contain a current Solr core so could not delete the case diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java index a27eb4db78..56f3a5bfc1 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java @@ -239,6 +239,8 @@ public class Server { private static final String CORE_PROPERTIES = "core.properties"; private static final boolean DEBUG = false;//(Version.getBuildType() == Version.Type.DEVELOPMENT); private static final int NUM_COLLECTION_CREATION_RETRIES = 5; + private static final int NUM_EMBEDDED_SERVER_RETRIES = 12; // attempt to connect to embedded Solr server for 1 minute + private static final int EMBEDDED_SERVER_RETRY_WAIT_SEC = 5; public enum CORE_EVT_STATES { @@ -269,6 +271,8 @@ public class Server { */ Server() { initSettings(); + + localSolrServer = getSolrClient("http://localhost:" + localSolrServerPort + "/solr"); serverAction = new ServerAction(); File solr8Folder = InstalledFileLocator.getDefault().locate("solr", Server.class.getPackage().getName(), false); //NON-NLS @@ -664,7 +668,7 @@ public class Server { */ @NbBundle.Messages({ "Server.status.failed.msg=Local Solr server did not respond to status request. This may be because the server failed to start or is taking too long to initialize.",}) - void startLocalSolr(SOLR_VERSION version) throws KeywordSearchModuleException, SolrServerNoPortException, SolrServerException { + synchronized void startLocalSolr(SOLR_VERSION version) throws KeywordSearchModuleException, SolrServerNoPortException, SolrServerException { if (isLocalSolrRunning()) { if (localServerVersion.equals(version)) { @@ -712,7 +716,7 @@ public class Server { logger.log(Level.INFO, "Starting Solr 8 server"); //NON-NLS localSolrFolder = InstalledFileLocator.getDefault().locate("solr", Server.class.getPackage().getName(), false); //NON-NLS curSolrProcess = runLocalSolr8ControlCommand(new ArrayList<>(Arrays.asList("start", "-p", //NON-NLS - Integer.toString(localSolrServerPort)))); //NON-NLS + Integer.toString(localSolrServerPort)))); //NON-NLS } else { // solr4 localSolrFolder = InstalledFileLocator.getDefault().locate("solr4", Server.class.getPackage().getName(), false); //NON-NLS @@ -721,11 +725,10 @@ public class Server { Arrays.asList("-Dbootstrap_confdir=../solr/configsets/AutopsyConfig/conf", //NON-NLS "-Dcollection.configName=AutopsyConfig"))); //NON-NLS } - + // Wait for the Solr server to start and respond to a statusRequest request. - for (int numRetries = 0; numRetries < 6; numRetries++) { + for (int numRetries = 0; numRetries < NUM_EMBEDDED_SERVER_RETRIES; numRetries++) { if (isLocalSolrRunning()) { - localSolrServer = getSolrClient("http://localhost:" + localSolrServerPort + "/solr"); final List pids = this.getSolrPIDs(); logger.log(Level.INFO, "New Solr process PID: {0}", pids); //NON-NLS return; @@ -734,7 +737,7 @@ public class Server { // Local Solr server did not respond so we sleep for // 5 seconds before trying again. try { - TimeUnit.SECONDS.sleep(5); + TimeUnit.SECONDS.sleep(EMBEDDED_SERVER_RETRY_WAIT_SEC); } catch (InterruptedException ex) { logger.log(Level.WARNING, "Timer interrupted"); //NON-NLS } @@ -1875,13 +1878,22 @@ public class Server { * @throws IOException */ private void connectToEmbeddedSolrServer() throws SolrServerException, IOException { - HttpSolrClient solrServer = getSolrClient("http://localhost:" + localSolrServerPort + "/solr"); TimingMetric metric = HealthMonitor.getTimingMetric("Solr: Connectivity check"); - CoreAdminRequest.getStatus(null, solrServer); + CoreAdminRequest.getStatus(null, localSolrServer); HealthMonitor.submitTimingMetric(metric); } - + /** + * Attempts to connect to the given Solr server, which is running in + * SoulrCloud mode. This API does not work for the local Solr which is NOT + * running in SolrCloud mode. + * + * @param host Host name of the remote Solr server + * @param port Port of the remote Solr server + * + * @throws SolrServerException + * @throws IOException + */ void connectToSolrServer(String host, String port) throws SolrServerException, IOException { try (HttpSolrClient solrServer = getSolrClient("http://" + host + ":" + port + "/solr")) { connectToSolrServer(solrServer); @@ -1945,47 +1957,7 @@ public class Server { throw new KeywordSearchModuleException( NbBundle.getMessage(this.getClass(), "Server.serverList.exception.msg", solrServer.getBaseURL())); } - } - - /* ELTODO leaving this for reference, will delete later - private boolean clusterStatusWithCollection(String collectionName) throws IOException, SolrServerException { - ModifiableSolrParams params = new ModifiableSolrParams(); - params.set("action", CollectionParams.CollectionAction.CLUSTERSTATUS.toString()); - params.set("collection", collectionName); - SolrRequest request = new QueryRequest(params); - request.setPath("/admin/collections"); - - NamedList statusResponse; - try { - statusResponse = currentSolrServer.request(request); - } catch (RemoteSolrException ex) { - // collection doesn't exist - return false; - } - - if (statusResponse == null) { - logger.log(Level.SEVERE, "Collections response should not be null"); //NON-NLS - return false; - } - - NamedList cluster = (NamedList) statusResponse.get("cluster"); - if (cluster == null) { - logger.log(Level.SEVERE, "Cluster should not be null"); //NON-NLS - return false; - } - NamedList collections = (NamedList) cluster.get("collections"); - if (cluster == null) { - logger.log(Level.SEVERE, "Collections should not be null in cluster state"); //NON-NLS - return false; - } - if (collections.size() == 0) { - logger.log(Level.SEVERE, "Collections should not be empty in cluster state"); //NON-NLS - return false; - } - - Object collection = collections.get(collectionName); - return (collection != null); - }*/ + } class Collection { diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java index 920ac1aaba..be3794c60e 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java @@ -269,6 +269,9 @@ public class SolrSearchService implements KeywordSearchService, AutopsyService { "SolrSearch.checkingForLatestIndex.msg=Looking for text index with latest Solr and schema version", "SolrSearch.indentifyingIndex.msg=Identifying text index to use", "SolrSearch.openCore.msg=Opening text index. For large cases this may take several minutes.", + "# {0} - futureVersion", "# {1} - currentVersion", + "SolrSearch.futureIndexVersion.msg=The text index for the case is for Solr {0}. This version of Autopsy is compatible with Solr {1}.", + "SolrSearch.unableToFindIndex.msg=Unable to find index that can be used for this case", "SolrSearch.complete.msg=Text index successfully opened"}) public void openCaseResources(CaseContext context) throws AutopsyServiceException { if (context.cancelRequested()) { @@ -320,8 +323,13 @@ public class SolrSearchService implements KeywordSearchService, AutopsyService { progress.progress(Bundle.SolrSearch_indentifyingIndex_msg(), progressUnitsCompleted); Index indexToUse = IndexFinder.identifyIndexToUse(indexes); if (indexToUse == null) { - // unable to find index that can be used - throw new AutopsyServiceException("Unable to find index that can be used for this case"); + // unable to find index that can be used. check if the available index is for a "future" version of Solr, + // i.e. the user is using an "old/legacy" version of Autopsy to open cases created by later versions of Autopsy. + String futureIndexVersion = IndexFinder.isFutureIndexPresent(indexes); + if (!futureIndexVersion.isEmpty()) { + throw new AutopsyServiceException(Bundle.SolrSearch_futureIndexVersion_msg(futureIndexVersion, IndexFinder.getCurrentSolrVersion())); + } + throw new AutopsyServiceException(Bundle.SolrSearch_unableToFindIndex_msg()); } if (context.cancelRequested()) { @@ -393,7 +401,9 @@ public class SolrSearchService implements KeywordSearchService, AutopsyService { throw new AutopsyServiceException(String.format("Failed to close core for %s", context.getCase().getCaseDirectory()), ex); } - context.getCase().getSleuthkitCase().unregisterForEvents(this); + if (context.getCase().getSleuthkitCase() != null) { + context.getCase().getSleuthkitCase().unregisterForEvents(this); + } } /** From bd20d16953f7c4b0a13745315d139cf40861f2b7 Mon Sep 17 00:00:00 2001 From: Eugene Livis Date: Mon, 1 Feb 2021 17:29:36 -0500 Subject: [PATCH 10/25] Minor --- .../autopsy/keywordsearch/SolrSearchService.java | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java index be3794c60e..a031a160f1 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java @@ -269,9 +269,6 @@ public class SolrSearchService implements KeywordSearchService, AutopsyService { "SolrSearch.checkingForLatestIndex.msg=Looking for text index with latest Solr and schema version", "SolrSearch.indentifyingIndex.msg=Identifying text index to use", "SolrSearch.openCore.msg=Opening text index. For large cases this may take several minutes.", - "# {0} - futureVersion", "# {1} - currentVersion", - "SolrSearch.futureIndexVersion.msg=The text index for the case is for Solr {0}. This version of Autopsy is compatible with Solr {1}.", - "SolrSearch.unableToFindIndex.msg=Unable to find index that can be used for this case", "SolrSearch.complete.msg=Text index successfully opened"}) public void openCaseResources(CaseContext context) throws AutopsyServiceException { if (context.cancelRequested()) { @@ -323,13 +320,8 @@ public class SolrSearchService implements KeywordSearchService, AutopsyService { progress.progress(Bundle.SolrSearch_indentifyingIndex_msg(), progressUnitsCompleted); Index indexToUse = IndexFinder.identifyIndexToUse(indexes); if (indexToUse == null) { - // unable to find index that can be used. check if the available index is for a "future" version of Solr, - // i.e. the user is using an "old/legacy" version of Autopsy to open cases created by later versions of Autopsy. - String futureIndexVersion = IndexFinder.isFutureIndexPresent(indexes); - if (!futureIndexVersion.isEmpty()) { - throw new AutopsyServiceException(Bundle.SolrSearch_futureIndexVersion_msg(futureIndexVersion, IndexFinder.getCurrentSolrVersion())); - } - throw new AutopsyServiceException(Bundle.SolrSearch_unableToFindIndex_msg()); + // unable to find index that can be used + throw new AutopsyServiceException("Unable to find index that can be used for this case"); } if (context.cancelRequested()) { From 65050b20fba7500ab3c13a55e45d32bc4d657353 Mon Sep 17 00:00:00 2001 From: Eugene Livis Date: Mon, 1 Feb 2021 17:32:05 -0500 Subject: [PATCH 11/25] Minor --- .../autopsy/keywordsearch/Bundle.properties-MERGED | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED index 02244f9477..88f0d4628a 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED @@ -37,7 +37,7 @@ KeywordSearchResultFactory.createNodeForKey.noResultsFound.text=No results found KeywordSearchResultFactory.query.exception.msg=Could not perform the query OpenIDE-Module-Display-Category=Ingest Module -OpenIDE-Module-Long-Description=Keyword Search ingest module.\n\nThe module indexes files found in the disk image at ingest time.\nIt then periodically runs the search on the indexed files using one or more keyword lists (containing pure words and/or regular expressions) and posts results.\n\nThe module also contains additional tools integrated in the main GUI, such as keyword list configuration, keyword search bar in the top-right corner, extracted text viewer and search results viewer showing highlighted keywords found. +OpenIDE-Module-Long-Description=Keyword Search ingest module.\n\nThe module indexes files found in the disk image at ingest time.\nIt then periodically runs the search on the indexed files using one or more keyword lists (containing pure words and/or regular expressions) and posts results.\n\n\The module also contains additional tools integrated in the main GUI, such as keyword list configuration, keyword search bar in the top-right corner, extracted text viewer and search results viewer showing highlighted keywords found. OpenIDE-Module-Name=KeywordSearch OptionsCategory_Name_KeywordSearchOptions=Keyword Search OptionsCategory_Keywords_KeywordSearchOptions=Keyword Search @@ -340,14 +340,10 @@ SolrSearch.checkingForLatestIndex.msg=Looking for text index with latest Solr an SolrSearch.complete.msg=Text index successfully opened SolrSearch.creatingNewIndex.msg=Creating new text index SolrSearch.findingIndexes.msg=Looking for existing text index directories -# {0} - futureVersion -# {1} - currentVersion -SolrSearch.futureIndexVersion.msg=The text index for the case is for Solr {0}. This version of Autopsy is compatible with Solr {1}. SolrSearch.indentifyingIndex.msg=Identifying text index to use SolrSearch.lookingForMetadata.msg=Looking for text index metadata file SolrSearch.openCore.msg=Opening text index. For large cases this may take several minutes. SolrSearch.readingIndexes.msg=Reading text index metadata file -SolrSearch.unableToFindIndex.msg=Unable to find index that can be used for this case # {0} - index folder path SolrSearchService.exceptionMessage.failedToDeleteIndexFiles=Failed to delete text index files at {0} SolrSearchService.exceptionMessage.noCurrentSolrCore=IndexMetadata did not contain a current Solr core so could not delete the case From ac5b8cbb44c114ab8ff4dd7450ed2811526ff120 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Tue, 2 Feb 2021 13:52:40 -0500 Subject: [PATCH 12/25] updates for embedded solr on os x --- .gitattributes | 6 ++ .../autopsy/keywordsearch/Server.java | 60 +++++++++++++++++++ unix_setup.sh | 3 + 3 files changed, 69 insertions(+) diff --git a/.gitattributes b/.gitattributes index cd5271c982..8749e5dadb 100644 --- a/.gitattributes +++ b/.gitattributes @@ -13,3 +13,9 @@ Doxyfile text *.py text diff=python *.pl text + +# ensure solr scripts that are bash scripts not ending with.sh are lf instead of crlf +/KeywordSearch/solr/bin/autopsy-solr eol=lf +/KeywordSearch/solr/bin/init.d/solr eol=lf +/KeywordSearch/solr/bin/post eol=lf +/KeywordSearch/solr/bin/solr eol=lf \ No newline at end of file diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java index a27eb4db78..8f4abc8d90 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java @@ -31,6 +31,7 @@ import java.io.InputStreamReader; import java.io.OutputStream; import java.io.OutputStreamWriter; import java.net.ConnectException; +import java.net.DatagramSocket; import java.net.ServerSocket; import java.net.SocketException; import java.nio.charset.Charset; @@ -767,6 +768,23 @@ public class Server { * @param port the port to check for availability */ static boolean isPortAvailable(int port) { + final String osName = PlatformUtil.getOSName().toLowerCase(); + if (osName != null && osName.toLowerCase().startsWith("mac")) { + return isPortAvailableOSX(port); + } else { + return isPortAvailableDefault(port); + } + } + + /** + * Checks to see if a specific port is available. + * + * NOTE: This is used on non-OS X systems as of right now but could be + * replaced with the OS X version. + * + * @param port the port to check for availability + */ + static boolean isPortAvailableDefault(int port) { ServerSocket ss = null; try { @@ -792,6 +810,48 @@ public class Server { return false; } + /** + * Checks to see if a specific port is available. + * + * NOTE: This is only used on OSX for now, but could replace default + * implementation in the future. + * + * @param port The port to check for availability. + * @throws IllegalArgumentException If port is outside range of possible ports. + */ + static boolean isPortAvailableOSX(int port) { + // implementation taken from https://stackoverflow.com/a/435579 + if (port < 1 || port > 65535) { + throw new IllegalArgumentException("Invalid start port: " + port); + } + + ServerSocket ss = null; + DatagramSocket ds = null; + try { + ss = new ServerSocket(port); + ss.setReuseAddress(true); + ds = new DatagramSocket(port); + ds.setReuseAddress(true); + return true; + } catch (IOException e) { + } finally { + if (ds != null) { + ds.close(); + } + + if (ss != null) { + try { + ss.close(); + } catch (IOException e) { + /* should not be thrown */ + } + } + } + + return false; + } + + /** * Changes the current solr server port. Only call this after available. * diff --git a/unix_setup.sh b/unix_setup.sh index 93a5eb4bea..40b1d858b4 100644 --- a/unix_setup.sh +++ b/unix_setup.sh @@ -76,6 +76,9 @@ fi chmod u+x autopsy/markmckinnon/Export* chmod u+x autopsy/markmckinnon/parse* +# allow solr dependencies to execute +chmod -R u+x autopsy/solr/bin + # make sure it is executable chmod u+x bin/autopsy From 81c65a8c014d20a65d5253d6deb0eea47f96be74 Mon Sep 17 00:00:00 2001 From: Mark McKinnon Date: Tue, 2 Feb 2021 17:25:05 -0500 Subject: [PATCH 13/25] Update LeappFileProcessor.java Normalize Domain name to TSK/CR standards --- .../leappanalyzers/LeappFileProcessor.java | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/LeappFileProcessor.java b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/LeappFileProcessor.java index c6fdb6ae1e..5318ddf85b 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/LeappFileProcessor.java +++ b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/LeappFileProcessor.java @@ -59,6 +59,7 @@ import org.sleuthkit.autopsy.casemodule.Case; import static org.sleuthkit.autopsy.casemodule.Case.getCurrentCase; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; import org.sleuthkit.autopsy.casemodule.services.FileManager; +import org.sleuthkit.autopsy.coreutils.NetworkUtils; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.PlatformUtil; import org.sleuthkit.autopsy.ingest.IngestModule.IngestModuleException; @@ -379,7 +380,9 @@ public final class LeappFileProcessor { return Collections.emptyList(); } - BlackboardAttribute attr = (value == null) ? null : getAttribute(colAttr.getAttributeType(), value, fileName); + String formattedValue = formatValueBasedOnAttrType(colAttr, value); + + BlackboardAttribute attr = (value == null) ? null : getAttribute(colAttr.getAttributeType(), formattedValue, fileName); if (attr == null) { logger.log(Level.WARNING, String.format("Blackboard attribute could not be parsed column %s at line %d in file %s. Omitting row.", colAttr.getColumnName(), lineNum, fileName)); return Collections.emptyList(); @@ -394,6 +397,21 @@ public final class LeappFileProcessor { return attrsToRet; } + /** + * Check type of attribute and possibly format string based on it. + * + * @param colAttr Column Attribute information + * @param value string to be formatted + * @return formatted string based on attribute type if no attribute type found then return original string + */ + private String formatValueBasedOnAttrType(TsvColumn colAttr, String value) { + if (colAttr.getAttributeType().getTypeName().equals("TSK_DOMAIN")) { + return NetworkUtils.extractDomain(value); + } + + return value; + } + /** * The format of time stamps in tsv. */ From e86e5fc58ec8369ff3d7e39a9052a9b0801fa7f6 Mon Sep 17 00:00:00 2001 From: Mark McKinnon Date: Tue, 2 Feb 2021 18:23:31 -0500 Subject: [PATCH 14/25] Check if aLeapp/iLeapp is on 32bit system and fail if it is Check is aLeapp/iLeapp is on a 32bit system and fail quietly if it is. --- .../leappanalyzers/ALeappAnalyzerIngestModule.java | 10 ++++++++++ .../autopsy/modules/leappanalyzers/Bundle.properties | 4 +++- .../modules/leappanalyzers/Bundle.properties-MERGED | 2 ++ .../leappanalyzers/ILeappAnalyzerIngestModule.java | 10 ++++++++++ 4 files changed, 25 insertions(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/ALeappAnalyzerIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/ALeappAnalyzerIngestModule.java index acb0acbc7d..6dad1ceda6 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/ALeappAnalyzerIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/ALeappAnalyzerIngestModule.java @@ -65,6 +65,7 @@ public class ALeappAnalyzerIngestModule implements DataSourceIngestModule { private static final Logger logger = Logger.getLogger(ALeappAnalyzerIngestModule.class.getName()); private static final String MODULE_NAME = ALeappAnalyzerModuleFactory.getModuleName(); + private final IngestServices services = IngestServices.getInstance(); private static final String ALEAPP = "aLeapp"; //NON-NLS private static final String ALEAPP_FS = "fs_"; //NON-NLS @@ -123,6 +124,15 @@ public class ALeappAnalyzerIngestModule implements DataSourceIngestModule { @Override public ProcessResult process(Content dataSource, DataSourceIngestModuleProgress statusHelper) { + if (false == PlatformUtil.is64BitOS()) { + IngestMessage ingestMessage = IngestMessage.createErrorMessage(ALeappAnalyzerModuleFactory.getModuleName(), + NbBundle.getMessage(this.getClass(), "AleappAnalyzerIngestModule.not.64.bit.os"), + NbBundle.getMessage(this.getClass(), "AleappAnalyzerIngestModule.not.64.bit.os")); + services.postMessage(ingestMessage); + logger.log(Level.INFO, "aLeapp will not run on 32bit operating system"); + return ProcessResult.OK; + } + Case currentCase = Case.getCurrentCase(); Path tempOutputPath = Paths.get(currentCase.getTempDirectory(), ALEAPP, ALEAPP_FS + dataSource.getId()); try { diff --git a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/Bundle.properties b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/Bundle.properties index 4236fad631..06d30c03a5 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/Bundle.properties @@ -2,7 +2,9 @@ ILeappAnalyzerIngestModule.init.exception.msg=Unable to find {0}. ILeappAnalyzerIngestModule.processing.file=Processing file {0} ILeappAnalyzerIngestModule.parsing.file=Parsing file {0} ILeappAnalyzerIngestModule.processing.filesystem=Processing filesystem +IleappAnalyzerIngestModule.not.64.bit.os=iLeapp will not run on 32bit operating system ALeappAnalyzerIngestModule.init.exception.msg=Unable to find {0}. ALeappAnalyzerIngestModule.processing.file=Processing file {0} ALeappAnalyzerIngestModule.parsing.file=Parsing file {0} -ALeappAnalyzerIngestModule.processing.filesystem=Processing filesystem \ No newline at end of file +ALeappAnalyzerIngestModule.processing.filesystem=Processing filesystem +AleappAnalyzerIngestModule.not.64.bit.os=aLeapp will not run on 32bit operating system \ No newline at end of file diff --git a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/Bundle.properties-MERGED index b23a428107..123986b7d6 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/Bundle.properties-MERGED @@ -22,10 +22,12 @@ ILeappAnalyzerIngestModule.init.exception.msg=Unable to find {0}. ILeappAnalyzerIngestModule.processing.file=Processing file {0} ILeappAnalyzerIngestModule.parsing.file=Parsing file {0} ILeappAnalyzerIngestModule.processing.filesystem=Processing filesystem +IleappAnalyzerIngestModule.not.64.bit.os=iLeapp will not run on 32bit operating system ALeappAnalyzerIngestModule.init.exception.msg=Unable to find {0}. ALeappAnalyzerIngestModule.processing.file=Processing file {0} ALeappAnalyzerIngestModule.parsing.file=Parsing file {0} ALeappAnalyzerIngestModule.processing.filesystem=Processing filesystem +AleappAnalyzerIngestModule.not.64.bit.os=aLeapp will not run on 32bit operating system ILeappAnalyzerIngestModule.report.name=iLeapp Html Report ILeappAnalyzerIngestModule.requires.windows=iLeapp module requires windows. ILeappAnalyzerIngestModule.running.iLeapp=Running iLeapp diff --git a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/ILeappAnalyzerIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/ILeappAnalyzerIngestModule.java index 70e811905c..35b02dad64 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/ILeappAnalyzerIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/ILeappAnalyzerIngestModule.java @@ -65,6 +65,7 @@ public class ILeappAnalyzerIngestModule implements DataSourceIngestModule { private static final Logger logger = Logger.getLogger(ILeappAnalyzerIngestModule.class.getName()); private static final String MODULE_NAME = ILeappAnalyzerModuleFactory.getModuleName(); + private final IngestServices services = IngestServices.getInstance(); private static final String ILEAPP = "iLeapp"; //NON-NLS private static final String ILEAPP_FS = "fs_"; //NON-NLS @@ -123,6 +124,15 @@ public class ILeappAnalyzerIngestModule implements DataSourceIngestModule { @Override public ProcessResult process(Content dataSource, DataSourceIngestModuleProgress statusHelper) { + if (false == PlatformUtil.is64BitOS()) { + IngestMessage ingestMessage = IngestMessage.createErrorMessage(ILeappAnalyzerModuleFactory.getModuleName(), + NbBundle.getMessage(this.getClass(), "IleappAnalyzerIngestModule.not.64.bit.os"), + NbBundle.getMessage(this.getClass(), "IleappAnalyzerIngestModule.not.64.bit.os")); + services.postMessage(ingestMessage); + logger.log(Level.INFO, "iLeapp will not run on 32bit operating system"); + return ProcessResult.OK; + } + Case currentCase = Case.getCurrentCase(); Path tempOutputPath = Paths.get(currentCase.getTempDirectory(), ILEAPP, ILEAPP_FS + dataSource.getId()); try { From 287f01f0c2ea1541f891dbac09c5d3b980a6ec77 Mon Sep 17 00:00:00 2001 From: Mark McKinnon Date: Wed, 3 Feb 2021 10:20:42 -0500 Subject: [PATCH 15/25] Update CorrelationAttributeNormalizer.java Add sanization check in CR --- .../datamodel/CorrelationAttributeNormalizer.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeNormalizer.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeNormalizer.java index 51b9b80f84..f7d423c85d 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeNormalizer.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeNormalizer.java @@ -27,6 +27,7 @@ import java.util.Set; import org.apache.commons.lang.StringUtils; import org.apache.commons.validator.routines.DomainValidator; import org.apache.commons.validator.routines.EmailValidator; +import org.sleuthkit.autopsy.coreutils.NetworkUtils; /** * Provides functions for normalizing data by attribute type before insertion or @@ -144,11 +145,11 @@ final public class CorrelationAttributeNormalizer { private static String normalizeDomain(String data) throws CorrelationAttributeNormalizationException { DomainValidator validator = DomainValidator.getInstance(true); if (validator.isValid(data)) { - return data.toLowerCase(); + return NetworkUtils.extractDomain(data.toLowerCase()); } else { final String validIpAddressRegex = "^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$"; if (data.matches(validIpAddressRegex)) { - return data; + return NetworkUtils.extractDomain(data); } else { throw new CorrelationAttributeNormalizationException(String.format("Data was expected to be a valid domain: %s", data)); } From 60fa44c5a6995e6095f7e6b4c9cdb286ebcfe7b0 Mon Sep 17 00:00:00 2001 From: Eugene Livis Date: Wed, 3 Feb 2021 11:32:49 -0500 Subject: [PATCH 16/25] Starting SU Solr server on background thread --- .../autopsy/keywordsearch/Installer.java | 38 +++++++++++++------ .../autopsy/keywordsearch/Server.java | 4 +- 2 files changed, 29 insertions(+), 13 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Installer.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Installer.java index 04bcbf6940..126d7fa4ba 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Installer.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Installer.java @@ -18,6 +18,9 @@ */ package org.sleuthkit.autopsy.keywordsearch; +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.logging.Level; import org.apache.solr.client.solrj.SolrServerException; import org.openide.modules.ModuleInstall; @@ -39,6 +42,7 @@ class Installer extends ModuleInstall { private static final Logger logger = Logger.getLogger(Installer.class.getName()); private static final long serialVersionUID = 1L; + private static final String KWS_START_THREAD_NAME = "KWS-server-start-%d"; @Override public void restored() { @@ -46,19 +50,29 @@ class Installer extends ModuleInstall { KeywordSearchSettings.setDefaults(); final Server server = KeywordSearch.getServer(); - try { - server.start(); - } catch (SolrServerNoPortException ex) { - logger.log(Level.SEVERE, "Failed to start Keyword Search server: ", ex); //NON-NLS - if (ex.getPortNumber() == server.getLocalSolrServerPort()) { - reportPortError(ex.getPortNumber()); - } else { - reportStopPortError(ex.getPortNumber()); + + ExecutorService jobProcessingExecutor = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setNameFormat(KWS_START_THREAD_NAME).build()); + Runnable kwsStartTask = new Runnable() { + public void run() { + try { + server.start(); + } catch (SolrServerNoPortException ex) { + logger.log(Level.SEVERE, "Failed to start Keyword Search server: ", ex); //NON-NLS + if (ex.getPortNumber() == server.getLocalSolrServerPort()) { + reportPortError(ex.getPortNumber()); + } else { + reportStopPortError(ex.getPortNumber()); + } + } catch (KeywordSearchModuleException | SolrServerException ex) { + logger.log(Level.SEVERE, "Failed to start Keyword Search server: ", ex); //NON-NLS + reportInitError(ex.getMessage()); + } } - } catch (KeywordSearchModuleException | SolrServerException ex) { - logger.log(Level.SEVERE, "Failed to start Keyword Search server: ", ex); //NON-NLS - reportInitError(ex.getMessage()); - } + }; + + // start KWS service on the background thread. Currently all it does is start the embedded Solr server. + jobProcessingExecutor.submit(kwsStartTask); + jobProcessingExecutor.shutdown(); // tell executor no more work is coming } @Override diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java index a27eb4db78..732d5ed53e 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java @@ -664,11 +664,13 @@ public class Server { */ @NbBundle.Messages({ "Server.status.failed.msg=Local Solr server did not respond to status request. This may be because the server failed to start or is taking too long to initialize.",}) - void startLocalSolr(SOLR_VERSION version) throws KeywordSearchModuleException, SolrServerNoPortException, SolrServerException { + synchronized void startLocalSolr(SOLR_VERSION version) throws KeywordSearchModuleException, SolrServerNoPortException, SolrServerException { + logger.log(Level.INFO, "Starting local Solr " + version + " server"); //NON-NLS if (isLocalSolrRunning()) { if (localServerVersion.equals(version)) { // this version of local server is already running + logger.log(Level.INFO, "Local Solr " + version + " server is already running"); //NON-NLS return; } else { // wrong version of local server is running, stop it From 16bc57487f734301cda362c637bdbd1da1309b02 Mon Sep 17 00:00:00 2001 From: Kelly Kelly Date: Wed, 3 Feb 2021 13:19:39 -0500 Subject: [PATCH 17/25] Added drone module name change to ingestjobsettings list --- Core/src/org/sleuthkit/autopsy/ingest/IngestJobSettings.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobSettings.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobSettings.java index 0dbbaebb57..918c30760f 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobSettings.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobSettings.java @@ -477,6 +477,9 @@ public final class IngestJobSettings { case "Exif Parser": //NON-NLS moduleNames.add("Picture Analyzer"); //NON-NLS break; + case "Drone Analyzer": + moduleNames.add("DJI Drone Analyzer"); + break; default: moduleNames.add(name); } From 138a5f57d08c348e48bf7bfb01d3caf5c4b56148 Mon Sep 17 00:00:00 2001 From: Mark McKinnon Date: Wed, 3 Feb 2021 15:09:42 -0500 Subject: [PATCH 18/25] Put check code in start of Ingest instead of Process Put check for 32bit in start of ingest instead of process. --- .../leappanalyzers/ALeappAnalyzerIngestModule.java | 14 ++++---------- .../leappanalyzers/ILeappAnalyzerIngestModule.java | 14 ++++---------- 2 files changed, 8 insertions(+), 20 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/ALeappAnalyzerIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/ALeappAnalyzerIngestModule.java index 6dad1ceda6..3409098d95 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/ALeappAnalyzerIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/ALeappAnalyzerIngestModule.java @@ -65,7 +65,6 @@ public class ALeappAnalyzerIngestModule implements DataSourceIngestModule { private static final Logger logger = Logger.getLogger(ALeappAnalyzerIngestModule.class.getName()); private static final String MODULE_NAME = ALeappAnalyzerModuleFactory.getModuleName(); - private final IngestServices services = IngestServices.getInstance(); private static final String ALEAPP = "aLeapp"; //NON-NLS private static final String ALEAPP_FS = "fs_"; //NON-NLS @@ -93,6 +92,10 @@ public class ALeappAnalyzerIngestModule implements DataSourceIngestModule { public void startUp(IngestJobContext context) throws IngestModuleException { this.context = context; + if (false == PlatformUtil.is64BitOS()) { + throw new IngestModuleException(NbBundle.getMessage(this.getClass(), "AleappAnalyzerIngestModule.not.64.bit.os")); + } + if (false == PlatformUtil.isWindowsOS()) { throw new IngestModuleException(Bundle.ALeappAnalyzerIngestModule_requires_windows()); } @@ -124,15 +127,6 @@ public class ALeappAnalyzerIngestModule implements DataSourceIngestModule { @Override public ProcessResult process(Content dataSource, DataSourceIngestModuleProgress statusHelper) { - if (false == PlatformUtil.is64BitOS()) { - IngestMessage ingestMessage = IngestMessage.createErrorMessage(ALeappAnalyzerModuleFactory.getModuleName(), - NbBundle.getMessage(this.getClass(), "AleappAnalyzerIngestModule.not.64.bit.os"), - NbBundle.getMessage(this.getClass(), "AleappAnalyzerIngestModule.not.64.bit.os")); - services.postMessage(ingestMessage); - logger.log(Level.INFO, "aLeapp will not run on 32bit operating system"); - return ProcessResult.OK; - } - Case currentCase = Case.getCurrentCase(); Path tempOutputPath = Paths.get(currentCase.getTempDirectory(), ALEAPP, ALEAPP_FS + dataSource.getId()); try { diff --git a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/ILeappAnalyzerIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/ILeappAnalyzerIngestModule.java index 35b02dad64..339be316a9 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/ILeappAnalyzerIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/ILeappAnalyzerIngestModule.java @@ -65,7 +65,6 @@ public class ILeappAnalyzerIngestModule implements DataSourceIngestModule { private static final Logger logger = Logger.getLogger(ILeappAnalyzerIngestModule.class.getName()); private static final String MODULE_NAME = ILeappAnalyzerModuleFactory.getModuleName(); - private final IngestServices services = IngestServices.getInstance(); private static final String ILEAPP = "iLeapp"; //NON-NLS private static final String ILEAPP_FS = "fs_"; //NON-NLS @@ -93,6 +92,10 @@ public class ILeappAnalyzerIngestModule implements DataSourceIngestModule { public void startUp(IngestJobContext context) throws IngestModuleException { this.context = context; + if (false == PlatformUtil.is64BitOS()) { + throw new IngestModuleException(NbBundle.getMessage(this.getClass(), "IleappAnalyzerIngestModule.not.64.bit.os")); + } + if (false == PlatformUtil.isWindowsOS()) { throw new IngestModuleException(Bundle.ILeappAnalyzerIngestModule_requires_windows()); } @@ -124,15 +127,6 @@ public class ILeappAnalyzerIngestModule implements DataSourceIngestModule { @Override public ProcessResult process(Content dataSource, DataSourceIngestModuleProgress statusHelper) { - if (false == PlatformUtil.is64BitOS()) { - IngestMessage ingestMessage = IngestMessage.createErrorMessage(ILeappAnalyzerModuleFactory.getModuleName(), - NbBundle.getMessage(this.getClass(), "IleappAnalyzerIngestModule.not.64.bit.os"), - NbBundle.getMessage(this.getClass(), "IleappAnalyzerIngestModule.not.64.bit.os")); - services.postMessage(ingestMessage); - logger.log(Level.INFO, "iLeapp will not run on 32bit operating system"); - return ProcessResult.OK; - } - Case currentCase = Case.getCurrentCase(); Path tempOutputPath = Paths.get(currentCase.getTempDirectory(), ILEAPP, ILEAPP_FS + dataSource.getId()); try { From c5e00cc8d46f69b1f96176951a91775715c39b11 Mon Sep 17 00:00:00 2001 From: Kelly Kelly Date: Wed, 3 Feb 2021 15:11:54 -0500 Subject: [PATCH 19/25] Fixed confusing message issue --- .../autopsy/imagegallery/ImageGalleryController.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java index 95509c4c16..d9746bda88 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java @@ -839,8 +839,11 @@ public final class ImageGalleryController { Content newDataSource = (Content) event.getNewValue(); if (isListeningEnabled()) { try { - drawableDB.insertOrUpdateDataSource(newDataSource.getId(), DrawableDB.DrawableDbBuildStatusEnum.UNKNOWN); - } catch (SQLException ex) { + // status of UNKNOWN from getDataSourceDbBuildStatus means that the data source is not known + if(drawableDB.getDataSourceDbBuildStatus(newDataSource.getId()) == DrawableDB.DrawableDbBuildStatusEnum.UNKNOWN) { + drawableDB.insertOrUpdateDataSource(newDataSource.getId(), DrawableDB.DrawableDbBuildStatusEnum.UNKNOWN); + } + } catch (SQLException | TskCoreException ex) { logger.log(Level.SEVERE, String.format("Error updating datasources table (data source object ID = %d, status = %s)", newDataSource.getId(), DrawableDB.DrawableDbBuildStatusEnum.UNKNOWN.toString()), ex); //NON-NLS } } From 5aed1e565ca7ffaa6d9b0a9cea2c84a3099e7ae8 Mon Sep 17 00:00:00 2001 From: Kelly Kelly Date: Wed, 3 Feb 2021 16:09:35 -0500 Subject: [PATCH 20/25] Fixed comment --- .../sleuthkit/autopsy/imagegallery/ImageGalleryController.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java index d9746bda88..0ddded910b 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java @@ -839,7 +839,7 @@ public final class ImageGalleryController { Content newDataSource = (Content) event.getNewValue(); if (isListeningEnabled()) { try { - // status of UNKNOWN from getDataSourceDbBuildStatus means that the data source is not known + // Only add the data source if is not in the database if(drawableDB.getDataSourceDbBuildStatus(newDataSource.getId()) == DrawableDB.DrawableDbBuildStatusEnum.UNKNOWN) { drawableDB.insertOrUpdateDataSource(newDataSource.getId(), DrawableDB.DrawableDbBuildStatusEnum.UNKNOWN); } From ae7e6137a2f1a2be82ecb10dc5b85275e694ed13 Mon Sep 17 00:00:00 2001 From: Kelly Kelly Date: Wed, 3 Feb 2021 17:48:16 -0500 Subject: [PATCH 21/25] Modified change comment for clarification --- .../sleuthkit/autopsy/imagegallery/ImageGalleryController.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java index 0ddded910b..d365df4e8c 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java @@ -839,7 +839,7 @@ public final class ImageGalleryController { Content newDataSource = (Content) event.getNewValue(); if (isListeningEnabled()) { try { - // Only add the data source if is not in the database + // If the data source already exists and has a status other than UNKNOWN, don’t overwrite it. if(drawableDB.getDataSourceDbBuildStatus(newDataSource.getId()) == DrawableDB.DrawableDbBuildStatusEnum.UNKNOWN) { drawableDB.insertOrUpdateDataSource(newDataSource.getId(), DrawableDB.DrawableDbBuildStatusEnum.UNKNOWN); } From bde399ffd8e284ec0ded7df4c2aab43b86863bf7 Mon Sep 17 00:00:00 2001 From: Mark McKinnon Date: Thu, 4 Feb 2021 14:49:51 -0500 Subject: [PATCH 22/25] Update Bundle.properties fix message to user. --- .../autopsy/modules/leappanalyzers/Bundle.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/Bundle.properties b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/Bundle.properties index 06d30c03a5..5dfed94624 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/Bundle.properties @@ -2,9 +2,9 @@ ILeappAnalyzerIngestModule.init.exception.msg=Unable to find {0}. ILeappAnalyzerIngestModule.processing.file=Processing file {0} ILeappAnalyzerIngestModule.parsing.file=Parsing file {0} ILeappAnalyzerIngestModule.processing.filesystem=Processing filesystem -IleappAnalyzerIngestModule.not.64.bit.os=iLeapp will not run on 32bit operating system +IleappAnalyzerIngestModule.not.64.bit.os=iLeapp will not run on a 32bit operating system ALeappAnalyzerIngestModule.init.exception.msg=Unable to find {0}. ALeappAnalyzerIngestModule.processing.file=Processing file {0} ALeappAnalyzerIngestModule.parsing.file=Parsing file {0} ALeappAnalyzerIngestModule.processing.filesystem=Processing filesystem -AleappAnalyzerIngestModule.not.64.bit.os=aLeapp will not run on 32bit operating system \ No newline at end of file +AleappAnalyzerIngestModule.not.64.bit.os=aLeapp will not run on a 32bit operating system \ No newline at end of file From bb4f15093b162175dbe6ca14c3cb12e314dc20e0 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Fri, 5 Feb 2021 11:05:54 -0500 Subject: [PATCH 23/25] file types implementation --- .../autopsy/datamodel/FileTypes.java | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/FileTypes.java b/Core/src/org/sleuthkit/autopsy/datamodel/FileTypes.java index aa1d0a04ec..1a7abab0f9 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/FileTypes.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/FileTypes.java @@ -20,6 +20,7 @@ package org.sleuthkit.autopsy.datamodel; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.List; import java.util.Observable; import java.util.Observer; @@ -34,6 +35,8 @@ import org.openide.util.Lookup; import org.openide.util.NbBundle; import org.openide.util.lookup.Lookups; import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.datamodel.AnalysisResult; +import org.sleuthkit.datamodel.AnalysisResultAdded; import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardAttribute; import org.sleuthkit.datamodel.Content; @@ -43,6 +46,7 @@ import org.sleuthkit.datamodel.Directory; import org.sleuthkit.datamodel.File; import org.sleuthkit.datamodel.LayoutFile; import org.sleuthkit.datamodel.LocalFile; +import org.sleuthkit.datamodel.Score; import org.sleuthkit.datamodel.SlackFile; import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.SleuthkitItemVisitor; @@ -449,5 +453,25 @@ public final class FileTypes implements AutopsyVisitableItem { public long getAllArtifactsCount() throws TskCoreException { return content.getAllArtifactsCount(); } + + @Override + public AnalysisResultAdded newAnalysisResult(BlackboardArtifact.Type type, Score score, String string, String string1, String string2, Collection clctn) throws TskCoreException { + return content.newAnalysisResult(type, score, string, string1, string2, clctn); + } + + @Override + public Score getAggregateScore() throws TskCoreException { + return content.getAggregateScore(); + } + + @Override + public List getAnalysisResults(BlackboardArtifact.Type type) throws TskCoreException { + return content.getAnalysisResults(type); + } + + @Override + public List getAllAnalysisResults() throws TskCoreException { + return content.getAllAnalysisResults(); + } } } From 72c95fb491a1f55ef7b2b45a0c027c0b21ad07b8 Mon Sep 17 00:00:00 2001 From: Kelly Kelly Date: Mon, 8 Feb 2021 10:33:49 -0500 Subject: [PATCH 24/25] Missing RA change for PR 6688 --- .../src/org/sleuthkit/autopsy/recentactivity/ExtractIE.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractIE.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractIE.java index 50568a2b2e..93f901d1a2 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractIE.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractIE.java @@ -267,7 +267,7 @@ class ExtractIE extends Extract { Collection bbattributes = new ArrayList<>(); bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL, RecentActivityExtracterModuleFactory.getModuleName(), url)); - bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME, + bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_CREATED, RecentActivityExtracterModuleFactory.getModuleName(), datetime)); bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_NAME, RecentActivityExtracterModuleFactory.getModuleName(), (name != null) ? name : "")); From 3812a2532d25650232c8fb7853e3a10a6a3c5af0 Mon Sep 17 00:00:00 2001 From: apriestman Date: Mon, 8 Feb 2021 15:51:17 -0500 Subject: [PATCH 25/25] Upgrade to JNA 5.6.0 --- CoreLibs/ivy.xml | 2 +- CoreLibs/nbproject/project.properties | 4 ++-- CoreLibs/nbproject/project.xml | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/CoreLibs/ivy.xml b/CoreLibs/ivy.xml index 1c83e48a09..7457154839 100644 --- a/CoreLibs/ivy.xml +++ b/CoreLibs/ivy.xml @@ -14,7 +14,7 @@ - + diff --git a/CoreLibs/nbproject/project.properties b/CoreLibs/nbproject/project.properties index fb53508bb4..3d8ebb951a 100644 --- a/CoreLibs/nbproject/project.properties +++ b/CoreLibs/nbproject/project.properties @@ -42,8 +42,8 @@ file.reference.javassist-3.12.1.GA.jar=release/modules/ext/javassist-3.12.1.GA.j file.reference.jfxtras-common-8.0-r4.jar=release/modules/ext/jfxtras-common-8.0-r4.jar file.reference.jfxtras-controls-8.0-r4.jar=release/modules/ext/jfxtras-controls-8.0-r4.jar file.reference.jfxtras-fxml-8.0-r4.jar=release/modules/ext/jfxtras-fxml-8.0-r4.jar -file.reference.jna-5.6.0.jar=release/modules/ext/jna-5.6.0.jar -file.reference.jna-platform-5.6.0.jar=release/modules/ext/jna-platform-5.6.0.jar +file.reference.jna-5.7.0.jar=release/modules/ext/jna-5.7.0.jar +file.reference.jna-platform-5.7.0.jar=release/modules/ext/jna-platform-5.7.0.jar file.reference.joda-time-2.4.jar=release/modules/ext/joda-time-2.4.jar file.reference.jsr305-1.3.9.jar=release/modules/ext/jsr305-1.3.9.jar file.reference.LGoodDatePicker-10.3.1.jar=release/modules/ext/LGoodDatePicker-10.3.1.jar diff --git a/CoreLibs/nbproject/project.xml b/CoreLibs/nbproject/project.xml index f697a317ee..24219c333d 100644 --- a/CoreLibs/nbproject/project.xml +++ b/CoreLibs/nbproject/project.xml @@ -923,8 +923,8 @@ release/modules/ext/commons-compress-1.18.jar - ext/jna-platform-5.6.0.jar - release\modules\ext\jna-platform-5.6.0.jar + ext/jna-platform-5.7.0.jar + release\modules\ext\jna-platform-5.7.0.jar ext/opencv-248.jar @@ -951,8 +951,8 @@ release/modules/ext/imageio-bmp-3.2.jar - ext/jna-5.6.0.jar - release\modules\ext\jna-5.6.0.jar + ext/jna-5.7.0.jar + release\modules\ext\jna-5.7.0.jar ext/commons-lang-2.6.jar