From 2cdd8f6e1a787c16cb07c90f6ba4826f217504f2 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Mon, 15 Jul 2019 17:50:00 -0400 Subject: [PATCH 001/102] Fixed position warning and demoted log messages to fine (aka DEBUG). --- Core/src/org/sleuthkit/autopsy/core/layer.xml | 2 +- .../corecomponents/DataContentTopComponent.java | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/core/layer.xml b/Core/src/org/sleuthkit/autopsy/core/layer.xml index 4706ea1b1d..41788864d6 100644 --- a/Core/src/org/sleuthkit/autopsy/core/layer.xml +++ b/Core/src/org/sleuthkit/autopsy/core/layer.xml @@ -436,7 +436,7 @@ - + diff --git a/Core/src/org/sleuthkit/autopsy/corecomponents/DataContentTopComponent.java b/Core/src/org/sleuthkit/autopsy/corecomponents/DataContentTopComponent.java index af015d0b4a..0d41c2d748 100644 --- a/Core/src/org/sleuthkit/autopsy/corecomponents/DataContentTopComponent.java +++ b/Core/src/org/sleuthkit/autopsy/corecomponents/DataContentTopComponent.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2011-2018 Basis Technology Corp. + * Copyright 2011-2019 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -40,9 +40,6 @@ import org.sleuthkit.autopsy.coreutils.Logger; * startup). */ // Registered as a service provider in layer.xml -//@TopComponent.Description(preferredID = "DataContentTopComponent") -//@TopComponent.Registration(mode = "output", openAtStartup = true) -//@TopComponent.OpenActionRegistration(displayName = "#CTL_DataContentAction", preferredID = "DataContentTopComponent") @SuppressWarnings("PMD.SingularField") // UI widgets cause lots of false positives public final class DataContentTopComponent extends TopComponent implements DataContent, ExplorerManager.Provider { @@ -125,15 +122,18 @@ public final class DataContentTopComponent extends TopComponent implements DataC public static synchronized DataContentTopComponent findInstance() { TopComponent win = WindowManager.getDefault().findTopComponent(PREFERRED_ID); if (win == null) { - logger.warning("Cannot find " + PREFERRED_ID + " component. It will not be located properly in the window system."); //NON-NLS + logger.log(Level.FINE, "Cannot find " + PREFERRED_ID + " component. It will " + + "not be located properly in the window system."); //NON-NLS return getDefault(); } + if (win instanceof DataContentTopComponent) { return (DataContentTopComponent) win; } - logger.warning( - "There seem to be multiple components with the '" + PREFERRED_ID //NON-NLS + + logger.log(Level.FINE, "There seem to be multiple components with the '" + PREFERRED_ID //NON-NLS + "' ID. That is a potential source of errors and unexpected behavior."); //NON-NLS + return getDefault(); } From 26221a245eedd86203c5aabd891793da98f6bfa3 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Mon, 15 Jul 2019 17:52:57 -0400 Subject: [PATCH 002/102] Upped level to INFO so it is visible --- .../autopsy/corecomponents/DataContentTopComponent.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/corecomponents/DataContentTopComponent.java b/Core/src/org/sleuthkit/autopsy/corecomponents/DataContentTopComponent.java index 0d41c2d748..e7992d6a85 100644 --- a/Core/src/org/sleuthkit/autopsy/corecomponents/DataContentTopComponent.java +++ b/Core/src/org/sleuthkit/autopsy/corecomponents/DataContentTopComponent.java @@ -122,7 +122,7 @@ public final class DataContentTopComponent extends TopComponent implements DataC public static synchronized DataContentTopComponent findInstance() { TopComponent win = WindowManager.getDefault().findTopComponent(PREFERRED_ID); if (win == null) { - logger.log(Level.FINE, "Cannot find " + PREFERRED_ID + " component. It will " + logger.log(Level.INFO, "Cannot find " + PREFERRED_ID + " component. It will " + "not be located properly in the window system."); //NON-NLS return getDefault(); } @@ -131,7 +131,7 @@ public final class DataContentTopComponent extends TopComponent implements DataC return (DataContentTopComponent) win; } - logger.log(Level.FINE, "There seem to be multiple components with the '" + PREFERRED_ID //NON-NLS + logger.log(Level.INFO, "There seem to be multiple components with the '" + PREFERRED_ID //NON-NLS + "' ID. That is a potential source of errors and unexpected behavior."); //NON-NLS return getDefault(); From 959511901bf926d0ff5a34b3e021e51fbdb8aa02 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Thu, 18 Jul 2019 16:09:58 -0400 Subject: [PATCH 003/102] Remove the location col from the property sheet --- .../datamodel/VirtualDirectoryNode.java | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/VirtualDirectoryNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/VirtualDirectoryNode.java index 19e8950d33..86aedbd0af 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/VirtualDirectoryNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/VirtualDirectoryNode.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2011-2018 Basis Technology Corp. + * Copyright 2011-2019 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,8 +20,6 @@ package org.sleuthkit.autopsy.datamodel; import java.sql.ResultSet; import java.sql.SQLException; -import java.util.LinkedHashMap; -import java.util.Map; import java.util.logging.Level; import org.openide.nodes.Sheet; import org.openide.util.NbBundle; @@ -119,7 +117,19 @@ public class VirtualDirectoryNode extends SpecialDirectoryNode { } //Otherwise default to the AAFN createSheet method. - return super.createSheet(); + Sheet defaultSheet = super.createSheet(); + Sheet.Set defaultSheetSet = defaultSheet.get(Sheet.PROPERTIES); + + //Pick out the location column + //This path should not show because VDs are not part of the data source + String locationCol = NbBundle.getMessage(AbstractAbstractFileNode.class, "AbstractAbstractFileNode.locationColLbl"); + for (Property p : defaultSheetSet.getProperties()) { + if(locationCol.equals(p.getName())) { + defaultSheetSet.remove(p.getName()); + } + } + + return defaultSheet; } @Override From e47d12a8bbacbe42f3875cc973e7c4bd8f0d2d43 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Fri, 19 Jul 2019 13:38:29 -0400 Subject: [PATCH 004/102] 5319 first pass of changing ingest listeners --- .../casemodule/CollaborationMonitor.java | 5 +- .../casemodule/IngestJobInfoPanel.java | 5 +- .../DataSourceSummaryDialog.java | 5 +- .../eventlisteners/IngestEventsListener.java | 14 +- .../optionspanel/GlobalSettingsPanel.java | 5 +- .../CommandLineIngestManager.java | 63 +++-- .../autopsy/communications/FiltersPanel.java | 264 +++++++++--------- .../datamodel/AbstractAbstractFileNode.java | 4 +- .../autopsy/datamodel/DeletedContent.java | 7 +- .../autopsy/datamodel/EmailExtracted.java | 30 +- .../autopsy/datamodel/ExtractedContent.java | 47 ++-- .../sleuthkit/autopsy/datamodel/FileSize.java | 36 +-- .../datamodel/FileTypesByExtension.java | 22 +- .../datamodel/FileTypesByMimeType.java | 10 +- .../autopsy/datamodel/HashsetHits.java | 47 ++-- .../autopsy/datamodel/ImageNode.java | 4 +- .../autopsy/datamodel/InterestingHits.java | 48 ++-- .../autopsy/datamodel/KeywordHits.java | 23 +- .../org/sleuthkit/autopsy/datamodel/Tags.java | 7 +- .../autopsy/datamodel/VolumeNode.java | 4 +- .../autopsy/datamodel/accounts/Accounts.java | 111 ++++---- .../DirectoryTreeTopComponent.java | 10 +- .../autopsy/imagewriter/ImageWriter.java | 210 +++++++------- .../autopsy/timeline/TimeLineController.java | 20 +- .../autopsy/testutils/IngestJobRunner.java | 8 +- .../autoingest/AutoIngestManager.java | 3 +- .../configuration/MultiUserTestTool.java | 45 +-- .../imagegallery/ImageGalleryController.java | 7 +- .../DropdownListSearchPanel.java | 1 + 29 files changed, 578 insertions(+), 487 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/CollaborationMonitor.java b/Core/src/org/sleuthkit/autopsy/casemodule/CollaborationMonitor.java index f99a643900..89778bb86e 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/CollaborationMonitor.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/CollaborationMonitor.java @@ -24,9 +24,11 @@ import java.beans.PropertyChangeListener; import java.io.Serializable; import java.time.Duration; import java.time.Instant; +import java.util.Arrays; import java.util.EnumSet; import java.util.HashMap; import java.util.Iterator; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.UUID; @@ -59,6 +61,7 @@ final class CollaborationMonitor { private static final String COLLABORATION_MONITOR_EVENT = "COLLABORATION_MONITOR_EVENT"; //NON-NLS private static final Set CASE_EVENTS_OF_INTEREST = EnumSet.of(Case.Events.ADDING_DATA_SOURCE, Case.Events.DATA_SOURCE_ADDED, Case.Events.ADDING_DATA_SOURCE_FAILED); + private static final Set INGEST_JOB_EVENTS_OF_INTEREST = EnumSet.of(IngestManager.IngestJobEvent.DATA_SOURCE_ANALYSIS_STARTED, IngestManager.IngestJobEvent.DATA_SOURCE_ANALYSIS_COMPLETED); private static final int NUMBER_OF_PERIODIC_TASK_THREADS = 2; private static final String PERIODIC_TASK_THREAD_NAME = "collab-monitor-periodic-tasks-%d"; //NON-NLS private static final long HEARTBEAT_INTERVAL_MINUTES = 1; @@ -113,7 +116,7 @@ final class CollaborationMonitor { * Create a local tasks manager to track and broadcast local tasks. */ localTasksManager = new LocalTasksManager(); - IngestManager.getInstance().addIngestJobEventListener(localTasksManager); + IngestManager.getInstance().addIngestJobEventListener(INGEST_JOB_EVENTS_OF_INTEREST, localTasksManager); Case.addEventTypeSubscriber(CASE_EVENTS_OF_INTEREST, localTasksManager); /** diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/IngestJobInfoPanel.java b/Core/src/org/sleuthkit/autopsy/casemodule/IngestJobInfoPanel.java index 7c7284d935..b1e6d83482 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/IngestJobInfoPanel.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/IngestJobInfoPanel.java @@ -23,7 +23,9 @@ import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Date; +import java.util.EnumSet; import java.util.List; +import java.util.Set; import java.util.logging.Level; import javax.swing.JOptionPane; import javax.swing.event.ListSelectionEvent; @@ -44,6 +46,7 @@ import org.sleuthkit.datamodel.DataSource; public final class IngestJobInfoPanel extends javax.swing.JPanel { private static final Logger logger = Logger.getLogger(IngestJobInfoPanel.class.getName()); + private static final Set INGEST_JOB_EVENTS_OF_INTEREST = EnumSet.of(IngestManager.IngestJobEvent.STARTED, IngestManager.IngestJobEvent.CANCELLED, IngestManager.IngestJobEvent.COMPLETED); private List ingestJobs; private final List ingestJobsForSelectedDataSource = new ArrayList<>(); private IngestJobTableModel ingestJobTableModel = new IngestJobTableModel(); @@ -69,7 +72,7 @@ public final class IngestJobInfoPanel extends javax.swing.JPanel { this.ingestModuleTable.setModel(this.ingestModuleTableModel); }); - IngestManager.getInstance().addIngestJobEventListener((PropertyChangeEvent evt) -> { + IngestManager.getInstance().addIngestJobEventListener(INGEST_JOB_EVENTS_OF_INTEREST , (PropertyChangeEvent evt) -> { if (evt.getPropertyName().equals(IngestManager.IngestJobEvent.STARTED.toString()) || evt.getPropertyName().equals(IngestManager.IngestJobEvent.CANCELLED.toString()) || evt.getPropertyName().equals(IngestManager.IngestJobEvent.COMPLETED.toString())) { diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/datasourcesummary/DataSourceSummaryDialog.java b/Core/src/org/sleuthkit/autopsy/casemodule/datasourcesummary/DataSourceSummaryDialog.java index 1535cbefe0..3c68b7b3ed 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/datasourcesummary/DataSourceSummaryDialog.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/datasourcesummary/DataSourceSummaryDialog.java @@ -20,9 +20,11 @@ package org.sleuthkit.autopsy.casemodule.datasourcesummary; import java.awt.Frame; import java.beans.PropertyChangeEvent; +import java.util.EnumSet; import java.util.Map; import java.util.Observable; import java.util.Observer; +import java.util.Set; import javax.swing.event.ListSelectionEvent; import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.casemodule.IngestJobInfoPanel; @@ -38,6 +40,7 @@ import org.sleuthkit.datamodel.IngestJobInfo; final class DataSourceSummaryDialog extends javax.swing.JDialog implements Observer { private static final long serialVersionUID = 1L; + private static final Set INGEST_JOB_EVENTS_OF_INTEREST = EnumSet.of(IngestManager.IngestJobEvent.DATA_SOURCE_ANALYSIS_COMPLETED); private final DataSourceSummaryCountsPanel countsPanel; private final DataSourceSummaryDetailsPanel detailsPanel; private final DataSourceBrowser dataSourcesPanel; @@ -77,7 +80,7 @@ final class DataSourceSummaryDialog extends javax.swing.JDialog implements Obser } }); //add listener to refresh jobs with Started status when they complete - IngestManager.getInstance().addIngestJobEventListener((PropertyChangeEvent evt) -> { + IngestManager.getInstance().addIngestJobEventListener(INGEST_JOB_EVENTS_OF_INTEREST, (PropertyChangeEvent evt) -> { if (evt instanceof DataSourceAnalysisCompletedEvent) { DataSourceAnalysisCompletedEvent dsEvent = (DataSourceAnalysisCompletedEvent) evt; if (dsEvent.getResult() == Reason.ANALYSIS_COMPLETED) { diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java index 58bf031359..6ec535a999 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java @@ -24,8 +24,10 @@ import java.beans.PropertyChangeListener; import static java.lang.Boolean.FALSE; import java.util.ArrayList; import java.util.Collection; +import java.util.EnumSet; import java.util.LinkedHashSet; import java.util.List; +import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.logging.Level; @@ -51,6 +53,7 @@ import org.sleuthkit.datamodel.BlackboardAttribute; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.autopsy.centralrepository.datamodel.EamDb; import org.sleuthkit.autopsy.coreutils.ThreadUtils; +import static org.sleuthkit.autopsy.ingest.IngestManager.IngestModuleEvent.DATA_ADDED; import org.sleuthkit.autopsy.ingest.events.DataSourceAnalysisCompletedEvent; import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.Image; @@ -63,16 +66,17 @@ import org.sleuthkit.datamodel.SleuthkitCase; public class IngestEventsListener { private static final Logger LOGGER = Logger.getLogger(CorrelationAttributeInstance.class.getName()); - - final Collection recentlyAddedCeArtifacts = new LinkedHashSet<>(); + private static final Set INGEST_JOB_EVENTS_OF_INTEREST = EnumSet.of(IngestManager.IngestJobEvent.DATA_SOURCE_ANALYSIS_COMPLETED); + private static final Set INGEST_MODULE_EVENTS_OF_INTEREST = EnumSet.of(DATA_ADDED); private static int correlationModuleInstanceCount; private static boolean flagNotableItems; private static boolean flagSeenDevices; private static boolean createCrProperties; - private final ExecutorService jobProcessingExecutor; private static final String INGEST_EVENT_THREAD_NAME = "Ingest-Event-Listener-%d"; + private final ExecutorService jobProcessingExecutor; private final PropertyChangeListener pcl1 = new IngestModuleEventListener(); private final PropertyChangeListener pcl2 = new IngestJobEventListener(); + final Collection recentlyAddedCeArtifacts = new LinkedHashSet<>(); IngestEventsListener() { jobProcessingExecutor = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setNameFormat(INGEST_EVENT_THREAD_NAME).build()); @@ -86,8 +90,8 @@ public class IngestEventsListener { * Add all of our Ingest Event Listeners to the IngestManager Instance. */ public void installListeners() { - IngestManager.getInstance().addIngestModuleEventListener(pcl1); - IngestManager.getInstance().addIngestJobEventListener(pcl2); + IngestManager.getInstance().addIngestModuleEventListener(INGEST_MODULE_EVENTS_OF_INTEREST, pcl1); + IngestManager.getInstance().addIngestJobEventListener(INGEST_JOB_EVENTS_OF_INTEREST, pcl2); } /* diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/GlobalSettingsPanel.java b/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/GlobalSettingsPanel.java index 49d80819ab..6236d3b8bd 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/GlobalSettingsPanel.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/GlobalSettingsPanel.java @@ -24,6 +24,7 @@ import org.sleuthkit.autopsy.coreutils.Logger; import java.beans.PropertyChangeEvent; import java.beans.PropertyChangeListener; import java.util.EnumSet; +import java.util.Set; import javax.swing.JOptionPane; import javax.swing.SwingUtilities; import org.netbeans.spi.options.OptionsPanelController; @@ -49,7 +50,7 @@ public final class GlobalSettingsPanel extends IngestModuleGlobalSettingsPanel i private static final long serialVersionUID = 1L; private static final Logger logger = Logger.getLogger(GlobalSettingsPanel.class.getName()); - + private static final Set INGEST_JOB_EVENTS_OF_INTEREST = EnumSet.of(IngestManager.IngestJobEvent.STARTED, IngestManager.IngestJobEvent.CANCELLED, IngestManager.IngestJobEvent.COMPLETED); private final IngestJobEventPropertyChangeListener ingestJobEventListener; /** @@ -72,7 +73,7 @@ public final class GlobalSettingsPanel extends IngestModuleGlobalSettingsPanel i } private void addIngestJobEventsListener() { - IngestManager.getInstance().addIngestJobEventListener(ingestJobEventListener); + IngestManager.getInstance().addIngestJobEventListener(INGEST_JOB_EVENTS_OF_INTEREST, ingestJobEventListener); ingestStateUpdated(Case.isCaseOpen()); } diff --git a/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineIngestManager.java b/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineIngestManager.java index bbe14c0020..9cc726c32e 100755 --- a/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineIngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineIngestManager.java @@ -27,7 +27,9 @@ import java.nio.file.Paths; import java.util.List; import java.util.UUID; import java.util.Collection; +import java.util.EnumSet; import java.util.Iterator; +import java.util.Set; import java.util.logging.Level; import org.netbeans.spi.sendopts.OptionProcessor; import org.openide.LifecycleManager; @@ -65,6 +67,7 @@ import org.sleuthkit.datamodel.Content; public class CommandLineIngestManager { private static final Logger LOGGER = Logger.getLogger(CommandLineIngestManager.class.getName()); + private static final Set INGEST_JOB_EVENTS_OF_INTEREST = EnumSet.of(IngestManager.IngestJobEvent.CANCELLED, IngestManager.IngestJobEvent.COMPLETED); private Path rootOutputDirectory; public CommandLineIngestManager() { @@ -198,10 +201,11 @@ public class CommandLineIngestManager { System.out.println("Unable to ingest data source " + dataSourcePath + ". Exiting..."); } catch (Throwable ex) { /* - * Unexpected runtime exceptions firewall. This task is designed to - * be able to be run in an executor service thread pool without - * calling get() on the task's Future, so this ensures that - * such errors get logged. + * Unexpected runtime exceptions firewall. This task is + * designed to be able to be run in an executor service + * thread pool without calling get() on the task's + * Future, so this ensures that such errors get + * logged. */ LOGGER.log(Level.SEVERE, "Unexpected error while ingesting data source " + dataSourcePath, ex); System.out.println("Unexpected error while ingesting data source " + dataSourcePath + ". Exiting..."); @@ -229,6 +233,7 @@ public class CommandLineIngestManager { * object. * * @param dataSource DataSource object + * * @return object ID */ private Long getDataSourceId(AutoIngestDataSource dataSource) { @@ -268,12 +273,33 @@ public class CommandLineIngestManager { * @param dataSource The data source. * * @throws - * AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException - * if there was a DSP processing error + * AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException if + * there + * was + * a + * DSP + * processing + * error * - * @throws InterruptedException if the thread running the job processing - * task is interrupted while blocked, i.e., if auto ingest is shutting - * down. + * @throws InterruptedException if + * the + * thread + * running + * the + * job + * processing + * task + * is + * interrupted + * while + * blocked, + * i.e., + * if + * auto + * ingest + * is + * shutting + * down. */ private void runDataSourceProcessor(Case caseForJob, AutoIngestDataSource dataSource) throws InterruptedException, AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException { @@ -295,7 +321,7 @@ public class CommandLineIngestManager { LOGGER.log(Level.SEVERE, "Unsupported data source {0}", dataSource.getPath()); // NON-NLS return; } - + DataSourceProcessorProgressMonitor progressMonitor = new DoNothingDSPProgressMonitor(); synchronized (ingestLock) { // Try each DSP in decreasing order of confidence @@ -375,16 +401,17 @@ public class CommandLineIngestManager { * @param dataSource The data source to analyze. * * @throws AnalysisStartupException if there is an error analyzing the - * data source. - * @throws InterruptedException if the thread running the job processing - * task is interrupted while blocked, i.e., if auto ingest is shutting - * down. + * data source. + * @throws InterruptedException if the thread running the job + * processing task is interrupted while + * blocked, i.e., if auto ingest is + * shutting down. */ private void analyze(AutoIngestDataSource dataSource) throws AnalysisStartupException, InterruptedException { LOGGER.log(Level.INFO, "Starting ingest modules analysis for {0} ", dataSource.getPath()); IngestJobEventListener ingestJobEventListener = new IngestJobEventListener(); - IngestManager.getInstance().addIngestJobEventListener(ingestJobEventListener); + IngestManager.getInstance().addIngestJobEventListener(INGEST_JOB_EVENTS_OF_INTEREST, ingestJobEventListener); try { synchronized (ingestLock) { IngestJobSettings ingestJobSettings = new IngestJobSettings(UserPreferences.getCommandLineModeIngestModuleContextString()); @@ -447,7 +474,7 @@ public class CommandLineIngestManager { * the path. * * @param caseFoldersPath The root case folders path. - * @param caseName The name of the case. + * @param caseName The name of the case. * * @return A case folder path with a time stamp suffix. */ @@ -461,8 +488,8 @@ public class CommandLineIngestManager { * for a case. * * @param folderToSearch The folder to be searched. - * @param caseName The name of the case for which a case folder is to be - * found. + * @param caseName The name of the case for which a case folder is + * to be found. * * @return The path of the case folder, or null if it is not found. */ diff --git a/Core/src/org/sleuthkit/autopsy/communications/FiltersPanel.java b/Core/src/org/sleuthkit/autopsy/communications/FiltersPanel.java index 445bf25c87..15e1d29b3c 100644 --- a/Core/src/org/sleuthkit/autopsy/communications/FiltersPanel.java +++ b/Core/src/org/sleuthkit/autopsy/communications/FiltersPanel.java @@ -34,6 +34,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.logging.Level; import java.util.stream.Collectors; @@ -81,7 +82,8 @@ final public class FiltersPanel extends JPanel { private static final long serialVersionUID = 1L; private static final Logger logger = Logger.getLogger(FiltersPanel.class.getName()); - + private static final Set INGEST_JOB_EVENTS_OF_INTEREST = EnumSet.of(IngestManager.IngestJobEvent.COMPLETED); + private static final Set INGEST_MODULE_EVENTS_OF_INTEREST = EnumSet.of(DATA_ADDED); /** * Map from Account.Type to the checkbox for that account type's filter. */ @@ -120,17 +122,17 @@ final public class FiltersPanel extends JPanel { * initially. */ private boolean deviceAccountTypeEnabled; - + private Case openCase = null; @NbBundle.Messages({"refreshText=Refresh Results", "applyText=Apply"}) public FiltersPanel() { initComponents(); - + CheckBoxIconPanel panel = createAccoutTypeCheckBoxPanel(Account.Type.DEVICE, true); accountTypeMap.put(Account.Type.DEVICE, panel.getCheckBox()); accountTypeListPane.add(panel); - + deviceRequiredLabel.setVisible(false); accountTypeRequiredLabel.setVisible(false); startDatePicker.setDate(LocalDate.now().minusWeeks(3)); @@ -151,8 +153,8 @@ final public class FiltersPanel extends JPanel { updateFilters(true); UserPreferences.addChangeListener(preferenceChangeEvent -> { - if (preferenceChangeEvent.getKey().equals(UserPreferences.DISPLAY_TIMES_IN_LOCAL_TIME) || - preferenceChangeEvent.getKey().equals(UserPreferences.TIME_ZONE_FOR_DISPLAYS)) { + if (preferenceChangeEvent.getKey().equals(UserPreferences.DISPLAY_TIMES_IN_LOCAL_TIME) + || preferenceChangeEvent.getKey().equals(UserPreferences.TIME_ZONE_FOR_DISPLAYS)) { updateTimeZone(); } }); @@ -166,22 +168,21 @@ final public class FiltersPanel extends JPanel { && (eventData.getBlackboardArtifactType().getTypeID() == BlackboardArtifact.ARTIFACT_TYPE.TSK_MESSAGE.getTypeID() || eventData.getBlackboardArtifactType().getTypeID() == BlackboardArtifact.ARTIFACT_TYPE.TSK_CONTACT.getTypeID() || eventData.getBlackboardArtifactType().getTypeID() == BlackboardArtifact.ARTIFACT_TYPE.TSK_CALLLOG.getTypeID() - || eventData.getBlackboardArtifactType().getTypeID() == BlackboardArtifact.ARTIFACT_TYPE.TSK_EMAIL_MSG.getTypeID())) - { - updateFilters(true); - needsRefresh = true; - validateFilters(); + || eventData.getBlackboardArtifactType().getTypeID() == BlackboardArtifact.ARTIFACT_TYPE.TSK_EMAIL_MSG.getTypeID())) { + updateFilters(true); + needsRefresh = true; + validateFilters(); } } }; - + this.ingestJobListener = pce -> { String eventType = pce.getPropertyName(); - if (eventType.equals(COMPLETED.toString()) && - updateFilters(true)) { - - needsRefresh = true; - validateFilters(); + if (eventType.equals(COMPLETED.toString()) + && updateFilters(true)) { + + needsRefresh = true; + validateFilters(); } }; @@ -208,16 +209,16 @@ final public class FiltersPanel extends JPanel { refreshButton.setEnabled(someDevice && someAccountType && needsRefresh && validLimit); needsRefreshLabel.setVisible(needsRefresh); } - + private boolean validateLimitValue() { - String selectedValue = (String)limitComboBox.getSelectedItem(); - if(selectedValue.trim().equalsIgnoreCase("all")) { + String selectedValue = (String) limitComboBox.getSelectedItem(); + if (selectedValue.trim().equalsIgnoreCase("all")) { return true; } else { - try{ + try { int value = Integer.parseInt(selectedValue); return value > 0; - } catch( NumberFormatException ex) { + } catch (NumberFormatException ex) { return false; } } @@ -242,7 +243,7 @@ final public class FiltersPanel extends JPanel { private boolean updateFilters(boolean initialState) { boolean newAccountType = updateAccountTypeFilter(initialState); boolean newDeviceFilter = updateDeviceFilter(initialState); - + // both or either are true, return true; return newAccountType || newDeviceFilter; } @@ -250,13 +251,13 @@ final public class FiltersPanel extends JPanel { @Override public void addNotify() { super.addNotify(); - IngestManager.getInstance().addIngestModuleEventListener(ingestListener); - IngestManager.getInstance().addIngestJobEventListener(ingestJobListener); + IngestManager.getInstance().addIngestModuleEventListener(INGEST_MODULE_EVENTS_OF_INTEREST, ingestListener); + IngestManager.getInstance().addIngestJobEventListener(INGEST_JOB_EVENTS_OF_INTEREST, ingestJobListener); Case.addEventTypeSubscriber(EnumSet.of(CURRENT_CASE), evt -> { //clear the device filter widget when the case changes. devicesMap.clear(); devicesListPane.removeAll(); - + accountTypeMap.clear(); accountTypeListPane.removeAll(); }); @@ -271,9 +272,9 @@ final public class FiltersPanel extends JPanel { /** * Populate the Account Types filter widgets - * + * * @param selected the initial value for the account type checkbox - * + * * @return True, if a new accountType was found */ private boolean updateAccountTypeFilter(boolean selected) { @@ -281,9 +282,9 @@ final public class FiltersPanel extends JPanel { try { final CommunicationsManager communicationsManager = Case.getCurrentCaseThrows().getSleuthkitCase().getCommunicationsManager(); List accountTypesInUse = communicationsManager.getAccountTypesInUse(); - + for (Account.Type type : accountTypesInUse) { - + if (!accountTypeMap.containsKey(type) && !type.equals(Account.Type.CREDIT_CARD)) { CheckBoxIconPanel panel = createAccoutTypeCheckBoxPanel(type, selected); accountTypeMap.put(type, panel.getCheckBox()); @@ -305,14 +306,14 @@ final public class FiltersPanel extends JPanel { return newOneFound; } - + /** * Helper function to create a new instance of the CheckBoxIconPanel base on * the Account.Type and initalState (check box state). - * - * @param type Account.Type to display on the panel + * + * @param type Account.Type to display on the panel * @param initalState initial check box state - * + * * @return instance of the CheckBoxIconPanel */ private CheckBoxIconPanel createAccoutTypeCheckBoxPanel(Account.Type type, boolean initalState) { @@ -324,12 +325,12 @@ final public class FiltersPanel extends JPanel { panel.addItemListener(validationListener); return panel; } - + /** * Populate the devices filter widgets - * + * * @param selected Sets the initial state of device check box - * + * * @return true if a new device was found */ private boolean updateDeviceFilter(boolean selected) { @@ -339,15 +340,15 @@ final public class FiltersPanel extends JPanel { for (DataSource dataSource : sleuthkitCase.getDataSources()) { String dsName = sleuthkitCase.getContentById(dataSource.getId()).getName(); - if(devicesMap.containsKey(dataSource.getDeviceId())) { + if (devicesMap.containsKey(dataSource.getDeviceId())) { continue; } - + final JCheckBox jCheckBox = new JCheckBox(dsName, selected); jCheckBox.addItemListener(validationListener); devicesListPane.add(jCheckBox); devicesMap.put(dataSource.getDeviceId(), jCheckBox); - + newOneFound = true; } @@ -356,36 +357,36 @@ final public class FiltersPanel extends JPanel { } catch (TskCoreException tskCoreException) { logger.log(Level.SEVERE, "There was a error loading the datasources for the case.", tskCoreException); } - - if(newOneFound) { + + if (newOneFound) { devicesListPane.revalidate(); } - + return newOneFound; } - + /** - * Given a list of subFilters, set the states of the panel controls + * Given a list of subFilters, set the states of the panel controls * accordingly. - * + * * @param commFilter Contains a list of subFilters */ public void setFilters(CommunicationsFilter commFilter) { List subFilters = commFilter.getAndFilters(); subFilters.forEach(subFilter -> { - if( subFilter instanceof DeviceFilter ) { - setDeviceFilter((DeviceFilter)subFilter); - } else if( subFilter instanceof AccountTypeFilter) { + if (subFilter instanceof DeviceFilter) { + setDeviceFilter((DeviceFilter) subFilter); + } else if (subFilter instanceof AccountTypeFilter) { setAccountTypeFilter((AccountTypeFilter) subFilter); - } else if (subFilter instanceof MostRecentFilter ) { - setMostRecentFilter((MostRecentFilter)subFilter); + } else if (subFilter instanceof MostRecentFilter) { + setMostRecentFilter((MostRecentFilter) subFilter); } }); } - + /** * Sets the state of the device filter check boxes - * + * * @param deviceFilter Selected devices */ private void setDeviceFilter(DeviceFilter deviceFilter) { @@ -394,23 +395,24 @@ final public class FiltersPanel extends JPanel { cb.setSelected(deviceIDs.contains(type)); }); } - - /** - * Set the state of the account type checkboxes to match the passed in filter - * + + /** + * Set the state of the account type checkboxes to match the passed in + * filter + * * @param typeFilter Account Types to be selected */ - private void setAccountTypeFilter(AccountTypeFilter typeFilter){ - + private void setAccountTypeFilter(AccountTypeFilter typeFilter) { + accountTypeMap.forEach((type, cb) -> { cb.setSelected(typeFilter.getAccountTypes().contains(type)); }); } - + /** - * Set up the startDatePicker and startCheckBox based on the passed in + * Set up the startDatePicker and startCheckBox based on the passed in * DateControlState. - * + * * @param state new control state */ private void setStartDateControlState(DateControlState state) { @@ -418,11 +420,11 @@ final public class FiltersPanel extends JPanel { startCheckBox.setSelected(state.isEnabled()); startDatePicker.setEnabled(state.isEnabled()); } - + /** - * Set up the endDatePicker and endCheckBox based on the passed in - * DateControlState. - * + * Set up the endDatePicker and endCheckBox based on the passed in + * DateControlState. + * * @param state new control state */ private void setEndDateControlState(DateControlState state) { @@ -430,25 +432,25 @@ final public class FiltersPanel extends JPanel { endCheckBox.setSelected(state.isEnabled()); endDatePicker.setEnabled(state.isEnabled()); } - + /** * Sets the state of the most recent UI controls based on the current values * in MostRecentFilter. - * + * * @param filter The MostRecentFilter state to be set */ private void setMostRecentFilter(MostRecentFilter filter) { int limit = filter.getLimit(); - if(limit > 0) { + if (limit > 0) { limitComboBox.setSelectedItem(filter.getLimit()); } else { limitComboBox.setSelectedItem("All"); } } - + @Subscribe void filtersBack(CVTEvents.StateChangeEvent event) { - if(event.getCommunicationsState().getCommunicationsFilter() != null){ + if (event.getCommunicationsState().getCommunicationsFilter() != null) { setFilters(event.getCommunicationsState().getCommunicationsFilter()); setStartDateControlState(event.getCommunicationsState().getStartControlState()); setEndDateControlState(event.getCommunicationsState().getEndControlState()); @@ -828,7 +830,7 @@ final public class FiltersPanel extends JPanel { /** * Get an instance of CommunicationsFilters base on the current panel state. - * + * * @return an instance of CommunicationsFilter */ protected CommunicationsFilter getFilter() { @@ -877,36 +879,37 @@ final public class FiltersPanel extends JPanel { */ private DateRangeFilter getDateRangeFilter() { ZoneId zone = Utils.getUserPreferredZoneId(); - - return new DateRangeFilter( startCheckBox.isSelected() ? startDatePicker.getDate().atStartOfDay(zone).toEpochSecond() : 0, - endCheckBox.isSelected() ? endDatePicker.getDate().atStartOfDay(zone).toEpochSecond() : 0); + + return new DateRangeFilter(startCheckBox.isSelected() ? startDatePicker.getDate().atStartOfDay(zone).toEpochSecond() : 0, + endCheckBox.isSelected() ? endDatePicker.getDate().atStartOfDay(zone).toEpochSecond() : 0); } - + /** - * Get a MostRecentFilter that based on the current state of the ui controls. - * - * @return A new instance of MostRecentFilter + * Get a MostRecentFilter that based on the current state of the ui + * controls. + * + * @return A new instance of MostRecentFilter */ private MostRecentFilter getMostRecentFilter() { - String value = (String)limitComboBox.getSelectedItem(); - if(value.trim().equalsIgnoreCase("all")){ + String value = (String) limitComboBox.getSelectedItem(); + if (value.trim().equalsIgnoreCase("all")) { return new MostRecentFilter(-1); - } else{ + } else { try { int count = Integer.parseInt(value); return new MostRecentFilter(count); - } catch(NumberFormatException ex) { + } catch (NumberFormatException ex) { return null; } } } - + private DateControlState getStartControlState() { - return new DateControlState (startDatePicker.getDate(), startCheckBox.isSelected()); + return new DateControlState(startDatePicker.getDate(), startCheckBox.isSelected()); } - + private DateControlState getEndControlState() { - return new DateControlState (endDatePicker.getDate(), endCheckBox.isSelected()); + return new DateControlState(endDatePicker.getDate(), endCheckBox.isSelected()); } /** @@ -940,32 +943,32 @@ final public class FiltersPanel extends JPanel { private void setAllSelected(Map map, boolean selected) { map.values().forEach(box -> box.setSelected(selected)); } - + /** * initalize the DateTimePickers by grabbing the earliest and latest time * from the autopsy db. */ private void initalizeDateTimeFilters() { Case currentCase = null; - try{ + try { currentCase = Case.getCurrentCaseThrows(); - } catch (NoCurrentCaseException ex) { - logger.log(Level.INFO, "Tried to intialize communication filters date range filters without an open case, using default values"); + } catch (NoCurrentCaseException ex) { + logger.log(Level.INFO, "Tried to intialize communication filters date range filters without an open case, using default values"); } - - if(currentCase == null) { + + if (currentCase == null) { setDateTimeFiltersToDefault(); openCase = null; return; } - - if(!currentCase.equals(openCase)) { + + if (!currentCase.equals(openCase)) { setDateTimeFiltersToDefault(); openCase = currentCase; (new DatePickerWorker()).execute(); } } - + private void setDateTimeFiltersToDefault() { startDatePicker.setDate(LocalDate.now().minusWeeks(3)); endDatePicker.setDate(LocalDate.now()); @@ -1002,46 +1005,47 @@ final public class FiltersPanel extends JPanel { }//GEN-LAST:event_limitComboBoxActionPerformed /** - * A class to wrap the state of the date controls that consist of a date picker - * and a checkbox. - * + * A class to wrap the state of the date controls that consist of a date + * picker and a checkbox. + * */ final class DateControlState { + private final LocalDate date; private final boolean enabled; - + /** * Wraps the state of the date controls that consist of a date picker * and checkbox - * - * @param date LocalDate value of the datepicker + * + * @param date LocalDate value of the datepicker * @param enabled State of the checkbox */ protected DateControlState(LocalDate date, boolean enabled) { this.date = date; this.enabled = enabled; } - + /** - * Returns the given LocalDate from the datepicker - * + * Returns the given LocalDate from the datepicker + * * @return Current state LocalDate */ - public LocalDate getDate(){ + public LocalDate getDate() { return date; } - + /** * Returns the given state of the datepicker checkbox - * + * * @return boolean, whether or not the datepicker was enabled */ public boolean isEnabled() { return enabled; } - + } - + // Variables declaration - do not modify//GEN-BEGIN:variables private final javax.swing.JPanel accountTypeListPane = new javax.swing.JPanel(); private final javax.swing.JLabel accountTypeRequiredLabel = new javax.swing.JLabel(); @@ -1078,59 +1082,59 @@ final public class FiltersPanel extends JPanel { private final javax.swing.JButton unCheckAllDevicesButton = new javax.swing.JButton(); // End of variables declaration//GEN-END:variables - /** - * This class is a small panel that appears to just be a checkbox but - * adds the functionality of being able to show an icon between the checkbox - * and label. + * This class is a small panel that appears to just be a checkbox but adds + * the functionality of being able to show an icon between the checkbox and + * label. */ - final class CheckBoxIconPanel extends JPanel{ + final class CheckBoxIconPanel extends JPanel { + private final JCheckBox checkbox; private final JLabel label; - + /** * Creates a JPanel instance with the specified label and image. - * + * * @param labelText The text to be displayed by the checkbox label. - * @param image The image to be dispayed by the label. + * @param image The image to be dispayed by the label. */ private CheckBoxIconPanel(String labelText, Icon image) { checkbox = new JCheckBox(); label = new JLabel(labelText); label.setIcon(image); setLayout(new BoxLayout(this, BoxLayout.X_AXIS)); - + add(checkbox); add(label); add(Box.createHorizontalGlue()); } - + /** * Sets the state of the checkbox. - * + * * @param selected true if the button is selected, otherwise false */ void setSelected(boolean selected) { checkbox.setSelected(selected); } - + @Override public void setEnabled(boolean enabled) { checkbox.setEnabled(enabled); } - + /** * Returns the instance of the JCheckBox. - * + * * @return JCheckbox instance */ JCheckBox getCheckBox() { return checkbox; } - + /** * Adds an ItemListener to the checkbox. - * + * * @param l the ItemListener to be added. */ void addItemListener(ItemListener l) { @@ -1139,8 +1143,8 @@ final public class FiltersPanel extends JPanel { } /** - * A simple class that implements CaseDbAccessQueryCallback. Can be used - * as an anonymous innerclass with the CaseDbAccessManager select function. + * A simple class that implements CaseDbAccessQueryCallback. Can be used as + * an anonymous innerclass with the CaseDbAccessManager select function. */ class FilterPanelQueryCallback implements CaseDbAccessQueryCallback { @@ -1149,7 +1153,7 @@ final public class FiltersPanel extends JPanel { // Subclasses can implement their own process function. } } - + final class DatePickerWorker extends SwingWorker, Void> { @Override @@ -1204,4 +1208,4 @@ final public class FiltersPanel extends JPanel { } } -} \ No newline at end of file +} diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java index eeacf491bf..0419bc362f 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java @@ -54,6 +54,7 @@ import static org.sleuthkit.autopsy.datamodel.AbstractAbstractFileNode.AbstractF import org.sleuthkit.autopsy.datamodel.BaseChildFactory.NoSuchEventBusException; import org.sleuthkit.autopsy.datamodel.BaseChildFactory.RefreshKeysEvent; import org.sleuthkit.autopsy.ingest.IngestManager; +import static org.sleuthkit.autopsy.ingest.IngestManager.IngestModuleEvent.CONTENT_CHANGED; import org.sleuthkit.autopsy.ingest.ModuleContentEvent; import org.sleuthkit.autopsy.texttranslation.NoServiceProviderException; import org.sleuthkit.autopsy.texttranslation.TextTranslationService; @@ -77,6 +78,7 @@ public abstract class AbstractAbstractFileNode extends A private static final Set CASE_EVENTS_OF_INTEREST = EnumSet.of(Case.Events.CURRENT_CASE, Case.Events.CONTENT_TAG_ADDED, Case.Events.CONTENT_TAG_DELETED, Case.Events.CR_COMMENT_CHANGED); + private static final Set INGEST_MODULE_EVENTS_OF_INTEREST = EnumSet.of(CONTENT_CHANGED); /** * @param abstractFile file to wrap @@ -89,7 +91,7 @@ public abstract class AbstractAbstractFileNode extends A // If this is an archive file we will listen for ingest events // that will notify us when new content has been identified. if (FileTypeExtensions.getArchiveExtensions().contains(ext)) { - IngestManager.getInstance().addIngestModuleEventListener(weakPcl); + IngestManager.getInstance().addIngestModuleEventListener(INGEST_MODULE_EVENTS_OF_INTEREST, weakPcl); } } diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/DeletedContent.java b/Core/src/org/sleuthkit/autopsy/datamodel/DeletedContent.java index adc5a34fa7..5ee741a496 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/DeletedContent.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/DeletedContent.java @@ -39,6 +39,7 @@ import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.ingest.IngestManager; +import static org.sleuthkit.autopsy.ingest.IngestManager.IngestModuleEvent.CONTENT_CHANGED; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.ContentVisitor; @@ -190,10 +191,12 @@ public class DeletedContent implements AutopsyVisitableItem { Case.Events.DATA_SOURCE_ADDED, Case.Events.CURRENT_CASE ); + private static final Set INGEST_JOB_EVENTS_OF_INTEREST = EnumSet.of(IngestManager.IngestJobEvent.COMPLETED, IngestManager.IngestJobEvent.CANCELLED); + private static final Set INGEST_MODULE_EVENTS_OF_INTEREST = EnumSet.of(CONTENT_CHANGED); DeletedContentsChildrenObservable() { - IngestManager.getInstance().addIngestJobEventListener(pcl); - IngestManager.getInstance().addIngestModuleEventListener(pcl); + IngestManager.getInstance().addIngestJobEventListener(INGEST_JOB_EVENTS_OF_INTEREST, pcl); + IngestManager.getInstance().addIngestModuleEventListener(INGEST_MODULE_EVENTS_OF_INTEREST, pcl); Case.addEventTypeSubscriber(CASE_EVENTS_OF_INTEREST, pcl); } diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/EmailExtracted.java b/Core/src/org/sleuthkit/autopsy/datamodel/EmailExtracted.java index 7285b2cb8d..a6652679ec 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/EmailExtracted.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/EmailExtracted.java @@ -42,6 +42,7 @@ import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.ingest.IngestManager; +import static org.sleuthkit.autopsy.ingest.IngestManager.IngestModuleEvent.CONTENT_CHANGED; import org.sleuthkit.autopsy.ingest.ModuleDataEvent; import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardAttribute; @@ -63,6 +64,9 @@ public class EmailExtracted implements AutopsyVisitableItem { private static final String MAIL_ACCOUNT = NbBundle.getMessage(EmailExtracted.class, "EmailExtracted.mailAccount.text"); private static final String MAIL_FOLDER = NbBundle.getMessage(EmailExtracted.class, "EmailExtracted.mailFolder.text"); private static final String MAIL_PATH_SEPARATOR = "/"; + private static final Set INGEST_JOB_EVENTS_OF_INTEREST = EnumSet.of(IngestManager.IngestJobEvent.COMPLETED, IngestManager.IngestJobEvent.CANCELLED); + private static final Set INGEST_MODULE_EVENTS_OF_INTEREST = EnumSet.of(IngestManager.IngestModuleEvent.DATA_ADDED); + /** * Parse the path of the email msg to get the account name and folder in * which the email is contained. @@ -88,37 +92,35 @@ public class EmailExtracted implements AutopsyVisitableItem { private final EmailResults emailResults; private final long filteringDSObjId; // 0 if not filtering/grouping by data source - - /** * Constructor - * + * * @param skCase Case DB */ public EmailExtracted(SleuthkitCase skCase) { this(skCase, 0); } - + /** * Constructor - * - * @param skCase Case DB - * @param objId Object id of the data source - * - */ + * + * @param skCase Case DB + * @param objId Object id of the data source + * + */ public EmailExtracted(SleuthkitCase skCase, long objId) { this.skCase = skCase; this.filteringDSObjId = objId; emailResults = new EmailResults(); } - @Override public T accept(AutopsyItemVisitor visitor) { return visitor.visit(this); } + private final class EmailResults extends Observable { - + // NOTE: the map can be accessed by multiple worker threads and needs to be synchronized private final Map>> accounts = new LinkedHashMap<>(); @@ -161,7 +163,7 @@ public class EmailExtracted implements AutopsyVisitableItem { + " AND blackboard_attributes.artifact_id=blackboard_artifacts.artifact_id" //NON-NLS + " AND blackboard_artifacts.artifact_type_id=" + artId; //NON-NLS if (filteringDSObjId > 0) { - query += " AND blackboard_artifacts.data_source_obj_id = " + filteringDSObjId; + query += " AND blackboard_artifacts.data_source_obj_id = " + filteringDSObjId; } try (CaseDbQuery dbQuery = skCase.executeQuery(query)) { @@ -307,8 +309,8 @@ public class EmailExtracted implements AutopsyVisitableItem { @Override protected void addNotify() { - IngestManager.getInstance().addIngestJobEventListener(pcl); - IngestManager.getInstance().addIngestModuleEventListener(pcl); + IngestManager.getInstance().addIngestJobEventListener(INGEST_JOB_EVENTS_OF_INTEREST, pcl); + IngestManager.getInstance().addIngestModuleEventListener(INGEST_MODULE_EVENTS_OF_INTEREST, pcl); Case.addEventTypeSubscriber(EnumSet.of(Case.Events.CURRENT_CASE), pcl); emailResults.update(); emailResults.addObserver(this); diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/ExtractedContent.java b/Core/src/org/sleuthkit/autopsy/datamodel/ExtractedContent.java index a51e36eb87..4ac4413e47 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/ExtractedContent.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/ExtractedContent.java @@ -26,6 +26,7 @@ import java.util.Comparator; import java.util.EnumSet; import java.util.HashMap; import java.util.List; +import java.util.Set; import java.util.logging.Level; import org.openide.nodes.ChildFactory; import org.openide.nodes.Children; @@ -59,14 +60,16 @@ import org.sleuthkit.datamodel.TskException; */ public class ExtractedContent implements AutopsyVisitableItem { - private SleuthkitCase skCase; // set to null after case has been closed - private Blackboard blackboard; + private static final Set INGEST_JOB_EVENTS_OF_INTEREST = EnumSet.of(IngestManager.IngestJobEvent.COMPLETED, IngestManager.IngestJobEvent.CANCELLED); + private static final Set INGEST_MODULE_EVENTS_OF_INTEREST = EnumSet.of(IngestManager.IngestModuleEvent.DATA_ADDED); public static final String NAME = NbBundle.getMessage(RootNode.class, "ExtractedContentNode.name.text"); private final long filteringDSObjId; // 0 if not filtering/grouping by data source + private SleuthkitCase skCase; // set to null after case has been closed + private Blackboard blackboard; /** - * Constructs extracted content object - * + * Constructs extracted content object + * * @param skCase Case DB */ public ExtractedContent(SleuthkitCase skCase) { @@ -74,17 +77,17 @@ public class ExtractedContent implements AutopsyVisitableItem { } /** - * Constructs extracted content object - * + * Constructs extracted content object + * * @param skCase Case DB - * @param objId Object id of the parent datasource + * @param objId Object id of the parent datasource */ public ExtractedContent(SleuthkitCase skCase, long objId) { this.skCase = skCase; this.filteringDSObjId = objId; this.blackboard = skCase.getBlackboard(); } - + @Override public T accept(AutopsyItemVisitor visitor) { return visitor.visit(this); @@ -144,8 +147,8 @@ public class ExtractedContent implements AutopsyVisitableItem { return filePath + "gps-search.png"; //NON-NLS } else if (typeID == BlackboardArtifact.ARTIFACT_TYPE.TSK_PROG_RUN.getTypeID()) { return filePath + "installed.png"; //NON-NLS - } else if (typeID == BlackboardArtifact.ARTIFACT_TYPE.TSK_ENCRYPTION_DETECTED.getTypeID() || - typeID == BlackboardArtifact.ARTIFACT_TYPE.TSK_ENCRYPTION_SUSPECTED.getTypeID()) { + } else if (typeID == BlackboardArtifact.ARTIFACT_TYPE.TSK_ENCRYPTION_DETECTED.getTypeID() + || typeID == BlackboardArtifact.ARTIFACT_TYPE.TSK_ENCRYPTION_SUSPECTED.getTypeID()) { return filePath + "encrypted-file.png"; //NON-NLS } else if (typeID == BlackboardArtifact.ARTIFACT_TYPE.TSK_EXT_MISMATCH_DETECTED.getTypeID()) { return filePath + "mismatch-16.png"; //NON-NLS @@ -235,7 +238,7 @@ public class ExtractedContent implements AutopsyVisitableItem { doNotShow.add(new BlackboardArtifact.Type(TSK_INTERESTING_ARTIFACT_HIT)); doNotShow.add(new BlackboardArtifact.Type(TSK_ACCOUNT)); doNotShow.add(new BlackboardArtifact.Type(TSK_DATA_SOURCE_USAGE)); - doNotShow.add(new BlackboardArtifact.Type(TSK_DOWNLOAD_SOURCE) ); + doNotShow.add(new BlackboardArtifact.Type(TSK_DOWNLOAD_SOURCE)); } private final PropertyChangeListener pcl = (PropertyChangeEvent evt) -> { @@ -288,8 +291,8 @@ public class ExtractedContent implements AutopsyVisitableItem { @Override protected void addNotify() { - IngestManager.getInstance().addIngestJobEventListener(pcl); - IngestManager.getInstance().addIngestModuleEventListener(pcl); + IngestManager.getInstance().addIngestJobEventListener(INGEST_JOB_EVENTS_OF_INTEREST, pcl); + IngestManager.getInstance().addIngestModuleEventListener(INGEST_MODULE_EVENTS_OF_INTEREST, pcl); Case.addEventTypeSubscriber(EnumSet.of(Case.Events.CURRENT_CASE), pcl); } @@ -305,10 +308,10 @@ public class ExtractedContent implements AutopsyVisitableItem { protected boolean createKeys(List list) { if (skCase != null) { try { - List types = (filteringDSObjId > 0) ? - blackboard.getArtifactTypesInUse(filteringDSObjId) : - skCase.getArtifactTypesInUse() ; - + List types = (filteringDSObjId > 0) + ? blackboard.getArtifactTypesInUse(filteringDSObjId) + : skCase.getArtifactTypesInUse(); + types.removeAll(doNotShow); Collections.sort(types, new Comparator() { @@ -370,9 +373,9 @@ public class ExtractedContent implements AutopsyVisitableItem { // a performance increase might be had by adding a // "getBlackboardArtifactCount()" method to skCase try { - this.childCount = (filteringDSObjId > 0) ? - blackboard.getArtifactsCount(type.getTypeID(), filteringDSObjId) : - skCase.getBlackboardArtifactsTypeCount(type.getTypeID()); + this.childCount = (filteringDSObjId > 0) + ? blackboard.getArtifactsCount(type.getTypeID(), filteringDSObjId) + : skCase.getBlackboardArtifactsTypeCount(type.getTypeID()); } catch (TskException ex) { Logger.getLogger(TypeNode.class.getName()) .log(Level.WARNING, "Error getting child count", ex); //NON-NLS @@ -480,8 +483,8 @@ public class ExtractedContent implements AutopsyVisitableItem { @Override protected void onAdd() { - IngestManager.getInstance().addIngestJobEventListener(pcl); - IngestManager.getInstance().addIngestModuleEventListener(pcl); + IngestManager.getInstance().addIngestJobEventListener(INGEST_JOB_EVENTS_OF_INTEREST, pcl); + IngestManager.getInstance().addIngestModuleEventListener(INGEST_MODULE_EVENTS_OF_INTEREST, pcl); } @Override diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/FileSize.java b/Core/src/org/sleuthkit/autopsy/datamodel/FileSize.java index 9cceeb5a29..172c11c037 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/FileSize.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/FileSize.java @@ -1,15 +1,15 @@ /* * Autopsy Forensic Browser - * + * * Copyright 2013-2019 Basis Technology Corp. * Contact: carrier sleuthkit org - * + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -104,7 +104,7 @@ public class FileSize implements AutopsyVisitableItem { this.skCase = skCase; this.filteringDSObjId = dsObjId; } - + @Override public T accept(AutopsyItemVisitor visitor) { return visitor.visit(this); @@ -117,6 +117,7 @@ public class FileSize implements AutopsyVisitableItem { long filteringDataSourceObjId() { return this.filteringDSObjId; } + /* * Root node. Children are nodes for specific sizes. */ @@ -169,7 +170,7 @@ public class FileSize implements AutopsyVisitableItem { public static class FileSizeRootChildren extends ChildFactory { private SleuthkitCase skCase; - private final long datasourceObjId; + private final long datasourceObjId; private Observable notifier; public FileSizeRootChildren(SleuthkitCase skCase, long datasourceObjId) { @@ -185,10 +186,12 @@ public class FileSize implements AutopsyVisitableItem { private static final class FileSizeRootChildrenObservable extends Observable { private static final Set CASE_EVENTS_OF_INTEREST = EnumSet.of(Case.Events.DATA_SOURCE_ADDED, Case.Events.CURRENT_CASE); + private static final Set INGEST_JOB_EVENTS_OF_INTEREST = EnumSet.of(IngestManager.IngestJobEvent.COMPLETED, IngestManager.IngestJobEvent.CANCELLED); + private static final Set INGEST_MODULE_EVENTS_OF_INTEREST = EnumSet.of(IngestManager.IngestModuleEvent.CONTENT_CHANGED); FileSizeRootChildrenObservable() { - IngestManager.getInstance().addIngestJobEventListener(pcl); - IngestManager.getInstance().addIngestModuleEventListener(pcl); + IngestManager.getInstance().addIngestJobEventListener(INGEST_JOB_EVENTS_OF_INTEREST, pcl); + IngestManager.getInstance().addIngestModuleEventListener(INGEST_MODULE_EVENTS_OF_INTEREST, pcl); Case.addEventTypeSubscriber(CASE_EVENTS_OF_INTEREST, pcl); } @@ -282,9 +285,10 @@ public class FileSize implements AutopsyVisitableItem { * * @param skCase * @param filter - * @param o Observable that provides updates when events are - * fired - * @param datasourceObjId filter by data source, if configured in user preferences + * @param o Observable that provides updates when + * events are fired + * @param datasourceObjId filter by data source, if configured in + * user preferences */ FileSizeNode(SleuthkitCase skCase, FileSizeFilter filter, Observable o, long datasourceObjId) { super(Children.create(new FileSizeChildren(filter, skCase, o, datasourceObjId), true), Lookups.singleton(filter.getDisplayName())); @@ -379,7 +383,7 @@ public class FileSize implements AutopsyVisitableItem { this.filter = filter; this.notifier = o; this.datasourceObjId = dsObjId; - + } @Override @@ -429,15 +433,15 @@ public class FileSize implements AutopsyVisitableItem { default: throw new IllegalArgumentException("Unsupported filter type to get files by size: " + filter); //NON-NLS } - + // Ignore unallocated block files. query = query + " AND (type != " + TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS.getFileType() + ")"; //NON-NLS - + // filter by datasource if indicated in case preferences if (filteringDSObjId > 0) { - query += " AND data_source_obj_id = " + filteringDSObjId; + query += " AND data_source_obj_id = " + filteringDSObjId; } - + return query; } diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/FileTypesByExtension.java b/Core/src/org/sleuthkit/autopsy/datamodel/FileTypesByExtension.java index 34ec74280a..19dac06410 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/FileTypesByExtension.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/FileTypesByExtension.java @@ -53,6 +53,8 @@ import org.sleuthkit.datamodel.TskData; public final class FileTypesByExtension implements AutopsyVisitableItem { private final static Logger logger = Logger.getLogger(FileTypesByExtension.class.getName()); + private static final Set INGEST_JOB_EVENTS_OF_INTEREST = EnumSet.of(IngestManager.IngestJobEvent.COMPLETED, IngestManager.IngestJobEvent.CANCELLED); + private static final Set INGEST_MODULE_EVENTS_OF_INTEREST = EnumSet.of(IngestManager.IngestModuleEvent.CONTENT_CHANGED); private final SleuthkitCase skCase; private final FileTypes typesRoot; @@ -72,8 +74,8 @@ public final class FileTypesByExtension implements AutopsyVisitableItem { long filteringDataSourceObjId() { return typesRoot.filteringDataSourceObjId(); - } - + } + /** * Listens for case and ingest invest. Updates observers when events are * fired. FileType and FileTypes nodes are all listening to this. @@ -115,8 +117,8 @@ public final class FileTypesByExtension implements AutopsyVisitableItem { } }; - IngestManager.getInstance().addIngestJobEventListener(pcl); - IngestManager.getInstance().addIngestModuleEventListener(pcl); + IngestManager.getInstance().addIngestJobEventListener(INGEST_JOB_EVENTS_OF_INTEREST, pcl); + IngestManager.getInstance().addIngestModuleEventListener(INGEST_MODULE_EVENTS_OF_INTEREST, pcl); Case.addEventTypeSubscriber(CASE_EVENTS_OF_INTEREST, pcl); } @@ -365,11 +367,11 @@ public final class FileTypesByExtension implements AutopsyVisitableItem { ? " AND (known IS NULL OR known != " + TskData.FileKnown.KNOWN.getFileKnownValue() + ")" : " ") + (filteringDataSourceObjId() > 0 - ? " AND data_source_obj_id = " + filteringDataSourceObjId() - : " ") + ? " AND data_source_obj_id = " + filteringDataSourceObjId() + : " ") + " AND (extension IN (" + filter.getFilter().stream() .map(String::toLowerCase) - .map(s -> "'"+StringUtils.substringAfter(s, ".")+"'") + .map(s -> "'" + StringUtils.substringAfter(s, ".") + "'") .collect(Collectors.joining(", ")) + "))"; } @@ -384,10 +386,10 @@ public final class FileTypesByExtension implements AutopsyVisitableItem { /** * - * @param filter Extensions to display + * @param filter Extensions to display * @param skCase - * @param o Observable that will notify when there could be new - * data to display + * @param o Observable that will notify when there could be new + * data to display * @param nodeName */ private FileExtensionNodeChildren(FileTypesByExtension.SearchFilterInterface filter, SleuthkitCase skCase, Observable o, String nodeName) { diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/FileTypesByMimeType.java b/Core/src/org/sleuthkit/autopsy/datamodel/FileTypesByMimeType.java index 86cc42aa8c..0ac8f8e8b0 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/FileTypesByMimeType.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/FileTypesByMimeType.java @@ -61,7 +61,7 @@ import org.sleuthkit.datamodel.TskData; public final class FileTypesByMimeType extends Observable implements AutopsyVisitableItem { private final static Logger logger = Logger.getLogger(FileTypesByMimeType.class.getName()); - + private static final Set INGEST_JOB_EVENTS_OF_INTEREST = EnumSet.of(IngestManager.IngestJobEvent.COMPLETED, IngestManager.IngestJobEvent.CANCELLED); private final SleuthkitCase skCase; /** * The nodes of this tree will be determined dynamically by the mimetypes @@ -99,9 +99,9 @@ public final class FileTypesByMimeType extends Observable implements AutopsyVisi + TskData.TSK_DB_FILES_TYPE_ENUM.DERIVED.ordinal() + "," + TskData.TSK_DB_FILES_TYPE_ENUM.LAYOUT_FILE.ordinal() + "," + TskData.TSK_DB_FILES_TYPE_ENUM.LOCAL.ordinal() - + (hideSlackFilesInViewsTree() ? "" : ("," + TskData.TSK_DB_FILES_TYPE_ENUM.SLACK.ordinal())) + + (hideSlackFilesInViewsTree() ? "" : ("," + TskData.TSK_DB_FILES_TYPE_ENUM.SLACK.ordinal())) + "))" - + ( (filteringDataSourceObjId() > 0) ? " AND data_source_obj_id = " + this.filteringDataSourceObjId() : " ") + + ((filteringDataSourceObjId() > 0) ? " AND data_source_obj_id = " + this.filteringDataSourceObjId() : " ") + (hideKnownFilesInViewsTree() ? (" AND (known IS NULL OR known != " + TskData.FileKnown.KNOWN.getFileKnownValue() + ")") : ""); } @@ -180,7 +180,7 @@ public final class FileTypesByMimeType extends Observable implements AutopsyVisi } } }; - IngestManager.getInstance().addIngestJobEventListener(pcl); + IngestManager.getInstance().addIngestJobEventListener(INGEST_JOB_EVENTS_OF_INTEREST, pcl); Case.addEventTypeSubscriber(CASE_EVENTS_OF_INTEREST, pcl); populateHashMap(); } @@ -193,7 +193,7 @@ public final class FileTypesByMimeType extends Observable implements AutopsyVisi long filteringDataSourceObjId() { return typesRoot.filteringDataSourceObjId(); } - + /** * Method to check if the node in question is a ByMimeTypeNode which is * empty. diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/HashsetHits.java b/Core/src/org/sleuthkit/autopsy/datamodel/HashsetHits.java index 03c72d2d2e..3a9a87c1e4 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/HashsetHits.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/HashsetHits.java @@ -1,15 +1,15 @@ /* * Autopsy Forensic Browser - * + * * Copyright 2011-2019 Basis Technology Corp. * Contact: carrier sleuthkit org - * + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -61,28 +61,29 @@ public class HashsetHits implements AutopsyVisitableItem { private static final String HASHSET_HITS = BlackboardArtifact.ARTIFACT_TYPE.TSK_HASHSET_HIT.getLabel(); private static final String DISPLAY_NAME = BlackboardArtifact.ARTIFACT_TYPE.TSK_HASHSET_HIT.getDisplayName(); private static final Logger logger = Logger.getLogger(HashsetHits.class.getName()); + private static final Set INGEST_JOB_EVENTS_OF_INTEREST = EnumSet.of(IngestManager.IngestJobEvent.COMPLETED, IngestManager.IngestJobEvent.CANCELLED); + private static final Set INGEST_MODULE_EVENTS_OF_INTEREST = EnumSet.of(IngestManager.IngestModuleEvent.DATA_ADDED); private SleuthkitCase skCase; private final HashsetResults hashsetResults; private final long filteringDSObjId; // 0 if not filtering/grouping by data source - - + /** * Constructor - * - * @param skCase Case DB - * - */ + * + * @param skCase Case DB + * + */ public HashsetHits(SleuthkitCase skCase) { this(skCase, 0); } - + /** * Constructor - * - * @param skCase Case DB - * @param objId Object id of the data source - * - */ + * + * @param skCase Case DB + * @param objId Object id of the data source + * + */ public HashsetHits(SleuthkitCase skCase, long objId) { this.skCase = skCase; this.filteringDSObjId = objId; @@ -118,7 +119,7 @@ public class HashsetHits implements AutopsyVisitableItem { } Set getArtifactIds(String hashSetName) { - synchronized (hashSetHitsMap) { + synchronized (hashSetHitsMap) { return hashSetHitsMap.get(hashSetName); } } @@ -141,9 +142,9 @@ public class HashsetHits implements AutopsyVisitableItem { + " AND blackboard_attributes.artifact_id=blackboard_artifacts.artifact_id" //NON-NLS + " AND blackboard_artifacts.artifact_type_id=" + artId; //NON-NLS if (filteringDSObjId > 0) { - query += " AND blackboard_artifacts.data_source_obj_id = " + filteringDSObjId; + query += " AND blackboard_artifacts.data_source_obj_id = " + filteringDSObjId; } - + try (CaseDbQuery dbQuery = skCase.executeQuery(query)) { ResultSet resultSet = dbQuery.getResultSet(); synchronized (hashSetHitsMap) { @@ -275,8 +276,8 @@ public class HashsetHits implements AutopsyVisitableItem { @Override protected void addNotify() { - IngestManager.getInstance().addIngestJobEventListener(pcl); - IngestManager.getInstance().addIngestModuleEventListener(pcl); + IngestManager.getInstance().addIngestJobEventListener(INGEST_JOB_EVENTS_OF_INTEREST, pcl); + IngestManager.getInstance().addIngestModuleEventListener(INGEST_MODULE_EVENTS_OF_INTEREST, pcl); Case.addEventTypeSubscriber(EnumSet.of(Case.Events.CURRENT_CASE), pcl); hashsetResults.update(); hashsetResults.addObserver(this); @@ -379,7 +380,7 @@ public class HashsetHits implements AutopsyVisitableItem { private String hashsetName; private Map artifactHits = new HashMap<>(); - + private HitFactory(String hashsetName) { super(hashsetName); this.hashsetName = hashsetName; @@ -396,7 +397,7 @@ public class HashsetHits implements AutopsyVisitableItem { } @Override - protected Node createNodeForKey(BlackboardArtifact key) { + protected Node createNodeForKey(BlackboardArtifact key) { return new BlackboardArtifactNode(key); } diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/ImageNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/ImageNode.java index da82c315b5..12f2340f33 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/ImageNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/ImageNode.java @@ -26,6 +26,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.EnumSet; import java.util.List; +import java.util.Set; import java.util.logging.Level; import javax.swing.Action; import org.apache.commons.lang3.tuple.Pair; @@ -59,6 +60,7 @@ import org.sleuthkit.datamodel.Tag; public class ImageNode extends AbstractContentNode { private static final Logger logger = Logger.getLogger(ImageNode.class.getName()); + private static final Set INGEST_MODULE_EVENTS_OF_INTEREST = EnumSet.of(IngestManager.IngestModuleEvent.CONTENT_CHANGED); /** * Helper so that the display name and the name used in building the path @@ -84,7 +86,7 @@ public class ImageNode extends AbstractContentNode { this.setIconBaseWithExtension("org/sleuthkit/autopsy/images/hard-drive-icon.jpg"); //NON-NLS // Listen for ingest events so that we can detect new added files (e.g. carved) - IngestManager.getInstance().addIngestModuleEventListener(pcl); + IngestManager.getInstance().addIngestModuleEventListener(INGEST_MODULE_EVENTS_OF_INTEREST, pcl); // Listen for case events so that we can detect when case is closed Case.addEventTypeSubscriber(EnumSet.of(Case.Events.CURRENT_CASE), pcl); } diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/InterestingHits.java b/Core/src/org/sleuthkit/autopsy/datamodel/InterestingHits.java index 482ebf1558..b12ceff6ae 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/InterestingHits.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/InterestingHits.java @@ -57,27 +57,29 @@ public class InterestingHits implements AutopsyVisitableItem { .getMessage(InterestingHits.class, "InterestingHits.interestingItems.text"); private static final String DISPLAY_NAME = NbBundle.getMessage(InterestingHits.class, "InterestingHits.displayName.text"); private static final Logger logger = Logger.getLogger(InterestingHits.class.getName()); + private static final Set INGEST_JOB_EVENTS_OF_INTEREST = EnumSet.of(IngestManager.IngestJobEvent.COMPLETED, IngestManager.IngestJobEvent.CANCELLED); + private static final Set INGEST_MODULE_EVENTS_OF_INTEREST = EnumSet.of(IngestManager.IngestModuleEvent.DATA_ADDED); private SleuthkitCase skCase; private final InterestingResults interestingResults = new InterestingResults(); private final long filteringDSObjId; // 0 if not filtering/grouping by data source /** * Constructor - * - * @param skCase Case DB - * - */ + * + * @param skCase Case DB + * + */ public InterestingHits(SleuthkitCase skCase) { this(skCase, 0); } - + /** * Constructor - * - * @param skCase Case DB - * @param objId Object id of the data source - * - */ + * + * @param skCase Case DB + * @param objId Object id of the data source + * + */ public InterestingHits(SleuthkitCase skCase, long objId) { this.skCase = skCase; this.filteringDSObjId = objId; @@ -132,7 +134,7 @@ public class InterestingHits implements AutopsyVisitableItem { + " AND blackboard_attributes.artifact_id=blackboard_artifacts.artifact_id" //NON-NLS + " AND blackboard_artifacts.artifact_type_id=" + artId; //NON-NLS if (filteringDSObjId > 0) { - query += " AND blackboard_artifacts.data_source_obj_id = " + filteringDSObjId; + query += " AND blackboard_artifacts.data_source_obj_id = " + filteringDSObjId; } try (CaseDbQuery dbQuery = skCase.executeQuery(query)) { @@ -217,17 +219,17 @@ public class InterestingHits implements AutopsyVisitableItem { if (eventType.equals(IngestManager.IngestModuleEvent.DATA_ADDED.toString())) { /** * Checking for a current case is a stop gap measure until a - * different way of handling the closing of cases is worked - * out. Currently, remote events may be received for a case - * that is already closed. + * different way of handling the closing of cases is worked out. + * Currently, remote events may be received for a case that is + * already closed. */ try { Case.getCurrentCaseThrows(); /** - * Even with the check above, it is still possible that - * the case will be closed in a different thread before - * this code executes. If that happens, it is possible - * for the event to have a null oldValue. + * Even with the check above, it is still possible that the + * case will be closed in a different thread before this + * code executes. If that happens, it is possible for the + * event to have a null oldValue. */ ModuleDataEvent eventData = (ModuleDataEvent) evt.getOldValue(); if (null != eventData && (eventData.getBlackboardArtifactType().getTypeID() == BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_ARTIFACT_HIT.getTypeID() @@ -243,9 +245,9 @@ public class InterestingHits implements AutopsyVisitableItem { || eventType.equals(IngestManager.IngestJobEvent.CANCELLED.toString())) { /** * Checking for a current case is a stop gap measure until a - * different way of handling the closing of cases is worked - * out. Currently, remote events may be received for a case - * that is already closed. + * different way of handling the closing of cases is worked out. + * Currently, remote events may be received for a case that is + * already closed. */ try { Case.getCurrentCaseThrows(); @@ -266,8 +268,8 @@ public class InterestingHits implements AutopsyVisitableItem { @Override protected void addNotify() { - IngestManager.getInstance().addIngestJobEventListener(pcl); - IngestManager.getInstance().addIngestModuleEventListener(pcl); + IngestManager.getInstance().addIngestJobEventListener(INGEST_JOB_EVENTS_OF_INTEREST, pcl); + IngestManager.getInstance().addIngestModuleEventListener(INGEST_MODULE_EVENTS_OF_INTEREST, pcl); Case.addEventTypeSubscriber(EnumSet.of(Case.Events.CURRENT_CASE), pcl); interestingResults.update(); interestingResults.addObserver(this); diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/KeywordHits.java b/Core/src/org/sleuthkit/autopsy/datamodel/KeywordHits.java index 655f0c1973..4f78a6da34 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/KeywordHits.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/KeywordHits.java @@ -62,7 +62,8 @@ import org.sleuthkit.datamodel.TskCoreException; public class KeywordHits implements AutopsyVisitableItem { private static final Logger logger = Logger.getLogger(KeywordHits.class.getName()); - + private static final Set INGEST_JOB_EVENTS_OF_INTEREST = EnumSet.of(IngestManager.IngestJobEvent.COMPLETED, IngestManager.IngestJobEvent.CANCELLED); + private static final Set INGEST_MODULE_EVENTS_OF_INTEREST = EnumSet.of(IngestManager.IngestModuleEvent.DATA_ADDED); @NbBundle.Messages("KeywordHits.kwHits.text=Keyword Hits") private static final String KEYWORD_HITS = KeywordHits_kwHits_text(); @NbBundle.Messages("KeywordHits.simpleLiteralSearch.text=Single Literal Keyword Search") @@ -155,25 +156,22 @@ public class KeywordHits implements AutopsyVisitableItem { Collections.sort(names, new Comparator() { @Override - public int compare(String o1, String o2) { + public int compare(String o1, String o2) { // ideally, they would not be hard coded, but this module // doesn't know about Keyword Search NBM if (o1.startsWith("Single Literal Keyword Search")) { return -1; - } - else if (o2.startsWith("Single Literal Keyword Search")) { + } else if (o2.startsWith("Single Literal Keyword Search")) { return 1; - } - else if (o1.startsWith("Single Regular Expression Search")) { + } else if (o1.startsWith("Single Regular Expression Search")) { return -1; - } - else if (o2.startsWith("Single Regular Expression Search")) { + } else if (o2.startsWith("Single Regular Expression Search")) { return 1; } return o1.compareTo(o2); } }); - + return names; } } @@ -501,8 +499,8 @@ public class KeywordHits implements AutopsyVisitableItem { @Override protected void addNotify() { - IngestManager.getInstance().addIngestJobEventListener(pcl); - IngestManager.getInstance().addIngestModuleEventListener(pcl); + IngestManager.getInstance().addIngestJobEventListener(INGEST_JOB_EVENTS_OF_INTEREST, pcl); + IngestManager.getInstance().addIngestModuleEventListener(INGEST_MODULE_EVENTS_OF_INTEREST, pcl); Case.addEventTypeSubscriber(EnumSet.of(Case.Events.CURRENT_CASE), pcl); keywordResults.update(); super.addNotify(); @@ -529,8 +527,9 @@ public class KeywordHits implements AutopsyVisitableItem { } private abstract class KWHitsNodeBase extends DisplayableItemNode implements Observer { + private String displayName; - + private KWHitsNodeBase(Children children, Lookup lookup, String displayName) { super(children, lookup); this.displayName = displayName; diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/Tags.java b/Core/src/org/sleuthkit/autopsy/datamodel/Tags.java index 2dfc02678b..8dd7805358 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/Tags.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/Tags.java @@ -54,9 +54,10 @@ public class Tags implements AutopsyVisitableItem { // by a CreateAutopsyNodeVisitor dispatched from the AbstractContentChildren // override of Children.Keys.createNodes(). - private final TagResults tagResults = new TagResults(); private final static String DISPLAY_NAME = NbBundle.getMessage(RootNode.class, "TagsNode.displayName.text"); + private static final Set INGEST_JOB_EVENTS_OF_INTEREST = EnumSet.of(IngestManager.IngestJobEvent.COMPLETED, IngestManager.IngestJobEvent.CANCELLED); private static final String USER_NAME_PROPERTY = "user.name"; //NON-NLS + private final TagResults tagResults = new TagResults(); private final String ICON_PATH = "org/sleuthkit/autopsy/images/tag-folder-blue-icon-16.png"; //NON-NLS private final long filteringDSObjId; // 0 if not filtering/grouping by data source @@ -223,8 +224,7 @@ public class Tags implements AutopsyVisitableItem { @Override protected void addNotify() { - IngestManager.getInstance().addIngestJobEventListener(pcl); - IngestManager.getInstance().addIngestModuleEventListener(pcl); + IngestManager.getInstance().addIngestJobEventListener(INGEST_JOB_EVENTS_OF_INTEREST, pcl); Case.addEventTypeSubscriber(CASE_EVENTS_OF_INTEREST, pcl); tagResults.update(); tagResults.addObserver(this); @@ -233,7 +233,6 @@ public class Tags implements AutopsyVisitableItem { @Override protected void removeNotify() { IngestManager.getInstance().removeIngestJobEventListener(pcl); - IngestManager.getInstance().removeIngestModuleEventListener(pcl); Case.removeEventTypeSubscriber(CASE_EVENTS_OF_INTEREST, pcl); tagResults.deleteObserver(this); } diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/VolumeNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/VolumeNode.java index 307013136f..0eacf5f699 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/VolumeNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/VolumeNode.java @@ -23,6 +23,7 @@ import java.beans.PropertyChangeListener; import java.util.ArrayList; import java.util.EnumSet; import java.util.List; +import java.util.Set; import java.util.logging.Level; import javax.swing.Action; import org.apache.commons.lang3.tuple.Pair; @@ -52,6 +53,7 @@ import org.sleuthkit.datamodel.Tag; public class VolumeNode extends AbstractContentNode { private static final Logger logger = Logger.getLogger(VolumeNode.class.getName()); + private static final Set INGEST_MODULE_EVENTS_OF_INTEREST = EnumSet.of(IngestManager.IngestModuleEvent.CONTENT_CHANGED); /** * Helper so that the display name and the name used in building the path @@ -81,7 +83,7 @@ public class VolumeNode extends AbstractContentNode { this.setIconBaseWithExtension("org/sleuthkit/autopsy/images/vol-icon.png"); //NON-NLS // Listen for ingest events so that we can detect new added files (e.g. carved) - IngestManager.getInstance().addIngestModuleEventListener(pcl); + IngestManager.getInstance().addIngestModuleEventListener(INGEST_MODULE_EVENTS_OF_INTEREST, pcl); // Listen for case events so that we can detect when case is closed Case.addEventTypeSubscriber(EnumSet.of(Case.Events.CURRENT_CASE), pcl); } diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/accounts/Accounts.java b/Core/src/org/sleuthkit/autopsy/datamodel/accounts/Accounts.java index a75bca2972..0e3a4e971d 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/accounts/Accounts.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/accounts/Accounts.java @@ -89,16 +89,20 @@ final public class Accounts implements AutopsyVisitableItem { private static final Logger LOGGER = Logger.getLogger(Accounts.class.getName()); private static final String ICON_BASE_PATH = "/org/sleuthkit/autopsy/images/"; //NON-NLS - + private static final Set INGEST_JOB_EVENTS_OF_INTEREST = EnumSet.of(IngestManager.IngestJobEvent.COMPLETED, IngestManager.IngestJobEvent.CANCELLED); + private static final Set INGEST_MODULE_EVENTS_OF_INTEREST = EnumSet.of(IngestManager.IngestModuleEvent.DATA_ADDED); + @NbBundle.Messages("AccountsRootNode.name=Accounts") final public static String NAME = Bundle.AccountsRootNode_name(); private SleuthkitCase skCase; private final long filteringDSObjId; // 0 if not filtering/grouping by data source - + private final EventBus reviewStatusBus = new EventBus("ReviewStatusBus"); - /* Should rejected accounts be shown in the accounts section of the tree. */ + /* + * Should rejected accounts be shown in the accounts section of the tree. + */ private boolean showRejected = false; //NOPMD redundant initializer private final RejectAccounts rejectActionInstance; @@ -117,7 +121,7 @@ final public class Accounts implements AutopsyVisitableItem { * Constructor * * @param skCase The SleuthkitCase object to use for db queries. - * @param objId Object id of the data source + * @param objId Object id of the data source */ public Accounts(SleuthkitCase skCase, long objId) { this.skCase = skCase; @@ -126,8 +130,7 @@ final public class Accounts implements AutopsyVisitableItem { this.rejectActionInstance = new RejectAccounts(); this.approveActionInstance = new ApproveAccounts(); } - - + @Override public T accept(AutopsyItemVisitor visitor) { return visitor.visit(this); @@ -147,14 +150,14 @@ final public class Accounts implements AutopsyVisitableItem { /** * Returns the clause to filter artifacts by data source. * - * @return A clause that will or will not filter artifacts by datasource - * based on the CasePreferences groupItemsInTreeByDataSource setting + * @return A clause that will or will not filter artifacts by datasource + * based on the CasePreferences groupItemsInTreeByDataSource setting */ private String getFilterByDataSourceClause() { if (filteringDSObjId > 0) { return " AND blackboard_artifacts.data_source_obj_id = " + filteringDSObjId + " "; } - + return " "; } @@ -320,14 +323,14 @@ final public class Accounts implements AutopsyVisitableItem { @Override protected boolean createKeys(List list) { - String accountTypesInUseQuery = - "SELECT DISTINCT blackboard_attributes.value_text as account_type " + String accountTypesInUseQuery + = "SELECT DISTINCT blackboard_attributes.value_text as account_type " + " FROM blackboard_artifacts " //NON-NLS + " JOIN blackboard_attributes ON blackboard_artifacts.artifact_id = blackboard_attributes.artifact_id " //NON-NLS + " WHERE blackboard_attributes.attribute_type_id = " + BlackboardAttribute.ATTRIBUTE_TYPE.TSK_ACCOUNT_TYPE.getTypeID() + getFilterByDataSourceClause(); - - try (SleuthkitCase.CaseDbQuery executeQuery = skCase.executeQuery(accountTypesInUseQuery ); + + try (SleuthkitCase.CaseDbQuery executeQuery = skCase.executeQuery(accountTypesInUseQuery); ResultSet resultSet = executeQuery.getResultSet()) { while (resultSet.next()) { String accountType = resultSet.getString("account_type"); @@ -368,8 +371,8 @@ final public class Accounts implements AutopsyVisitableItem { @Override protected void addNotify() { - IngestManager.getInstance().addIngestJobEventListener(pcl); - IngestManager.getInstance().addIngestModuleEventListener(pcl); + IngestManager.getInstance().addIngestJobEventListener(INGEST_JOB_EVENTS_OF_INTEREST, pcl); + IngestManager.getInstance().addIngestModuleEventListener(INGEST_MODULE_EVENTS_OF_INTEREST, pcl); Case.addEventTypeSubscriber(EnumSet.of(Case.Events.CURRENT_CASE), pcl); super.addNotify(); refresh(true); @@ -439,8 +442,8 @@ final public class Accounts implements AutopsyVisitableItem { @Override protected void addNotify() { - IngestManager.getInstance().addIngestJobEventListener(pcl); - IngestManager.getInstance().addIngestModuleEventListener(pcl); + IngestManager.getInstance().addIngestJobEventListener(INGEST_JOB_EVENTS_OF_INTEREST, pcl); + IngestManager.getInstance().addIngestModuleEventListener(INGEST_MODULE_EVENTS_OF_INTEREST, pcl); Case.addEventTypeSubscriber(EnumSet.of(Case.Events.CURRENT_CASE), pcl); super.addNotify(); } @@ -455,8 +458,8 @@ final public class Accounts implements AutopsyVisitableItem { @Override protected boolean createKeys(List list) { - String query = - "SELECT blackboard_artifacts.artifact_id " //NON-NLS + String query + = "SELECT blackboard_artifacts.artifact_id " //NON-NLS + " FROM blackboard_artifacts " //NON-NLS + " JOIN blackboard_attributes ON blackboard_artifacts.artifact_id = blackboard_attributes.artifact_id " //NON-NLS + " WHERE blackboard_artifacts.artifact_type_id = " + BlackboardArtifact.ARTIFACT_TYPE.TSK_ACCOUNT.getTypeID() //NON-NLS @@ -603,8 +606,8 @@ final public class Accounts implements AutopsyVisitableItem { @Override protected void addNotify() { - IngestManager.getInstance().addIngestJobEventListener(pcl); - IngestManager.getInstance().addIngestModuleEventListener(pcl); + IngestManager.getInstance().addIngestJobEventListener(INGEST_JOB_EVENTS_OF_INTEREST, pcl); + IngestManager.getInstance().addIngestModuleEventListener(INGEST_MODULE_EVENTS_OF_INTEREST, pcl); Case.addEventTypeSubscriber(EnumSet.of(Case.Events.CURRENT_CASE), pcl); super.addNotify(); } @@ -727,8 +730,8 @@ final public class Accounts implements AutopsyVisitableItem { @Override protected void addNotify() { - IngestManager.getInstance().addIngestJobEventListener(pcl); - IngestManager.getInstance().addIngestModuleEventListener(pcl); + IngestManager.getInstance().addIngestJobEventListener(INGEST_JOB_EVENTS_OF_INTEREST, pcl); + IngestManager.getInstance().addIngestModuleEventListener(INGEST_MODULE_EVENTS_OF_INTEREST, pcl); Case.addEventTypeSubscriber(EnumSet.of(Case.Events.CURRENT_CASE), pcl); super.addNotify(); } @@ -755,8 +758,8 @@ final public class Accounts implements AutopsyVisitableItem { @Override protected boolean createKeys(List list) { - String query = - "SELECT blackboard_artifacts.obj_id," //NON-NLS + String query + = "SELECT blackboard_artifacts.obj_id," //NON-NLS + " solr_attribute.value_text AS solr_document_id, "; //NON-NLS if (skCase.getDatabaseType().equals(DbType.POSTGRESQL)) { query += " string_agg(blackboard_artifacts.artifact_id::character varying, ',') AS artifact_IDs, " //NON-NLS @@ -833,8 +836,8 @@ final public class Accounts implements AutopsyVisitableItem { "# {0} - number of children", "Accounts.ByFileNode.displayName=By File ({0})"}) private void updateDisplayName() { - String query = - "SELECT count(*) FROM ( SELECT count(*) AS documents " + String query + = "SELECT count(*) FROM ( SELECT count(*) AS documents " + " FROM blackboard_artifacts " //NON-NLS + " LEFT JOIN blackboard_attributes as solr_attribute ON blackboard_artifacts.artifact_id = solr_attribute.artifact_id " //NON-NLS + " AND solr_attribute.attribute_type_id = " + BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_SEARCH_DOCUMENT_ID.getTypeID() //NON-NLS @@ -842,7 +845,7 @@ final public class Accounts implements AutopsyVisitableItem { + " AND account_type.attribute_type_id = " + BlackboardAttribute.ATTRIBUTE_TYPE.TSK_ACCOUNT_TYPE.getTypeID() //NON-NLS + " AND account_type.value_text = '" + Account.Type.CREDIT_CARD.getTypeName() + "'" //NON-NLS + " WHERE blackboard_artifacts.artifact_type_id = " + BlackboardArtifact.ARTIFACT_TYPE.TSK_ACCOUNT.getTypeID() //NON-NLS - + getFilterByDataSourceClause() + + getFilterByDataSourceClause() + getRejectedArtifactFilterClause() + " GROUP BY blackboard_artifacts.obj_id, solr_attribute.value_text ) AS foo"; try (SleuthkitCase.CaseDbQuery results = skCase.executeQuery(query); @@ -941,8 +944,8 @@ final public class Accounts implements AutopsyVisitableItem { @Override protected void addNotify() { - IngestManager.getInstance().addIngestJobEventListener(pcl); - IngestManager.getInstance().addIngestModuleEventListener(pcl); + IngestManager.getInstance().addIngestJobEventListener(INGEST_JOB_EVENTS_OF_INTEREST, pcl); + IngestManager.getInstance().addIngestModuleEventListener(INGEST_MODULE_EVENTS_OF_INTEREST, pcl); Case.addEventTypeSubscriber(EnumSet.of(Case.Events.CURRENT_CASE), pcl); super.addNotify(); } @@ -972,14 +975,14 @@ final public class Accounts implements AutopsyVisitableItem { RangeMap binRanges = TreeRangeMap.create(); - String query = - "SELECT SUBSTR(blackboard_attributes.value_text,1,8) AS BIN, " //NON-NLS + String query + = "SELECT SUBSTR(blackboard_attributes.value_text,1,8) AS BIN, " //NON-NLS + " COUNT(blackboard_artifacts.artifact_id) AS count " //NON-NLS + " FROM blackboard_artifacts " //NON-NLS + " JOIN blackboard_attributes ON blackboard_artifacts.artifact_id = blackboard_attributes.artifact_id" //NON-NLS + " WHERE blackboard_artifacts.artifact_type_id = " + BlackboardArtifact.ARTIFACT_TYPE.TSK_ACCOUNT.getTypeID() //NON-NLS + " AND blackboard_attributes.attribute_type_id = " + BlackboardAttribute.ATTRIBUTE_TYPE.TSK_CARD_NUMBER.getTypeID() //NON-NLS - + getFilterByDataSourceClause() + + getFilterByDataSourceClause() + getRejectedArtifactFilterClause() + " GROUP BY BIN " //NON-NLS + " ORDER BY BIN "; //NON-NLS @@ -1040,13 +1043,13 @@ final public class Accounts implements AutopsyVisitableItem { "# {0} - number of children", "Accounts.ByBINNode.displayName=By BIN ({0})"}) private void updateDisplayName() { - String query = - "SELECT count(distinct SUBSTR(blackboard_attributes.value_text,1,8)) AS BINs " //NON-NLS + String query + = "SELECT count(distinct SUBSTR(blackboard_attributes.value_text,1,8)) AS BINs " //NON-NLS + " FROM blackboard_artifacts " //NON-NLS + " JOIN blackboard_attributes ON blackboard_artifacts.artifact_id = blackboard_attributes.artifact_id" //NON-NLS + " WHERE blackboard_artifacts.artifact_type_id = " + BlackboardArtifact.ARTIFACT_TYPE.TSK_ACCOUNT.getTypeID() //NON-NLS + " AND blackboard_attributes.attribute_type_id = " + BlackboardAttribute.ATTRIBUTE_TYPE.TSK_CARD_NUMBER.getTypeID() //NON-NLS - + getFilterByDataSourceClause() + + getFilterByDataSourceClause() + getRejectedArtifactFilterClause(); //NON-NLS try (SleuthkitCase.CaseDbQuery results = skCase.executeQuery(query); ResultSet resultSet = results.getResultSet();) { @@ -1335,14 +1338,14 @@ final public class Accounts implements AutopsyVisitableItem { @Override protected boolean createKeys(List list) { - String query = - "SELECT blackboard_artifacts.artifact_id " //NON-NLS + String query + = "SELECT blackboard_artifacts.artifact_id " //NON-NLS + " FROM blackboard_artifacts " //NON-NLS + " JOIN blackboard_attributes ON blackboard_artifacts.artifact_id = blackboard_attributes.artifact_id " //NON-NLS + " WHERE blackboard_artifacts.artifact_type_id = " + BlackboardArtifact.ARTIFACT_TYPE.TSK_ACCOUNT.getTypeID() //NON-NLS + " AND blackboard_attributes.attribute_type_id = " + BlackboardAttribute.ATTRIBUTE_TYPE.TSK_CARD_NUMBER.getTypeID() //NON-NLS + " AND blackboard_attributes.value_text >= '" + bin.getBINStart() + "' AND blackboard_attributes.value_text < '" + (bin.getBINEnd() + 1) + "'" //NON-NLS - + getFilterByDataSourceClause() + + getFilterByDataSourceClause() + getRejectedArtifactFilterClause() + " ORDER BY blackboard_attributes.value_text"; //NON-NLS try (SleuthkitCase.CaseDbQuery results = skCase.executeQuery(query); @@ -1383,7 +1386,9 @@ final public class Accounts implements AutopsyVisitableItem { final public class BINNode extends DisplayableItemNode { - /** Creates the nodes for the credit card numbers */ + /** + * Creates the nodes for the credit card numbers + */ private final BinResult bin; private BINNode(BinResult bin) { @@ -1407,14 +1412,14 @@ final public class Accounts implements AutopsyVisitableItem { } private void updateDisplayName() { - String query = - "SELECT count(blackboard_artifacts.artifact_id ) AS count" //NON-NLS + String query + = "SELECT count(blackboard_artifacts.artifact_id ) AS count" //NON-NLS + " FROM blackboard_artifacts " //NON-NLS + " JOIN blackboard_attributes ON blackboard_artifacts.artifact_id = blackboard_attributes.artifact_id " //NON-NLS + " WHERE blackboard_artifacts.artifact_type_id = " + BlackboardArtifact.ARTIFACT_TYPE.TSK_ACCOUNT.getTypeID() //NON-NLS + " AND blackboard_attributes.attribute_type_id = " + BlackboardAttribute.ATTRIBUTE_TYPE.TSK_CARD_NUMBER.getTypeID() //NON-NLS + " AND blackboard_attributes.value_text >= '" + bin.getBINStart() + "' AND blackboard_attributes.value_text < '" + (bin.getBINEnd() + 1) + "'" //NON-NLS - + getFilterByDataSourceClause() + + getFilterByDataSourceClause() + getRejectedArtifactFilterClause(); try (SleuthkitCase.CaseDbQuery results = skCase.executeQuery(query); ResultSet resultSet = results.getResultSet();) { @@ -1549,7 +1554,9 @@ final public class Accounts implements AutopsyVisitableItem { return true; } - /** The number of accounts with this BIN */ + /** + * The number of accounts with this BIN + */ private final long count; private final BINRange binRange; @@ -1702,10 +1709,10 @@ final public class Accounts implements AutopsyVisitableItem { reviewStatusBus.post(new ReviewStatusChangeEvent(Collections.emptySet(), null)); } } - + /** * Update the user interface to show or hide rejected artifacts. - * + * * @param showRejected Show rejected artifacts? Yes if true; otherwise no. */ public void setShowRejected(boolean showRejected) { @@ -1726,8 +1733,10 @@ final public class Accounts implements AutopsyVisitableItem { @Override public void actionPerformed(ActionEvent e) { - /* get paths for selected nodes to reselect after applying review - * status change */ + /* + * get paths for selected nodes to reselect after applying review + * status change + */ List selectedPaths = Utilities.actionsGlobalContext().lookupAll(Node.class).stream() .map(node -> { String[] createPath; @@ -1746,9 +1755,11 @@ final public class Accounts implements AutopsyVisitableItem { : siblings.get(Integer.max(indexOf + 1, siblings.size() - 1)); createPath = NodeOp.createPath(sibling, null); } else { - /* if there are no other siblings to select, + /* + * if there are no other siblings to select, * just return null, but note we need to filter - * this out of stream below */ + * this out of stream below + */ return null; } } else { diff --git a/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java b/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java index 280ccd3c3a..42d285ef65 100644 --- a/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java +++ b/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java @@ -85,7 +85,6 @@ import org.sleuthkit.autopsy.datamodel.Tags; import org.sleuthkit.autopsy.datamodel.ViewsNode; import org.sleuthkit.autopsy.datamodel.accounts.Accounts; import org.sleuthkit.autopsy.datamodel.accounts.BINRange; -import org.sleuthkit.autopsy.ingest.IngestManager; import org.sleuthkit.datamodel.Account; import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardAttribute; @@ -128,7 +127,7 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat getTree().setSelectionMode(TreeSelectionModel.SINGLE_TREE_SELECTION); //Hook into the JTree and pre-expand the Views Node and Results node when a user //expands an item in the tree that makes these nodes visible. - ((ExpansionBeanTreeView )getTree()).addTreeExpansionListener(new TreeExpansionListener() { + ((ExpansionBeanTreeView) getTree()).addTreeExpansionListener(new TreeExpansionListener() { @Override public void treeExpanded(TreeExpansionEvent event) { //Bail immediately if we are not in the Group By view. @@ -238,8 +237,6 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat Case.addEventTypeSubscriber(EnumSet.of(Case.Events.CURRENT_CASE, Case.Events.DATA_SOURCE_ADDED), this); this.em.addPropertyChangeListener(this); - IngestManager.getInstance().addIngestJobEventListener(this); - IngestManager.getInstance().addIngestModuleEventListener(this); } public void setDirectoryListingActive() { @@ -799,10 +796,7 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat } // change in node selection else if (changed.equals(ExplorerManager.PROP_SELECTED_NODES)) { respondSelection((Node[]) event.getOldValue(), (Node[]) event.getNewValue()); - } else if (changed.equals(IngestManager.IngestModuleEvent.DATA_ADDED.toString())) { - // nothing to do here. - // all nodes should be listening for these events and update accordingly. - } + } } } diff --git a/Core/src/org/sleuthkit/autopsy/imagewriter/ImageWriter.java b/Core/src/org/sleuthkit/autopsy/imagewriter/ImageWriter.java index 4200620749..29ecb29844 100644 --- a/Core/src/org/sleuthkit/autopsy/imagewriter/ImageWriter.java +++ b/Core/src/org/sleuthkit/autopsy/imagewriter/ImageWriter.java @@ -21,6 +21,8 @@ package org.sleuthkit.autopsy.imagewriter; import com.google.common.util.concurrent.ThreadFactoryBuilder; import java.beans.PropertyChangeEvent; import java.beans.PropertyChangeListener; +import java.util.EnumSet; +import java.util.Set; import java.util.concurrent.Callable; import java.util.concurrent.Executors; import java.util.concurrent.Future; @@ -44,18 +46,19 @@ import org.sleuthkit.datamodel.TskCoreException; /** * The ImageWriter class is used to complete VHD copies created from local disks - * after the ingest process completes. The AddImageTask for this data source must have included - * a non-empty imageWriterPath parameter to enable Image Writer. - * + * after the ingest process completes. The AddImageTask for this data source + * must have included a non-empty imageWriterPath parameter to enable Image + * Writer. + * * Most of the cancellation/cleanup is handled through ImageWriterService */ -class ImageWriter implements PropertyChangeListener{ - - private final Logger logger = Logger.getLogger(ImageWriter.class.getName()); - +class ImageWriter implements PropertyChangeListener { + + private static final Set INGEST_JOB_EVENTS_OF_INTEREST = EnumSet.of(IngestManager.IngestJobEvent.DATA_SOURCE_ANALYSIS_COMPLETED); + private static final Logger logger = Logger.getLogger(ImageWriter.class.getName()); private final Long dataSourceId; private final ImageWriterSettings settings; - + private Long imageHandle = null; private Future finishTask = null; private ProgressHandle progressHandle = null; @@ -63,63 +66,65 @@ class ImageWriter implements PropertyChangeListener{ private boolean isCancelled = false; private boolean isStarted = false; private final Object currentTasksLock = new Object(); // Get this lock before accessing imageHandle, finishTask, progressHandle, progressUpdateTask, - // isCancelled, isStarted, or isFinished - + // isCancelled, isStarted, or isFinished + private ScheduledThreadPoolExecutor periodicTasksExecutor = null; private final boolean doUI; private SleuthkitCase caseDb = null; - + /** - * Create the Image Writer object. - * After creation, startListeners() should be called. - * @param dataSourceId + * Create the Image Writer object. After creation, startListeners() should + * be called. + * + * @param dataSourceId */ - ImageWriter(Long dataSourceId, ImageWriterSettings settings){ - this.dataSourceId = dataSourceId; + ImageWriter(Long dataSourceId, ImageWriterSettings settings) { + this.dataSourceId = dataSourceId; this.settings = settings; - doUI = RuntimeProperties.runningWithGUI(); - + doUI = RuntimeProperties.runningWithGUI(); + // We save the reference to the sleuthkit case here in case getOpenCase() is set to // null before Image Writer finishes. The user can still elect to wait for image writer // (in ImageWriterService.closeCaseResources) even though the case is closing. - try{ + try { caseDb = Case.getCurrentCaseThrows().getSleuthkitCase(); - } catch (NoCurrentCaseException ex){ + } catch (NoCurrentCaseException ex) { logger.log(Level.SEVERE, "Unable to load case. Image writer will be cancelled."); this.isCancelled = true; } } - + /** * Add this ImageWriter object as a listener to the necessary events */ - void subscribeToEvents(){ - IngestManager.getInstance().addIngestJobEventListener(this); + void subscribeToEvents() { + IngestManager.getInstance().addIngestJobEventListener(INGEST_JOB_EVENTS_OF_INTEREST, this); } - + /** - * Deregister this object from the events. This is ok to call multiple times. + * Deregister this object from the events. This is ok to call multiple + * times. */ - void unsubscribeFromEvents(){ - IngestManager.getInstance().removeIngestJobEventListener(this); + void unsubscribeFromEvents() { + IngestManager.getInstance().removeIngestJobEventListener(this); } - + /** - * Handle the events: - * DATA_SOURCE_ANALYSIS_COMPLETED - start the finish image process and clean up after it is complete + * Handle the events: DATA_SOURCE_ANALYSIS_COMPLETED - start the finish + * image process and clean up after it is complete */ @Override public void propertyChange(PropertyChangeEvent evt) { - if(evt instanceof DataSourceAnalysisCompletedEvent){ - - DataSourceAnalysisCompletedEvent event = (DataSourceAnalysisCompletedEvent)evt; + if (evt instanceof DataSourceAnalysisCompletedEvent) { - if(event.getDataSource() != null){ + DataSourceAnalysisCompletedEvent event = (DataSourceAnalysisCompletedEvent) evt; + + if (event.getDataSource() != null) { long imageId = event.getDataSource().getId(); String name = event.getDataSource().getName(); - + // Check that the event corresponds to this datasource - if(imageId != dataSourceId){ + if (imageId != dataSourceId) { return; } new Thread(() -> { @@ -131,30 +136,30 @@ class ImageWriter implements PropertyChangeListener{ } } } - + @Messages({ - "# {0} - data source name", + "# {0} - data source name", "ImageWriter.progressBar.message=Finishing acquisition of {0} (unplug device to cancel)" }) - private void startFinishImage(String dataSourceName){ - - synchronized(currentTasksLock){ - if(isCancelled){ + private void startFinishImage(String dataSourceName) { + + synchronized (currentTasksLock) { + if (isCancelled) { return; } - + // If we've already started the finish process for this datasource, return. // Multiple DataSourceAnalysisCompletedEvent events can come from // the same image if more ingest modules are run later - if(isStarted){ + if (isStarted) { return; } - + Image image; - try{ + try { image = Case.getCurrentCaseThrows().getSleuthkitCase().getImageById(dataSourceId); imageHandle = image.getImageHandle(); - } catch (NoCurrentCaseException ex){ + } catch (NoCurrentCaseException ex) { // This exception means that getOpenCase() failed because no case was open. // This can happen when the user closes the case while ingest is ongoing - canceling // ingest fires off the DataSourceAnalysisCompletedEvent while the case is in the @@ -162,15 +167,15 @@ class ImageWriter implements PropertyChangeListener{ logger.log(Level.WARNING, String.format("Case closed before ImageWriter could start the finishing process for %s", dataSourceName)); return; - } catch (TskCoreException ex){ + } catch (TskCoreException ex) { logger.log(Level.SEVERE, "Error loading image", ex); return; } - logger.log(Level.INFO, String.format("Finishing VHD image for %s", + logger.log(Level.INFO, String.format("Finishing VHD image for %s", dataSourceName)); //NON-NLS - if(doUI){ + if (doUI) { periodicTasksExecutor = new ScheduledThreadPoolExecutor(1, new ThreadFactoryBuilder().setNameFormat("image-writer-progress-update-%d").build()); //NON-NLS progressHandle = ProgressHandle.createHandle(Bundle.ImageWriter_progressBar_message(dataSourceName)); progressHandle.start(100); @@ -181,138 +186,139 @@ class ImageWriter implements PropertyChangeListener{ // The added complexity here with the Future is because we absolutely need to make sure // the call to finishImageWriter returns before allowing the TSK data structures to be freed // during case close. - finishTask = Executors.newSingleThreadExecutor().submit(new Callable(){ + finishTask = Executors.newSingleThreadExecutor().submit(new Callable() { @Override - public Integer call() throws TskCoreException{ - try{ + public Integer call() throws TskCoreException { + try { int result = SleuthkitJNI.finishImageWriter(imageHandle); - + // We've decided to always update the path to the VHD, even if it wasn't finished. // This supports the case where an analyst has partially ingested a device // but has to stop before completion. They will at least have part of the image. - if(settings.getUpdateDatabasePath()){ + if (settings.getUpdateDatabasePath()) { caseDb.updateImagePath(settings.getPath(), dataSourceId); } return result; - } catch (TskCoreException ex){ + } catch (TskCoreException ex) { logger.log(Level.SEVERE, "Error finishing VHD image", ex); //NON-NLS return -1; } } }); - + // Setting this means that finishTask and all the UI updaters are initialized (if running UI) isStarted = true; } // Wait for finishImageWriter to complete int result = 0; - try{ + try { // The call to get() can happen multiple times if the user closes the case, which is ok result = finishTask.get(); - } catch (InterruptedException | ExecutionException ex){ + } catch (InterruptedException | ExecutionException ex) { logger.log(Level.SEVERE, "Error finishing VHD image", ex); //NON-NLS } - - synchronized(currentTasksLock){ - if(doUI){ + + synchronized (currentTasksLock) { + if (doUI) { // Some of these may be called twice if the user closes the case progressUpdateTask.cancel(true); progressHandle.finish(); periodicTasksExecutor.shutdown(); - } + } } - if(result == 0){ + if (result == 0) { logger.log(Level.INFO, String.format("Successfully finished writing VHD image for %s", dataSourceName)); //NON-NLS } else { logger.log(Level.INFO, String.format("Finished VHD image for %s with errors", dataSourceName)); //NON-NLS } } - + /** - * If a task hasn't been started yet, set the cancel flag so it can no longer - * start. - * This is intended to be used in case close so a job doesn't suddenly start - * up during cleanup. + * If a task hasn't been started yet, set the cancel flag so it can no + * longer start. This is intended to be used in case close so a job doesn't + * suddenly start up during cleanup. */ - void cancelIfNotStarted(){ - synchronized(currentTasksLock){ - if(! isStarted){ + void cancelIfNotStarted() { + synchronized (currentTasksLock) { + if (!isStarted) { isCancelled = true; } } } - + /** * Check if the finishTask process is running. - * @return true if the finish task is still going on, false if it is finished or - * never started + * + * @return true if the finish task is still going on, false if it is + * finished or never started */ - boolean jobIsInProgress(){ - synchronized(currentTasksLock){ - return((isStarted) && (! finishTask.isDone())); + boolean jobIsInProgress() { + synchronized (currentTasksLock) { + return ((isStarted) && (!finishTask.isDone())); } } - + /** - * Cancels a single job. - * Does not wait for the job to complete. Safe to call with Image Writer in any state. + * Cancels a single job. Does not wait for the job to complete. Safe to call + * with Image Writer in any state. */ - void cancelJob(){ - synchronized(currentTasksLock){ + void cancelJob() { + synchronized (currentTasksLock) { // All of the following is redundant but safe to call on a complete job isCancelled = true; - if(isStarted){ + if (isStarted) { SleuthkitJNI.cancelFinishImage(imageHandle); - + // Stop the progress bar update task. // The thread from startFinishImage will also stop it // once the task completes, but we don't have a guarantee on // when that happens. // Since we've stopped the update task, we'll stop the associated progress // bar now, too. - if(doUI){ + if (doUI) { progressUpdateTask.cancel(true); progressHandle.finish(); } - } + } } } - + /** - * Blocks while all finishImage tasks complete. - * Also makes sure the progressUpdateTask is canceled. + * Blocks while all finishImage tasks complete. Also makes sure the + * progressUpdateTask is canceled. */ - void waitForJobToFinish(){ - synchronized(currentTasksLock){ + void waitForJobToFinish() { + synchronized (currentTasksLock) { // Wait for the finish task to end - if(isStarted){ - try{ + if (isStarted) { + try { finishTask.get(); - } catch (InterruptedException | ExecutionException ex){ + } catch (InterruptedException | ExecutionException ex) { Logger.getLogger(ImageWriter.class.getName()).log(Level.SEVERE, "Error finishing VHD image", ex); //NON-NLS } - if(doUI){ + if (doUI) { progressUpdateTask.cancel(true); } - } + } } } - + /** - * Task to query the Sleuthkit processing to get the percentage done. + * Task to query the Sleuthkit processing to get the percentage done. */ private final class ProgressUpdateTask implements Runnable { + final long imageHandle; final ProgressHandle progressHandle; - - ProgressUpdateTask(ProgressHandle progressHandle, long imageHandle){ + + ProgressUpdateTask(ProgressHandle progressHandle, long imageHandle) { this.imageHandle = imageHandle; this.progressHandle = progressHandle; } - + @Override public void run() { try { diff --git a/Core/src/org/sleuthkit/autopsy/timeline/TimeLineController.java b/Core/src/org/sleuthkit/autopsy/timeline/TimeLineController.java index a230743a77..bdc5aa07a9 100644 --- a/Core/src/org/sleuthkit/autopsy/timeline/TimeLineController.java +++ b/Core/src/org/sleuthkit/autopsy/timeline/TimeLineController.java @@ -26,8 +26,10 @@ import java.time.ZoneId; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.EnumSet; import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.TimeZone; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; @@ -117,7 +119,8 @@ import org.sleuthkit.datamodel.BlackboardArtifact; public class TimeLineController { private static final Logger LOGGER = Logger.getLogger(TimeLineController.class.getName()); - + private static final Set INGEST_JOB_EVENTS_OF_INTEREST = EnumSet.of(IngestManager.IngestJobEvent.DATA_SOURCE_ANALYSIS_COMPLETED); + private static final Set INGEST_MODULE_EVENTS_OF_INTEREST = EnumSet.of(IngestManager.IngestModuleEvent.DATA_ADDED, IngestManager.IngestModuleEvent.CONTENT_CHANGED); private static final ReadOnlyObjectWrapper timeZone = new ReadOnlyObjectWrapper<>(TimeZone.getDefault()); public static ZoneId getTimeZoneID() { @@ -454,8 +457,8 @@ public class TimeLineController { TimeLineController.this.showFullRange(); } else { //prompt user to pick specific event and time range - ShowInTimelineDialog showInTimelineDilaog = - (file == null) + ShowInTimelineDialog showInTimelineDilaog + = (file == null) ? new ShowInTimelineDialog(TimeLineController.this, artifact) : new ShowInTimelineDialog(TimeLineController.this, file); Optional dialogResult = showInTimelineDilaog.showAndWait(); @@ -571,8 +574,8 @@ public class TimeLineController { void showTimeLine(AbstractFile file, BlackboardArtifact artifact) { // listen for case changes (specifically images being added, and case changes). if (Case.isCaseOpen() && !listeningToAutopsy) { - IngestManager.getInstance().addIngestModuleEventListener(ingestModuleListener); - IngestManager.getInstance().addIngestJobEventListener(ingestJobListener); + IngestManager.getInstance().addIngestModuleEventListener(INGEST_MODULE_EVENTS_OF_INTEREST, ingestModuleListener); + IngestManager.getInstance().addIngestJobEventListener(INGEST_JOB_EVENTS_OF_INTEREST, ingestJobListener); Case.addPropertyChangeListener(caseListener); listeningToAutopsy = true; } @@ -962,7 +965,7 @@ public class TimeLineController { //since black board artifacts or new derived content have been added, the DB is stale. Platform.runLater(() -> setEventsDBStale(true)); break; - case FILE_DONE: + default: /* * Do nothing, since we have captured all new results in * CONTENT_CHANGED and DATA_ADDED or the IngestJob listener, @@ -986,10 +989,7 @@ public class TimeLineController { Platform.runLater(() -> setEventsDBStale(true)); filteredEvents.postAutopsyEventLocally((AutopsyEvent) evt); break; - case DATA_SOURCE_ANALYSIS_STARTED: - case CANCELLED: - case COMPLETED: - case STARTED: + default: break; } } diff --git a/Core/test/qa-functional/src/org/sleuthkit/autopsy/testutils/IngestJobRunner.java b/Core/test/qa-functional/src/org/sleuthkit/autopsy/testutils/IngestJobRunner.java index 0abaebf48b..2f2440e2ce 100755 --- a/Core/test/qa-functional/src/org/sleuthkit/autopsy/testutils/IngestJobRunner.java +++ b/Core/test/qa-functional/src/org/sleuthkit/autopsy/testutils/IngestJobRunner.java @@ -22,7 +22,9 @@ import java.beans.PropertyChangeEvent; import java.beans.PropertyChangeListener; import java.util.Collection; import java.util.Collections; +import java.util.EnumSet; import java.util.List; +import java.util.Set; import org.sleuthkit.autopsy.events.AutopsyEvent; import org.sleuthkit.autopsy.ingest.IngestJobSettings; import org.sleuthkit.autopsy.ingest.IngestJobStartResult; @@ -35,6 +37,8 @@ import org.sleuthkit.datamodel.Content; */ public final class IngestJobRunner { + private static final Set INGEST_JOB_EVENTS_OF_INTEREST = EnumSet.of(IngestManager.IngestJobEvent.COMPLETED, IngestManager.IngestJobEvent.CANCELLED); + /** * Runs an ingest job, blocking until the job is completed. * @@ -51,7 +55,7 @@ public final class IngestJobRunner { Object ingestMonitor = new Object(); IngestJobCompletiontListener completiontListener = new IngestJobCompletiontListener(ingestMonitor); IngestManager ingestManager = IngestManager.getInstance(); - ingestManager.addIngestJobEventListener(completiontListener); + ingestManager.addIngestJobEventListener(INGEST_JOB_EVENTS_OF_INTEREST, completiontListener); try { synchronized (ingestMonitor) { IngestJobStartResult jobStartResult = ingestManager.beginIngestJob(dataSources, settings); @@ -111,5 +115,5 @@ public final class IngestJobRunner { } } } - + } diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java index 4bade61c1c..48b86025af 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java @@ -142,6 +142,7 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen ControlEventType.SHUTDOWN.toString(), Event.CANCEL_JOB.toString(), Event.REPROCESS_JOB.toString()})); + private static final Set INGEST_JOB_EVENTS_OF_INTEREST = EnumSet.of(IngestManager.IngestJobEvent.COMPLETED, IngestManager.IngestJobEvent.CANCELLED); private static final long JOB_STATUS_EVENT_INTERVAL_SECONDS = 10; private static final String JOB_STATUS_PUBLISHING_THREAD_NAME = "AIM-job-status-event-publisher-%d"; private static final long MAX_MISSED_JOB_STATUS_UPDATES = 10; @@ -2670,7 +2671,7 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen Path caseDirectoryPath = currentJob.getCaseDirectoryPath(); AutoIngestJobLogger jobLogger = new AutoIngestJobLogger(manifestPath, manifest.getDataSourceFileName(), caseDirectoryPath); IngestJobEventListener ingestJobEventListener = new IngestJobEventListener(); - IngestManager.getInstance().addIngestJobEventListener(ingestJobEventListener); + IngestManager.getInstance().addIngestJobEventListener(INGEST_JOB_EVENTS_OF_INTEREST, ingestJobEventListener); try { synchronized (ingestLock) { IngestJobSettings ingestJobSettings = new IngestJobSettings(AutoIngestUserPreferences.getAutoModeIngestModuleContextString()); diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/configuration/MultiUserTestTool.java b/Experimental/src/org/sleuthkit/autopsy/experimental/configuration/MultiUserTestTool.java index f37c7cefa0..8504bfd7c0 100755 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/configuration/MultiUserTestTool.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/configuration/MultiUserTestTool.java @@ -26,7 +26,9 @@ import java.nio.charset.Charset; import java.nio.file.Paths; import java.sql.ResultSet; import java.sql.SQLException; +import java.util.EnumSet; import java.util.List; +import java.util.Set; import java.util.UUID; import java.util.logging.Level; import org.apache.commons.io.FileUtils; @@ -68,8 +70,9 @@ class MultiUserTestTool { private static final Logger LOGGER = Logger.getLogger(MultiUserTestTool.class.getName()); private static final String TEST_FILE_NAME = "AutopsyTempFile"; private static final Object INGEST_LOCK = new Object(); + private static final Set INGEST_JOB_EVENTS_OF_INTEREST = EnumSet.of(IngestManager.IngestJobEvent.COMPLETED, IngestManager.IngestJobEvent.CANCELLED); static final String MULTI_USER_TEST_SUCCESSFUL = NbBundle.getMessage(AutoIngestSettingsPanel.class, "AutoIngestSettingsPanel.Success"); - + private MultiUserTestTool() { } @@ -86,17 +89,17 @@ class MultiUserTestTool { "# {0} - serviceName", "MultiUserTestTool.serviceDown=Multi User service is down: {0}", "# {0} - serviceName", - "MultiUserTestTool.unableToCheckService=Unable to check Multi User service state: {0}" + "MultiUserTestTool.unableToCheckService=Unable to check Multi User service state: {0}" }) static String runTest(String rootOutputDirectory) { - + // run standard tests for all services. this detects many problems sooner. try { if (!isServiceUp(ServicesMonitor.Service.REMOTE_CASE_DATABASE.toString())) { return NbBundle.getMessage(MultiUserTestTool.class, "MultiUserTestTool.serviceDown", ServicesMonitor.Service.REMOTE_CASE_DATABASE.getDisplayName()); } } catch (ServicesMonitor.ServicesMonitorException ex) { - return NbBundle.getMessage(MultiUserTestTool.class, "MultiUserTestTool.unableToCheckService", + return NbBundle.getMessage(MultiUserTestTool.class, "MultiUserTestTool.unableToCheckService", ServicesMonitor.Service.REMOTE_CASE_DATABASE.getDisplayName() + ". " + ex.getMessage()); } @@ -105,7 +108,7 @@ class MultiUserTestTool { return NbBundle.getMessage(MultiUserTestTool.class, "MultiUserTestTool.serviceDown", ServicesMonitor.Service.REMOTE_KEYWORD_SEARCH.getDisplayName()); } } catch (ServicesMonitor.ServicesMonitorException ex) { - return NbBundle.getMessage(MultiUserTestTool.class, "MultiUserTestTool.unableToCheckService", + return NbBundle.getMessage(MultiUserTestTool.class, "MultiUserTestTool.unableToCheckService", ServicesMonitor.Service.REMOTE_KEYWORD_SEARCH.getDisplayName() + ". " + ex.getMessage()); } @@ -114,7 +117,7 @@ class MultiUserTestTool { return NbBundle.getMessage(MultiUserTestTool.class, "MultiUserTestTool.serviceDown", ServicesMonitor.Service.MESSAGING.getDisplayName()); } } catch (ServicesMonitor.ServicesMonitorException ex) { - return NbBundle.getMessage(MultiUserTestTool.class, "MultiUserTestTool.unableToCheckService", + return NbBundle.getMessage(MultiUserTestTool.class, "MultiUserTestTool.unableToCheckService", ServicesMonitor.Service.MESSAGING.getDisplayName() + ". " + ex.getMessage()); } @@ -225,10 +228,12 @@ class MultiUserTestTool { /** * Creates a new multi user case. * - * @param baseCaseName Case name (will get time stamp appended to it) + * @param baseCaseName Case name (will get time stamp appended to it) * @param rootOutputDirectory Full path to directory in which the case will - * be created + * be created + * * @return Case object + * * @throws CaseActionException */ private static Case createCase(String baseCaseName, String rootOutputDirectory) throws CaseActionException { @@ -251,16 +256,17 @@ class MultiUserTestTool { * @param dataSource The data source. * * @return Error String if there was an error, empty string if the data - * source was added successfully + * source was added successfully * * @throws InterruptedException if the thread running the job processing - * task is interrupted while blocked, i.e., if ingest is shutting down. + * task is interrupted while blocked, i.e., if + * ingest is shutting down. */ @NbBundle.Messages({ "MultiUserTestTool.noContent=Test data source failed to produce content", "# {0} - errorMessage", "MultiUserTestTool.criticalError=Critical error running data source processor on test data source: {0}" - }) + }) private static String runLogicalFilesDSP(Case caseForJob, AutoIngestDataSource dataSource) throws InterruptedException { AutoIngestDataSourceProcessor selectedProcessor = new LocalFilesDSProcessor(); @@ -298,23 +304,24 @@ class MultiUserTestTool { * @param dataSource The data source to analyze. * * @return Error String if there was an error, empty string if the data - * source was analyzed successfully + * source was analyzed successfully * * @throws InterruptedException if the thread running the job processing - * task is interrupted while blocked, i.e., if auto ingest is shutting down. + * task is interrupted while blocked, i.e., if + * auto ingest is shutting down. */ @NbBundle.Messages({ "# {0} - cancellationReason", "MultiUserTestTool.ingestCancelled=Ingest cancelled due to {0}", "MultiUserTestTool.startupError=Failed to analyze data source due to ingest job startup error", "MultiUserTestTool.errorStartingIngestJob=Ingest manager error while starting ingest job", - "MultiUserTestTool.ingestSettingsError=Failed to analyze data source due to ingest settings errors" + "MultiUserTestTool.ingestSettingsError=Failed to analyze data source due to ingest settings errors" }) private static String analyze(AutoIngestDataSource dataSource) throws InterruptedException { LOGGER.log(Level.INFO, "Starting ingest modules analysis for {0} ", dataSource.getPath()); IngestJobEventListener ingestJobEventListener = new IngestJobEventListener(); - IngestManager.getInstance().addIngestJobEventListener(ingestJobEventListener); + IngestManager.getInstance().addIngestJobEventListener(INGEST_JOB_EVENTS_OF_INTEREST, ingestJobEventListener); try { synchronized (INGEST_LOCK) { IngestJobSettings ingestJobSettings = new IngestJobSettings(AutoIngestUserPreferences.getAutoModeIngestModuleContextString()); @@ -324,9 +331,9 @@ class MultiUserTestTool { IngestJob ingestJob = ingestJobStartResult.getJob(); if (null != ingestJob) { /* - * Block until notified by the ingest job event - * listener or until interrupted because auto ingest - * is shutting down. + * Block until notified by the ingest job event listener + * or until interrupted because auto ingest is shutting + * down. */ INGEST_LOCK.wait(); LOGGER.log(Level.INFO, "Finished ingest modules analysis for {0} ", dataSource.getPath()); @@ -381,7 +388,7 @@ class MultiUserTestTool { * @return True if the service is running, false otherwise. * * @throws ServicesMonitorException if there is an error querying the - * services monitor. + * services monitor. */ private static boolean isServiceUp(String serviceName) throws ServicesMonitor.ServicesMonitorException { return (ServicesMonitor.getInstance().getServiceStatus(serviceName).equals(ServicesMonitor.ServiceStatus.UP.toString())); diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java index fd365877dd..624c914d56 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java @@ -89,7 +89,8 @@ import org.sleuthkit.datamodel.TskData; public final class ImageGalleryController { private static final Logger logger = Logger.getLogger(ImageGalleryController.class.getName()); - + private static final Set INGEST_JOB_EVENTS_OF_INTEREST = EnumSet.of(IngestManager.IngestJobEvent.DATA_SOURCE_ANALYSIS_STARTED, IngestManager.IngestJobEvent.DATA_SOURCE_ANALYSIS_COMPLETED); + private static final Set INGEST_MODULE_EVENTS_OF_INTEREST = EnumSet.of(IngestManager.IngestModuleEvent.DATA_ADDED, IngestManager.IngestModuleEvent.FILE_DONE); /* * The file limit for image gallery. If the selected data source (or all * data sources, if that option is selected) has more than this many files @@ -267,8 +268,8 @@ public final class ImageGalleryController { dbTaskQueueSize.addListener(obs -> this.updateRegroupDisabled()); Case.addEventTypeSubscriber(CASE_EVENTS_OF_INTEREST, caseEventListener); - IngestManager.getInstance().addIngestJobEventListener(ingestJobEventListener); - IngestManager.getInstance().addIngestModuleEventListener(ingestModuleEventListener); + IngestManager.getInstance().addIngestJobEventListener(INGEST_JOB_EVENTS_OF_INTEREST, ingestJobEventListener); + IngestManager.getInstance().addIngestModuleEventListener(INGEST_MODULE_EVENTS_OF_INTEREST, ingestModuleEventListener); SwingUtilities.invokeLater(() -> { topComponent = ImageGalleryTopComponent.getTopComponent(); diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/DropdownListSearchPanel.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/DropdownListSearchPanel.java index 089e8a0031..f5ef3f7b91 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/DropdownListSearchPanel.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/DropdownListSearchPanel.java @@ -52,6 +52,7 @@ import org.sleuthkit.autopsy.ingest.IngestManager; class DropdownListSearchPanel extends AdHocSearchPanel { private static final Logger logger = Logger.getLogger(DropdownListSearchPanel.class.getName()); + private static DropdownListSearchPanel instance; private XmlKeywordSearchList loader; private final KeywordListsTableModel listsTableModel; From bcdd3156582c7b27dd19a996f14ae5c9ba2f2399 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Fri, 19 Jul 2019 15:10:40 -0400 Subject: [PATCH 005/102] 5319 clean up and update copyrights --- .../autopsy/casemodule/CollaborationMonitor.java | 7 +++---- .../eventlisteners/IngestEventsListener.java | 3 ++- .../optionspanel/GlobalSettingsPanel.java | 2 +- .../commandlineingest/CommandLineIngestManager.java | 1 + .../sleuthkit/autopsy/communications/FiltersPanel.java | 6 ++++-- .../org/sleuthkit/autopsy/datamodel/EmailExtracted.java | 1 - .../sleuthkit/autopsy/datamodel/ExtractedContent.java | 9 ++++----- Core/src/org/sleuthkit/autopsy/datamodel/FileSize.java | 4 ++-- .../autopsy/datamodel/FileTypesByExtension.java | 8 ++++---- .../sleuthkit/autopsy/datamodel/FileTypesByMimeType.java | 2 +- .../src/org/sleuthkit/autopsy/datamodel/HashsetHits.java | 9 ++++----- Core/src/org/sleuthkit/autopsy/datamodel/ImageNode.java | 2 +- Core/src/org/sleuthkit/autopsy/datamodel/Tags.java | 2 +- Core/src/org/sleuthkit/autopsy/datamodel/VolumeNode.java | 2 +- .../sleuthkit/autopsy/datamodel/accounts/Accounts.java | 7 ++++--- .../org/sleuthkit/autopsy/imagewriter/ImageWriter.java | 2 +- Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java | 3 +-- .../sleuthkit/autopsy/timeline/TimeLineController.java | 5 ++--- .../org/sleuthkit/autopsy/testutils/IngestJobRunner.java | 2 +- 19 files changed, 38 insertions(+), 39 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/CollaborationMonitor.java b/Core/src/org/sleuthkit/autopsy/casemodule/CollaborationMonitor.java index 89778bb86e..e6a7772c73 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/CollaborationMonitor.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/CollaborationMonitor.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2011-2017 Basis Technology Corp. + * Copyright 2011-2019 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -24,11 +24,10 @@ import java.beans.PropertyChangeListener; import java.io.Serializable; import java.time.Duration; import java.time.Instant; -import java.util.Arrays; +import java.util.Collections; import java.util.EnumSet; import java.util.HashMap; import java.util.Iterator; -import java.util.List; import java.util.Map; import java.util.Set; import java.util.UUID; @@ -541,7 +540,7 @@ final class CollaborationMonitor { * @return A mapping of task IDs to current tasks */ Map getCurrentTasks() { - return currentTasks; + return Collections.unmodifiableMap(currentTasks); } } diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java index 6ec535a999..f2942dd3eb 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java @@ -55,6 +55,7 @@ import org.sleuthkit.autopsy.centralrepository.datamodel.EamDb; import org.sleuthkit.autopsy.coreutils.ThreadUtils; import static org.sleuthkit.autopsy.ingest.IngestManager.IngestModuleEvent.DATA_ADDED; import org.sleuthkit.autopsy.ingest.events.DataSourceAnalysisCompletedEvent; +import org.sleuthkit.autopsy.ingest.events.DataSourceAnalysisEvent; import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.Image; import org.sleuthkit.datamodel.SleuthkitCase; @@ -365,7 +366,7 @@ public class IngestEventsListener { String dataSourceName = ""; long dataSourceObjectId = -1; try { - dataSource = ((DataSourceAnalysisCompletedEvent) event).getDataSource(); + dataSource = ((DataSourceAnalysisEvent) event).getDataSource(); /* * We only care about Images for the purpose of updating hash diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/GlobalSettingsPanel.java b/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/GlobalSettingsPanel.java index 6236d3b8bd..c388d081a9 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/GlobalSettingsPanel.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/GlobalSettingsPanel.java @@ -1,7 +1,7 @@ /* * Central Repository * - * Copyright 2015-2018 Basis Technology Corp. + * Copyright 2015-2019 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineIngestManager.java b/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineIngestManager.java index 9cc726c32e..31de1cfdfe 100755 --- a/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineIngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineIngestManager.java @@ -103,6 +103,7 @@ public class CommandLineIngestManager { } } + @Override public void run() { LOGGER.log(Level.INFO, "Job processing task started"); diff --git a/Core/src/org/sleuthkit/autopsy/communications/FiltersPanel.java b/Core/src/org/sleuthkit/autopsy/communications/FiltersPanel.java index 15e1d29b3c..a804735c32 100644 --- a/Core/src/org/sleuthkit/autopsy/communications/FiltersPanel.java +++ b/Core/src/org/sleuthkit/autopsy/communications/FiltersPanel.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2017-2018 Basis Technology Corp. + * Copyright 2017-2019 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -833,7 +833,7 @@ final public class FiltersPanel extends JPanel { * * @return an instance of CommunicationsFilter */ - protected CommunicationsFilter getFilter() { + private CommunicationsFilter getFilter() { CommunicationsFilter commsFilter = new CommunicationsFilter(); commsFilter.addAndFilter(getDeviceFilter()); commsFilter.addAndFilter(getAccountTypeFilter()); @@ -1089,6 +1089,8 @@ final public class FiltersPanel extends JPanel { */ final class CheckBoxIconPanel extends JPanel { + private static final long serialVersionUID = 1L; + private final JCheckBox checkbox; private final JLabel label; diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/EmailExtracted.java b/Core/src/org/sleuthkit/autopsy/datamodel/EmailExtracted.java index a6652679ec..545cace37e 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/EmailExtracted.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/EmailExtracted.java @@ -42,7 +42,6 @@ import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.ingest.IngestManager; -import static org.sleuthkit.autopsy.ingest.IngestManager.IngestModuleEvent.CONTENT_CHANGED; import org.sleuthkit.autopsy.ingest.ModuleDataEvent; import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardAttribute; diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/ExtractedContent.java b/Core/src/org/sleuthkit/autopsy/datamodel/ExtractedContent.java index 4ac4413e47..cefd59a041 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/ExtractedContent.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/ExtractedContent.java @@ -52,7 +52,6 @@ import static org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE.TSK_KEYWO import static org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE.TSK_DOWNLOAD_SOURCE; import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.TskCoreException; -import org.sleuthkit.datamodel.TskException; /** * Parent of the "extracted content" artifacts to be displayed in the tree. @@ -226,7 +225,7 @@ public class ExtractedContent implements AutopsyVisitableItem { // maps the artifact type to its child node private final HashMap typeNodeList = new HashMap<>(); - public TypeFactory() { + TypeFactory() { super(); // these are shown in other parts of the UI tree @@ -376,7 +375,7 @@ public class ExtractedContent implements AutopsyVisitableItem { this.childCount = (filteringDSObjId > 0) ? blackboard.getArtifactsCount(type.getTypeID(), filteringDSObjId) : skCase.getBlackboardArtifactsTypeCount(type.getTypeID()); - } catch (TskException ex) { + } catch (TskCoreException ex) { Logger.getLogger(TypeNode.class.getName()) .log(Level.WARNING, "Error getting child count", ex); //NON-NLS } @@ -428,7 +427,7 @@ public class ExtractedContent implements AutopsyVisitableItem { private BlackboardArtifact.Type type; - public ArtifactFactory(BlackboardArtifact.Type type) { + ArtifactFactory(BlackboardArtifact.Type type) { super(type.getTypeName()); this.type = type; } @@ -505,7 +504,7 @@ public class ExtractedContent implements AutopsyVisitableItem { return (filteringDSObjId > 0) ? blackboard.getArtifacts(type.getTypeID(), filteringDSObjId) : skCase.getBlackboardArtifacts(type.getTypeID()); - } catch (TskException ex) { + } catch (TskCoreException ex) { Logger.getLogger(ArtifactFactory.class.getName()).log(Level.SEVERE, "Couldn't get blackboard artifacts from database", ex); //NON-NLS } } diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/FileSize.java b/Core/src/org/sleuthkit/autopsy/datamodel/FileSize.java index 172c11c037..a29a853320 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/FileSize.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/FileSize.java @@ -269,7 +269,7 @@ public class FileSize implements AutopsyVisitableItem { */ public class FileSizeNode extends DisplayableItemNode { - private FileSizeFilter filter; + private final FileSizeFilter filter; private final long datasourceObjId; // use version with observer instead so that it updates @@ -364,11 +364,11 @@ public class FileSize implements AutopsyVisitableItem { */ static class FileSizeChildren extends BaseChildFactory { + private static final Logger logger = Logger.getLogger(FileSizeChildren.class.getName()); private final SleuthkitCase skCase; private final FileSizeFilter filter; private final Observable notifier; private final long datasourceObjId; - private static final Logger logger = Logger.getLogger(FileSizeChildren.class.getName()); /** * diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/FileTypesByExtension.java b/Core/src/org/sleuthkit/autopsy/datamodel/FileTypesByExtension.java index 19dac06410..344a24cb79 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/FileTypesByExtension.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/FileTypesByExtension.java @@ -281,7 +281,7 @@ public final class FileTypesByExtension implements AutopsyVisitableItem { * Node for a specific file type / extension. Children of it will be the * files of that type. */ - class FileExtensionNode extends FileTypes.BGCountUpdatingNode { + final class FileExtensionNode extends FileTypes.BGCountUpdatingNode { private final FileTypesByExtension.SearchFilterInterface filter; @@ -495,7 +495,7 @@ public final class FileTypesByExtension implements AutopsyVisitableItem { @Override public List getFilter() { - return this.filter; + return Collections.unmodifiableList(this.filter); } } @@ -552,7 +552,7 @@ public final class FileTypesByExtension implements AutopsyVisitableItem { @Override public List getFilter() { - return this.filter; + return Collections.unmodifiableList(this.filter); } } @@ -599,7 +599,7 @@ public final class FileTypesByExtension implements AutopsyVisitableItem { @Override public List getFilter() { - return this.filter; + return Collections.unmodifiableList(this.filter); } } diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/FileTypesByMimeType.java b/Core/src/org/sleuthkit/autopsy/datamodel/FileTypesByMimeType.java index 0ac8f8e8b0..f1baac477a 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/FileTypesByMimeType.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/FileTypesByMimeType.java @@ -370,7 +370,7 @@ public final class FileTypesByMimeType extends Observable implements AutopsyVisi * Node which represents the media sub type in the By MIME type tree, the * media subtype is the portion of the MIME type following the /. */ - class MediaSubTypeNode extends FileTypes.BGCountUpdatingNode { + final class MediaSubTypeNode extends FileTypes.BGCountUpdatingNode { @NbBundle.Messages({"FileTypesByMimeTypeNode.createSheet.mediaSubtype.name=Subtype", "FileTypesByMimeTypeNode.createSheet.mediaSubtype.displayName=Subtype", diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/HashsetHits.java b/Core/src/org/sleuthkit/autopsy/datamodel/HashsetHits.java index 3a9a87c1e4..724b3563d0 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/HashsetHits.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/HashsetHits.java @@ -51,7 +51,6 @@ import org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE; import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.SleuthkitCase.CaseDbQuery; import org.sleuthkit.datamodel.TskCoreException; -import org.sleuthkit.datamodel.TskException; /** * Hash set hits node support. Inner classes have all of the nodes in the tree. @@ -152,7 +151,7 @@ public class HashsetHits implements AutopsyVisitableItem { String setName = resultSet.getString("value_text"); //NON-NLS long artifactId = resultSet.getLong("artifact_id"); //NON-NLS if (!hashSetHitsMap.containsKey(setName)) { - hashSetHitsMap.put(setName, new HashSet()); + hashSetHitsMap.put(setName, new HashSet<>()); } hashSetHitsMap.get(setName).add(artifactId); } @@ -378,8 +377,8 @@ public class HashsetHits implements AutopsyVisitableItem { */ private class HitFactory extends BaseChildFactory implements Observer { - private String hashsetName; - private Map artifactHits = new HashMap<>(); + private final String hashsetName; + private final Map artifactHits = new HashMap<>(); private HitFactory(String hashsetName) { super(hashsetName); @@ -416,7 +415,7 @@ public class HashsetHits implements AutopsyVisitableItem { BlackboardArtifact art = skCase.getBlackboardArtifact(id); artifactHits.put(id, art); } - } catch (TskException ex) { + } catch (TskCoreException ex) { logger.log(Level.SEVERE, "TSK Exception occurred", ex); //NON-NLS } }); diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/ImageNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/ImageNode.java index 12f2340f33..b81fd6760a 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/ImageNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/ImageNode.java @@ -119,7 +119,7 @@ public class ImageNode extends AbstractContentNode { actionsList.add(new RunIngestModulesAction(Collections.singletonList(content))); actionsList.add(new NewWindowViewAction( NbBundle.getMessage(this.getClass(), "ImageNode.getActions.viewInNewWin.text"), this)); - return actionsList.toArray(new Action[0]); + return actionsList.toArray(new Action[actionsList.size()]); } @Override diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/Tags.java b/Core/src/org/sleuthkit/autopsy/datamodel/Tags.java index 8dd7805358..aa47872c82 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/Tags.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/Tags.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2011-2018 Basis Technology Corp. + * Copyright 2011-2019 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/VolumeNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/VolumeNode.java index 0eacf5f699..541a842ea7 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/VolumeNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/VolumeNode.java @@ -161,7 +161,7 @@ public class VolumeNode extends AbstractContentNode { NbBundle.getMessage(this.getClass(), "VolumeNode.getActions.viewInNewWin.text"), this)); actionsList.addAll(ExplorerNodeActionVisitor.getActions(content)); - return actionsList.toArray(new Action[0]); + return actionsList.toArray(new Action[actionsList.size()]); } @Override diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/accounts/Accounts.java b/Core/src/org/sleuthkit/autopsy/datamodel/accounts/Accounts.java index 0e3a4e971d..b703867a89 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/accounts/Accounts.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/accounts/Accounts.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2011-2018 Basis Technology Corp. + * Copyright 2011-2019 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -193,6 +193,7 @@ final public class Accounts implements AutopsyVisitableItem { * Create of keys used by this Children object to represent the child * nodes. */ + @Override abstract protected boolean createKeys(List list); /** @@ -1174,7 +1175,7 @@ final public class Accounts implements AutopsyVisitableItem { * @return the artifact ids of the account artifacts from this file. */ public List getArtifactIDs() { - return artifactIDs; + return Collections.unmodifiableList(artifactIDs); } /** @@ -1192,7 +1193,7 @@ final public class Accounts implements AutopsyVisitableItem { * @return the status(s) of the account artifacts from this file. */ public Set getStatuses() { - return statuses; + return Collections.unmodifiableSet(statuses); } } diff --git a/Core/src/org/sleuthkit/autopsy/imagewriter/ImageWriter.java b/Core/src/org/sleuthkit/autopsy/imagewriter/ImageWriter.java index 29ecb29844..3e44c4718a 100644 --- a/Core/src/org/sleuthkit/autopsy/imagewriter/ImageWriter.java +++ b/Core/src/org/sleuthkit/autopsy/imagewriter/ImageWriter.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2011-2018 Basis Technology Corp. + * Copyright 2011-2019 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java index e345db3d65..36aef30a9c 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java @@ -367,7 +367,6 @@ public class IngestManager implements IngestProgressSnapshotProvider { "IngestManager.startupErr.dlgErrorList=Errors:" }) private IngestJobStartResult startIngestJob(IngestJob job) { - List errors = null; Case openCase; try { openCase = Case.getCurrentCaseThrows(); @@ -404,7 +403,7 @@ public class IngestManager implements IngestProgressSnapshotProvider { ingestJobsById.put(job.getId(), job); } IngestManager.logger.log(Level.INFO, "Starting ingest job {0}", job.getId()); //NON-NLS - errors = job.start(); + List errors = job.start(); if (errors.isEmpty()) { this.fireIngestJobStarted(job.getId()); } else { diff --git a/Core/src/org/sleuthkit/autopsy/timeline/TimeLineController.java b/Core/src/org/sleuthkit/autopsy/timeline/TimeLineController.java index bdc5aa07a9..7aa77f2c1b 100644 --- a/Core/src/org/sleuthkit/autopsy/timeline/TimeLineController.java +++ b/Core/src/org/sleuthkit/autopsy/timeline/TimeLineController.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2014-2018 Basis Technology Corp. + * Copyright 2014-2019 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -81,7 +81,6 @@ import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; import org.sleuthkit.autopsy.coreutils.ThreadConfined; import org.sleuthkit.autopsy.events.AutopsyEvent; import org.sleuthkit.autopsy.ingest.IngestManager; -import static org.sleuthkit.autopsy.ingest.IngestManager.IngestJobEvent.CANCELLED; import org.sleuthkit.autopsy.timeline.datamodel.FilteredEventsModel; import org.sleuthkit.autopsy.timeline.datamodel.TimeLineEvent; import org.sleuthkit.autopsy.timeline.datamodel.eventtype.EventType; @@ -150,7 +149,7 @@ public class TimeLineController { private final ReadOnlyStringWrapper taskTitle = new ReadOnlyStringWrapper(); private final ReadOnlyStringWrapper statusMessage = new ReadOnlyStringWrapper(); - private EventBus eventbus = new EventBus("TimeLineController_EventBus"); + private final EventBus eventbus = new EventBus("TimeLineController_EventBus"); /** * Status is a string that will be displayed in the status bar as a kind of diff --git a/Core/test/qa-functional/src/org/sleuthkit/autopsy/testutils/IngestJobRunner.java b/Core/test/qa-functional/src/org/sleuthkit/autopsy/testutils/IngestJobRunner.java index 2f2440e2ce..c424e057c2 100755 --- a/Core/test/qa-functional/src/org/sleuthkit/autopsy/testutils/IngestJobRunner.java +++ b/Core/test/qa-functional/src/org/sleuthkit/autopsy/testutils/IngestJobRunner.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2018 Basis Technology Corp. + * Copyright 2018-2019 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); From a4bfdb773c3765bb944f324051355e04436753c5 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Fri, 19 Jul 2019 15:36:35 -0400 Subject: [PATCH 006/102] 5319 undo unintentially commited changes --- .../CommandLineIngestManager.java | 37 ++++--------------- .../autopsy/ingest/IngestManager.java | 3 +- .../DropdownListSearchPanel.java | 1 - 3 files changed, 9 insertions(+), 32 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineIngestManager.java b/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineIngestManager.java index 31de1cfdfe..29faa16356 100755 --- a/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineIngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineIngestManager.java @@ -275,32 +275,10 @@ public class CommandLineIngestManager { * * @throws * AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException if - * there - * was - * a - * DSP - * processing - * error + * there was a DSP processing error * - * @throws InterruptedException if - * the - * thread - * running - * the - * job - * processing - * task - * is - * interrupted - * while - * blocked, - * i.e., - * if - * auto - * ingest - * is - * shutting - * down. + * @throws InterruptedException if the thread running the job processing + * task is interrupted while blocked, i.e., if auto ingest is shutting down. */ private void runDataSourceProcessor(Case caseForJob, AutoIngestDataSource dataSource) throws InterruptedException, AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException { @@ -402,11 +380,10 @@ public class CommandLineIngestManager { * @param dataSource The data source to analyze. * * @throws AnalysisStartupException if there is an error analyzing the - * data source. - * @throws InterruptedException if the thread running the job - * processing task is interrupted while - * blocked, i.e., if auto ingest is - * shutting down. + * data source. + * @throws InterruptedException if the thread running the job processing + * task is interrupted while blocked, i.e., if auto ingest is shutting + * down. */ private void analyze(AutoIngestDataSource dataSource) throws AnalysisStartupException, InterruptedException { diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java index 36aef30a9c..e345db3d65 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java @@ -367,6 +367,7 @@ public class IngestManager implements IngestProgressSnapshotProvider { "IngestManager.startupErr.dlgErrorList=Errors:" }) private IngestJobStartResult startIngestJob(IngestJob job) { + List errors = null; Case openCase; try { openCase = Case.getCurrentCaseThrows(); @@ -403,7 +404,7 @@ public class IngestManager implements IngestProgressSnapshotProvider { ingestJobsById.put(job.getId(), job); } IngestManager.logger.log(Level.INFO, "Starting ingest job {0}", job.getId()); //NON-NLS - List errors = job.start(); + errors = job.start(); if (errors.isEmpty()) { this.fireIngestJobStarted(job.getId()); } else { diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/DropdownListSearchPanel.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/DropdownListSearchPanel.java index f5ef3f7b91..089e8a0031 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/DropdownListSearchPanel.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/DropdownListSearchPanel.java @@ -52,7 +52,6 @@ import org.sleuthkit.autopsy.ingest.IngestManager; class DropdownListSearchPanel extends AdHocSearchPanel { private static final Logger logger = Logger.getLogger(DropdownListSearchPanel.class.getName()); - private static DropdownListSearchPanel instance; private XmlKeywordSearchList loader; private final KeywordListsTableModel listsTableModel; From b5421a43e7a3870a2428d8812809ebeb0dfea658 Mon Sep 17 00:00:00 2001 From: Mark McKinnon Date: Tue, 30 Jul 2019 13:16:16 -0400 Subject: [PATCH 007/102] Changed logger class to use Autopsy version Changed logger class to use Autopsy version. --- .../datasourcesummary/DataSourceLabeledValueCallback.java | 2 +- .../datasourcesummary/DataSourceSingleValueCallback.java | 2 +- .../CommonAttributeSearchResultRootNode.java | 2 +- Core/src/org/sleuthkit/autopsy/core/AutopsyOptionProcessor.java | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/datasourcesummary/DataSourceLabeledValueCallback.java b/Core/src/org/sleuthkit/autopsy/casemodule/datasourcesummary/DataSourceLabeledValueCallback.java index 3edd92c8b0..1a44e2a76b 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/datasourcesummary/DataSourceLabeledValueCallback.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/datasourcesummary/DataSourceLabeledValueCallback.java @@ -24,7 +24,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.logging.Level; -import java.util.logging.Logger; +import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.datamodel.CaseDbAccessManager; /** diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/datasourcesummary/DataSourceSingleValueCallback.java b/Core/src/org/sleuthkit/autopsy/casemodule/datasourcesummary/DataSourceSingleValueCallback.java index 241b74c87e..fc7eea15e5 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/datasourcesummary/DataSourceSingleValueCallback.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/datasourcesummary/DataSourceSingleValueCallback.java @@ -24,7 +24,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.logging.Level; -import java.util.logging.Logger; +import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.datamodel.CaseDbAccessManager; /** diff --git a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeSearchResultRootNode.java b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeSearchResultRootNode.java index 27156f97fa..2c1ad550cb 100644 --- a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeSearchResultRootNode.java +++ b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributeSearchResultRootNode.java @@ -20,7 +20,7 @@ package org.sleuthkit.autopsy.commonpropertiessearch; import java.util.List; import java.util.Map; -import java.util.logging.Logger; +import org.sleuthkit.autopsy.coreutils.Logger; import org.openide.nodes.ChildFactory; import org.openide.nodes.Children; import org.openide.nodes.Node; diff --git a/Core/src/org/sleuthkit/autopsy/core/AutopsyOptionProcessor.java b/Core/src/org/sleuthkit/autopsy/core/AutopsyOptionProcessor.java index dd4ba98ae6..e4cab76761 100644 --- a/Core/src/org/sleuthkit/autopsy/core/AutopsyOptionProcessor.java +++ b/Core/src/org/sleuthkit/autopsy/core/AutopsyOptionProcessor.java @@ -22,7 +22,7 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; import java.util.logging.Level; -import java.util.logging.Logger; +import org.sleuthkit.autopsy.coreutils.Logger; import org.netbeans.api.sendopts.CommandException; import org.netbeans.spi.sendopts.Env; import org.netbeans.spi.sendopts.Option; From 83b9b56914aec316da85774ceda067316e09c6c1 Mon Sep 17 00:00:00 2001 From: Mark McKinnon Date: Tue, 30 Jul 2019 14:50:52 -0400 Subject: [PATCH 008/102] Change logger from java to Autopsy Change the logger used from java to Autopsy --- .../autopsy/casemodule/datasourcesummary/DataSourceSummary.java | 2 +- .../autopsy/commandlineingest/CommandLineOptionProcessor.java | 2 +- .../CommonAttributesSearchResultsViewerTable.java | 2 +- .../autopsy/communications/relationships/SelectionInfo.java | 2 +- Core/src/org/sleuthkit/autopsy/contentviewers/HtmlPanel.java | 2 +- Core/src/org/sleuthkit/autopsy/datamodel/BaseChildFactory.java | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/datasourcesummary/DataSourceSummary.java b/Core/src/org/sleuthkit/autopsy/casemodule/datasourcesummary/DataSourceSummary.java index 390dce1afe..b50c3596bd 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/datasourcesummary/DataSourceSummary.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/datasourcesummary/DataSourceSummary.java @@ -21,7 +21,7 @@ package org.sleuthkit.autopsy.casemodule.datasourcesummary; import java.sql.ResultSet; import java.sql.SQLException; import java.util.logging.Level; -import java.util.logging.Logger; +import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; import org.sleuthkit.datamodel.CaseDbAccessManager; diff --git a/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineOptionProcessor.java b/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineOptionProcessor.java index 8b94961fa6..057ab7b840 100755 --- a/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineOptionProcessor.java +++ b/Core/src/org/sleuthkit/autopsy/commandlineingest/CommandLineOptionProcessor.java @@ -23,7 +23,7 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; import java.util.logging.Level; -import java.util.logging.Logger; +import org.sleuthkit.autopsy.coreutils.Logger; import org.netbeans.api.sendopts.CommandException; import org.netbeans.spi.sendopts.Env; import org.netbeans.spi.sendopts.Option; diff --git a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributesSearchResultsViewerTable.java b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributesSearchResultsViewerTable.java index 095615af7b..f42309ba04 100644 --- a/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributesSearchResultsViewerTable.java +++ b/Core/src/org/sleuthkit/autopsy/commonpropertiessearch/CommonAttributesSearchResultsViewerTable.java @@ -24,7 +24,7 @@ import java.util.Enumeration; import java.util.HashMap; import java.util.Map; import java.util.logging.Level; -import java.util.logging.Logger; +import org.sleuthkit.autopsy.coreutils.Logger; import javax.swing.table.TableColumn; import javax.swing.table.TableColumnModel; import org.openide.util.NbBundle; diff --git a/Core/src/org/sleuthkit/autopsy/communications/relationships/SelectionInfo.java b/Core/src/org/sleuthkit/autopsy/communications/relationships/SelectionInfo.java index 45ece0a5be..25727fa157 100755 --- a/Core/src/org/sleuthkit/autopsy/communications/relationships/SelectionInfo.java +++ b/Core/src/org/sleuthkit/autopsy/communications/relationships/SelectionInfo.java @@ -21,7 +21,7 @@ package org.sleuthkit.autopsy.communications.relationships; import java.util.HashSet; import java.util.Set; import java.util.logging.Level; -import java.util.logging.Logger; +import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; import org.sleuthkit.datamodel.Account; diff --git a/Core/src/org/sleuthkit/autopsy/contentviewers/HtmlPanel.java b/Core/src/org/sleuthkit/autopsy/contentviewers/HtmlPanel.java index b53b1dc258..7a9de37fa3 100755 --- a/Core/src/org/sleuthkit/autopsy/contentviewers/HtmlPanel.java +++ b/Core/src/org/sleuthkit/autopsy/contentviewers/HtmlPanel.java @@ -22,7 +22,7 @@ import java.io.IOException; import java.io.StringReader; import java.util.List; import java.util.logging.Level; -import java.util.logging.Logger; +import org.sleuthkit.autopsy.coreutils.Logger; import javafx.application.Platform; import javafx.beans.value.ChangeListener; import javafx.beans.value.ObservableValue; diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/BaseChildFactory.java b/Core/src/org/sleuthkit/autopsy/datamodel/BaseChildFactory.java index 9727d83e39..d3a27a1eb3 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/BaseChildFactory.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/BaseChildFactory.java @@ -28,7 +28,7 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.function.Predicate; import java.util.logging.Level; -import java.util.logging.Logger; +import org.sleuthkit.autopsy.coreutils.Logger; import java.util.prefs.PreferenceChangeEvent; import java.util.stream.Collectors; import org.openide.nodes.ChildFactory; From ba611f81e639d065da76c4d57d9f3f09a8e1fe4c Mon Sep 17 00:00:00 2001 From: Mark McKinnon Date: Tue, 30 Jul 2019 14:54:13 -0400 Subject: [PATCH 009/102] Change Logger from java to Autopsy Change the logger from java to Autopsy --- .../datasourcesummary/DataSourceBrowser.java | 2 +- .../DataSourceInfoUtilities.java | 2 +- .../DataSourceSummaryCountsPanel.form | 6 +- .../DataSourceSummaryCountsPanel.java | 2 +- .../DataSourceSummaryDetailsPanel.form | 68 +++++++++---------- .../DataSourceSummaryDetailsPanel.java | 2 +- 6 files changed, 41 insertions(+), 41 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/datasourcesummary/DataSourceBrowser.java b/Core/src/org/sleuthkit/autopsy/casemodule/datasourcesummary/DataSourceBrowser.java index ac09010de1..92967738b0 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/datasourcesummary/DataSourceBrowser.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/datasourcesummary/DataSourceBrowser.java @@ -30,7 +30,7 @@ import java.util.List; import java.util.Map; import java.util.Observer; import java.util.logging.Level; -import java.util.logging.Logger; +import org.sleuthkit.autopsy.coreutils.Logger; import javax.swing.event.ListSelectionListener; import org.openide.nodes.Node; import org.sleuthkit.autopsy.casemodule.Case; diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/datasourcesummary/DataSourceInfoUtilities.java b/Core/src/org/sleuthkit/autopsy/casemodule/datasourcesummary/DataSourceInfoUtilities.java index e1efb0dc50..c2d8263056 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/datasourcesummary/DataSourceInfoUtilities.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/datasourcesummary/DataSourceInfoUtilities.java @@ -22,7 +22,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.logging.Level; -import java.util.logging.Logger; +import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.TskCoreException; import java.util.Collections; diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/datasourcesummary/DataSourceSummaryCountsPanel.form b/Core/src/org/sleuthkit/autopsy/casemodule/datasourcesummary/DataSourceSummaryCountsPanel.form index 232c6889ba..d3a17e0205 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/datasourcesummary/DataSourceSummaryCountsPanel.form +++ b/Core/src/org/sleuthkit/autopsy/casemodule/datasourcesummary/DataSourceSummaryCountsPanel.form @@ -81,7 +81,7 @@ - + @@ -104,14 +104,14 @@ - + - + diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/datasourcesummary/DataSourceSummaryCountsPanel.java b/Core/src/org/sleuthkit/autopsy/casemodule/datasourcesummary/DataSourceSummaryCountsPanel.java index c04926d84f..c31f89f34d 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/datasourcesummary/DataSourceSummaryCountsPanel.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/datasourcesummary/DataSourceSummaryCountsPanel.java @@ -21,7 +21,7 @@ package org.sleuthkit.autopsy.casemodule.datasourcesummary; import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.logging.Logger; +import org.sleuthkit.autopsy.coreutils.Logger; import javax.swing.JLabel; import javax.swing.table.AbstractTableModel; import javax.swing.table.DefaultTableCellRenderer; diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/datasourcesummary/DataSourceSummaryDetailsPanel.form b/Core/src/org/sleuthkit/autopsy/casemodule/datasourcesummary/DataSourceSummaryDetailsPanel.form index d45990f66d..80a8a890c7 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/datasourcesummary/DataSourceSummaryDetailsPanel.form +++ b/Core/src/org/sleuthkit/autopsy/casemodule/datasourcesummary/DataSourceSummaryDetailsPanel.form @@ -40,7 +40,7 @@ - + @@ -52,7 +52,7 @@ - + @@ -64,7 +64,7 @@ - + @@ -76,7 +76,7 @@ - + @@ -88,10 +88,10 @@ - + - + @@ -103,7 +103,7 @@ - + @@ -115,7 +115,7 @@ - + @@ -127,7 +127,7 @@ - + @@ -139,10 +139,10 @@ - + - + @@ -179,7 +179,7 @@ - <ResourceString bundle="org/sleuthkit/autopsy/casemodule/datasourcesummary/Bundle.properties" key="DataSourceSummaryDetailsPanel.filePathsTable.columnModel.title0" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, "{key}")"/> + <ResourceString bundle="org/sleuthkit/autopsy/casemodule/datasourceSummary/Bundle.properties" key="DataSourceSummaryDetailsPanel.filePathsTable.columnModel.title0" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, "{key}")"/> @@ -196,7 +196,7 @@ - + @@ -208,7 +208,7 @@ - + @@ -220,10 +220,10 @@ - + - + @@ -235,10 +235,10 @@ - + - + @@ -250,7 +250,7 @@ - + @@ -262,7 +262,7 @@ - + @@ -274,7 +274,7 @@ - + @@ -286,7 +286,7 @@ - + @@ -298,7 +298,7 @@ - + @@ -310,7 +310,7 @@ - + @@ -322,7 +322,7 @@ - + @@ -334,7 +334,7 @@ - + @@ -346,7 +346,7 @@ - + @@ -358,7 +358,7 @@ - + @@ -370,7 +370,7 @@ - + @@ -382,7 +382,7 @@ - + @@ -394,7 +394,7 @@ - + @@ -427,7 +427,7 @@ - + @@ -469,7 +469,7 @@ - + @@ -481,7 +481,7 @@ - + diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/datasourcesummary/DataSourceSummaryDetailsPanel.java b/Core/src/org/sleuthkit/autopsy/casemodule/datasourcesummary/DataSourceSummaryDetailsPanel.java index 7702a0a20b..cf06a198cb 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/datasourcesummary/DataSourceSummaryDetailsPanel.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/datasourcesummary/DataSourceSummaryDetailsPanel.java @@ -22,7 +22,7 @@ import java.text.DecimalFormat; import java.util.Map; import java.util.HashMap; import java.util.logging.Level; -import java.util.logging.Logger; +import org.sleuthkit.autopsy.coreutils.Logger; import javax.swing.table.DefaultTableModel; import org.openide.util.NbBundle.Messages; import org.sleuthkit.datamodel.DataSource; From 79a61ba85109b8e32e340e16565b5680611b0b19 Mon Sep 17 00:00:00 2001 From: Mark McKinnon Date: Tue, 30 Jul 2019 15:00:06 -0400 Subject: [PATCH 010/102] Change logger from java to Autopsy Change the logger from java to Autopsy. --- Testing/src/org/sleuthkit/autopsy/testing/AutopsyTestCases.java | 2 +- .../src/org/sleuthkit/autopsy/testing/RegressionTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Testing/src/org/sleuthkit/autopsy/testing/AutopsyTestCases.java b/Testing/src/org/sleuthkit/autopsy/testing/AutopsyTestCases.java index 2a140bc84f..dd877b3a11 100644 --- a/Testing/src/org/sleuthkit/autopsy/testing/AutopsyTestCases.java +++ b/Testing/src/org/sleuthkit/autopsy/testing/AutopsyTestCases.java @@ -32,7 +32,7 @@ import java.util.Date; import java.util.List; import java.util.Random; import java.util.logging.Level; -import java.util.logging.Logger; +import org.sleuthkit.autopsy.coreutils.Logger; import javax.imageio.ImageIO; import javax.swing.JDialog; import javax.swing.text.JTextComponent; diff --git a/Testing/test/qa-functional/src/org/sleuthkit/autopsy/testing/RegressionTest.java b/Testing/test/qa-functional/src/org/sleuthkit/autopsy/testing/RegressionTest.java index 8f1ef48a4c..0a7feaa46c 100644 --- a/Testing/test/qa-functional/src/org/sleuthkit/autopsy/testing/RegressionTest.java +++ b/Testing/test/qa-functional/src/org/sleuthkit/autopsy/testing/RegressionTest.java @@ -20,7 +20,7 @@ package org.sleuthkit.autopsy.testing; import java.io.File; import java.io.IOException; -import java.util.logging.Logger; +import org.sleuthkit.autopsy.coreutils.Logger; import junit.framework.Test; import junit.framework.TestCase; import org.netbeans.jemmy.Timeouts; From 06dc0fc5ebce24bb2cbb1c9007fa90bd533eed8b Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Tue, 30 Jul 2019 15:51:11 -0400 Subject: [PATCH 011/102] 5217 provide users feed back when image gallery fails to open --- .../autopsy/imagegallery/actions/Bundle.properties-MERGED | 2 ++ .../sleuthkit/autopsy/imagegallery/actions/OpenAction.java | 6 +++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/actions/Bundle.properties-MERGED b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/actions/Bundle.properties-MERGED index 3ac90630fa..32b96181cb 100755 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/actions/Bundle.properties-MERGED +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/actions/Bundle.properties-MERGED @@ -35,6 +35,8 @@ OpenAction.multiUserDialog.Header=Multi-user Image Gallery OpenAction.noControllerDialog.header=Cannot open Image Gallery OpenAction.noControllerDialog.text=An initialization error ocurred.\nPlease see the log for details. OpenAction.notAnalyzedDlg.msg=No image/video files available to display yet.\nPlease run FileType and EXIF ingest modules. +OpenAction.openTopComponent.error.message=An error occurred while attempting to open Image Gallery. +OpenAction.openTopComponent.error.title=Failed to open Image Gallery OpenAction.stale.confDlg.msg=The image / video database may be out of date. Do you want to update and listen for further ingest results?\nChoosing 'yes' will update the database and enable listening to future ingests. OpenAction.stale.confDlg.title=Image Gallery OpenExternalViewerAction.displayName=External Viewer diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/actions/OpenAction.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/actions/OpenAction.java index dec4d44c31..2df0ee2ed1 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/actions/OpenAction.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/actions/OpenAction.java @@ -34,6 +34,7 @@ import javafx.stage.Modality; import javax.swing.ImageIcon; import javax.swing.JButton; import javax.swing.JMenuItem; +import javax.swing.JOptionPane; import javax.swing.SwingUtilities; import org.openide.awt.ActionID; import org.openide.awt.ActionReference; @@ -43,6 +44,7 @@ import org.openide.util.HelpCtx; import org.openide.util.NbBundle; import org.openide.util.NbBundle.Messages; import org.openide.util.actions.CallableSystemAction; +import org.openide.windows.WindowManager; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; import org.sleuthkit.autopsy.core.Installer; @@ -292,13 +294,15 @@ public final class OpenAction extends CallableSystemAction { ); } + @Messages({"OpenAction.openTopComponent.error.message=An error occurred while attempting to open Image Gallery.", + "OpenAction.openTopComponent.error.title=Failed to open Image Gallery"}) private void openTopComponent() { SwingUtilities.invokeLater(() -> { try { ImageGalleryTopComponent.openTopComponent(); } catch (TskCoreException ex) { logger.log(Level.SEVERE, "Failed to open Image Gallery top component", ex); //NON-NLS} - // TODO (JIRA-5217): Give the user some feedback here + JOptionPane.showMessageDialog(WindowManager.getDefault().getMainWindow(), Bundle.OpenAction_openTopComponent_error_message(), Bundle.OpenAction_openTopComponent_error_title(), JOptionPane.PLAIN_MESSAGE); } }); } From 01bd63c29c66b5967c38e1d39cd66f6acbb12966 Mon Sep 17 00:00:00 2001 From: Brian Carrier Date: Fri, 2 Aug 2019 17:14:18 -0400 Subject: [PATCH 012/102] Updated news --- NEWS.txt | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/NEWS.txt b/NEWS.txt index e9c61baf7b..b4bc6e8dbf 100644 --- a/NEWS.txt +++ b/NEWS.txt @@ -1,3 +1,48 @@ +---------------- VERSION 4.12.0 -------------- +Collection +- Added ability to configure a USB drive to use new logical imager tool. +- Added logical imager tool that runs on a live Windows computer and saves results to a USB drive. +- Added ability to import logical imager results into Autopsy as a data source. + +Ingest Modules: +- Changed file type detection so that Tika does not rely only on extension. +- Email ingest module assigns thread IDs to messages +- Android ingest modules store thread ID from their databases. + +Content Viewers (lower right of UI): +- New “Text” viewer that consolidates previous Strings and “Indexed Text” viewers. +- New “Translation” panel was added to the new “Text” viewer. +- Added integration with Google and Bing translation (credentials required) +- Redesigned “Other Occurrences” viewer to have 4th column with details of selected item. +- Added Willi Ballentin’s “Registry Hive Viewer” panel to the “Application” viewer. +- Improved HTML viewer to use style sheets and better layout. +- Added ability to draw a box on a picture while tagging it. + +Result Table (upper right of UI) +- Added paging to all views for faster loading of large data sets. +- Improved speed of displaying results when a column was sorted. + +Reporting +- Portable cases can contain files marked as Interesting Items +- Portable cases can be compressed and chunked +- “Files - Text” report can use either tabs or commas as the delimiter +- “Files - Text” report better handles Unicode text. +- Added ability to create a CSV report for the contents of a table +- HTML report for tagged pictures includes a copy with the overlay box + +Communications: +- Added Account Summary view +- Added Contacts panel to show all contacts associated with an account. +- Added Media panel to show media attachments associated with an account +- Added filter to show accounts if they involved with the most recent messages. +- Messages can be grouped by thread. + +Auto Ingest +- New Test button was added to help diagnose permission and configuration issues. + +Documentation: +- Created new Triage Standard Operating Procedure (SOP) section to the User Docs + ---------------- VERSION 4.11.0 -------------- New Features: From a1d91a8b6bdbd9a9573b15d9a6c272c4100f4ae2 Mon Sep 17 00:00:00 2001 From: Kelly Kelly Date: Mon, 12 Aug 2019 12:02:23 -0400 Subject: [PATCH 013/102] Correct the cvt summary media attachments count and added total count --- .../relationships/Bundle.properties | 6 +- .../relationships/Bundle.properties-MERGED | 6 +- .../relationships/SelectionInfo.java | 120 ++++++++++-------- .../relationships/SummaryViewer.form | 38 ++++-- .../relationships/SummaryViewer.java | 45 ++++--- 5 files changed, 135 insertions(+), 80 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/communications/relationships/Bundle.properties b/Core/src/org/sleuthkit/autopsy/communications/relationships/Bundle.properties index 701a7b1261..9eb64c45be 100755 --- a/Core/src/org/sleuthkit/autopsy/communications/relationships/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/communications/relationships/Bundle.properties @@ -1,12 +1,10 @@ ContactDetailsPane.nameLabel.text=Placeholder SummaryViewer.countsPanel.border.title=Counts SummaryViewer.contactsLabel.text=Contacts: -SummaryViewer.attachmentsLabel.text=Media Attachments: OutlineViewPanel.messageLabel.text= SummaryViewer.messagesDataLabel.text=messages SummaryViewer.callLogsDataLabel.text=callLogs SummaryViewer.contactsDataLabel.text=contacts -SummaryViewer.attachmentsDataLabel.text=attachments SummaryViewer.messagesLabel.text=Messages: SummaryViewer.callLogsLabel.text=Call Logs: ThreadRootMessagePanel.showAllCheckBox.text=Show All Messages @@ -19,3 +17,7 @@ MessageViewer.showingMessagesLabel.text=Showing Messages for Thread: MessageViewer.backButton.AccessibleContext.accessibleDescription= MessageViewer.backButton.text=Threads MessageViewer.showAllButton.text=All Messages +SummaryViewer.thumbnailCntLabel.text=Media Attachments: +SummaryViewer.attachmentsLable.text=Attachments: +SummaryViewer.thumbnailsDataLabel.text=attachments +SummaryViewer.attachmentDataLabel.text=jLabel1 diff --git a/Core/src/org/sleuthkit/autopsy/communications/relationships/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/communications/relationships/Bundle.properties-MERGED index f2aa0df8e1..67971ffa88 100755 --- a/Core/src/org/sleuthkit/autopsy/communications/relationships/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/communications/relationships/Bundle.properties-MERGED @@ -37,12 +37,10 @@ MessageViewer_viewMessage_selected=Selected MessageViewer_viewMessage_unthreaded=Unthreaded SummaryViewer.countsPanel.border.title=Counts SummaryViewer.contactsLabel.text=Contacts: -SummaryViewer.attachmentsLabel.text=Media Attachments: OutlineViewPanel.messageLabel.text= SummaryViewer.messagesDataLabel.text=messages SummaryViewer.callLogsDataLabel.text=callLogs SummaryViewer.contactsDataLabel.text=contacts -SummaryViewer.attachmentsDataLabel.text=attachments SummaryViewer.messagesLabel.text=Messages: SummaryViewer.callLogsLabel.text=Call Logs: SummaryViewer_CaseRefNameColumn_Title=Case Name @@ -61,3 +59,7 @@ MessageViewer.showingMessagesLabel.text=Showing Messages for Thread: MessageViewer.backButton.AccessibleContext.accessibleDescription= MessageViewer.backButton.text=Threads MessageViewer.showAllButton.text=All Messages +SummaryViewer.thumbnailCntLabel.text=Media Attachments: +SummaryViewer.attachmentsLable.text=Attachments: +SummaryViewer.thumbnailsDataLabel.text=attachments +SummaryViewer.attachmentDataLabel.text=jLabel1 diff --git a/Core/src/org/sleuthkit/autopsy/communications/relationships/SelectionInfo.java b/Core/src/org/sleuthkit/autopsy/communications/relationships/SelectionInfo.java index 45ece0a5be..1e83de831e 100755 --- a/Core/src/org/sleuthkit/autopsy/communications/relationships/SelectionInfo.java +++ b/Core/src/org/sleuthkit/autopsy/communications/relationships/SelectionInfo.java @@ -24,6 +24,7 @@ import java.util.logging.Level; import java.util.logging.Logger; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; +import org.sleuthkit.autopsy.coreutils.ImageUtils; import org.sleuthkit.datamodel.Account; import org.sleuthkit.datamodel.AccountDeviceInstance; import org.sleuthkit.datamodel.BlackboardArtifact; @@ -37,30 +38,30 @@ import org.sleuthkit.datamodel.TskCoreException; * VisualizationPane */ public final class SelectionInfo { - + private static final Logger logger = Logger.getLogger(SelectionInfo.class.getName()); private final Set selectedNodes; private final Set selectedEdges; private final CommunicationsFilter communicationFilter; private final Set accounts; - + private Set accountArtifacts = null; private SelectionSummary summary = null; /** * Wraps the details of the currently selected accounts. * - * @param selectedNodes Selected AccountDeviceInstances - * @param selectedEdges Selected pairs of AccountDeviceInstances - * @param communicationFilter Currently selected communications filters + * @param selectedNodes Selected AccountDeviceInstances + * @param selectedEdges Selected pairs of AccountDeviceInstances + * @param communicationFilter Currently selected communications filters */ - public SelectionInfo(Set selectedNodes, Set selectedEdges, + public SelectionInfo(Set selectedNodes, Set selectedEdges, CommunicationsFilter communicationFilter) { this.selectedNodes = selectedNodes; this.selectedEdges = selectedEdges; this.communicationFilter = communicationFilter; - + accounts = new HashSet<>(); selectedNodes.forEach((instance) -> { accounts.add(instance.getAccount()); @@ -75,10 +76,10 @@ public final class SelectionInfo { public Set getSelectedNodes() { return selectedNodes; } - + /** * Returns the currently selected edges - * + * * @return Set of GraphEdge objects */ public Set getSelectedEdges() { @@ -93,16 +94,17 @@ public final class SelectionInfo { public CommunicationsFilter getCommunicationsFilter() { return communicationFilter; } - + public Set getAccounts() { return accounts; } - + /** * Get the set of relationship sources from the case database - * + * * @return the relationship sources (may be empty) - * @throws TskCoreException + * + * @throws TskCoreException */ Set getRelationshipSources() throws TskCoreException { @@ -112,28 +114,28 @@ public final class SelectionInfo { } catch (NoCurrentCaseException ex) { throw new TskCoreException("Failed to get current case", ex); } - + Set relationshipSources = new HashSet<>(); try { // Add all nodes relationshipSources.addAll(communicationManager.getRelationshipSources(getSelectedNodes(), getCommunicationsFilter())); - + // Add all edges. For edges, the relationship has to include both endpoints for (SelectionInfo.GraphEdge edge : getSelectedEdges()) { - relationshipSources.addAll(communicationManager.getRelationshipSources(edge.getStartNode(), + relationshipSources.addAll(communicationManager.getRelationshipSources(edge.getStartNode(), edge.getEndNode(), getCommunicationsFilter())); } } catch (TskCoreException ex) { logger.log(Level.SEVERE, "Failed to get relationships from case database.", ex); //NON-NLS - + } return relationshipSources; } - + public Set getArtifacts() { - if(accountArtifacts == null) { + if (accountArtifacts == null) { accountArtifacts = new HashSet<>(); - + try { final Set relationshipSources = getRelationshipSources(); relationshipSources.stream().filter((content) -> (content instanceof BlackboardArtifact)).forEachOrdered((content) -> { @@ -144,58 +146,67 @@ public final class SelectionInfo { return accountArtifacts; } } - + return accountArtifacts; } - + public SelectionSummary getSummary() { - if(summary == null) { + if (summary == null) { summary = new SelectionSummary(); } - + return summary; } - - final class SelectionSummary{ + + final class SelectionSummary { + int attachmentCnt; int messagesCnt; int emailCnt; int callLogCnt; int contactsCnt; - + int mediaCnt; + SelectionSummary() { getCounts(); } - - private void getCounts(){ - for(BlackboardArtifact artifact: getArtifacts()) { + + private void getCounts() { + for (BlackboardArtifact artifact : getArtifacts()) { BlackboardArtifact.ARTIFACT_TYPE fromID = BlackboardArtifact.ARTIFACT_TYPE.fromID(artifact.getArtifactTypeID()); - if(null != fromID) switch (fromID) { - case TSK_EMAIL_MSG: - emailCnt++; - break; - case TSK_CALLLOG: - callLogCnt++; - break; - case TSK_MESSAGE: - messagesCnt++; - break; - case TSK_CONTACT: - contactsCnt++; - break; - default: - break; + if (null != fromID) { + switch (fromID) { + case TSK_EMAIL_MSG: + emailCnt++; + break; + case TSK_CALLLOG: + callLogCnt++; + break; + case TSK_MESSAGE: + messagesCnt++; + break; + case TSK_CONTACT: + contactsCnt++; + break; + default: + break; + } } - try{ - attachmentCnt+= artifact.getChildrenCount(); + try { + attachmentCnt += artifact.getChildrenCount(); + for (Content childContent : artifact.getChildren()) { + if (ImageUtils.thumbnailSupported(childContent)) { + mediaCnt++; + } + } } catch (TskCoreException ex) { logger.log(Level.WARNING, String.format("Exception thrown " - + "from getChildrenCount artifactID: %d", + + "from getChildrenCount artifactID: %d", artifact.getArtifactID()), ex); //NON-NLS } } } - + public int getAttachmentCnt() { return attachmentCnt; } @@ -215,24 +226,29 @@ public final class SelectionInfo { public int getContactsCnt() { return contactsCnt; } + + public int getThumbnailCnt() { + return mediaCnt; + } } /** * Utility class to represent an edge from the graph visualization. */ public static class GraphEdge { + AccountDeviceInstance startNode; AccountDeviceInstance endNode; - + public GraphEdge(AccountDeviceInstance startNode, AccountDeviceInstance endNode) { this.startNode = startNode; this.endNode = endNode; } - + public AccountDeviceInstance getStartNode() { return startNode; } - + public AccountDeviceInstance getEndNode() { return endNode; } diff --git a/Core/src/org/sleuthkit/autopsy/communications/relationships/SummaryViewer.form b/Core/src/org/sleuthkit/autopsy/communications/relationships/SummaryViewer.form index 85ddc8a2c8..73ef1a68ba 100755 --- a/Core/src/org/sleuthkit/autopsy/communications/relationships/SummaryViewer.form +++ b/Core/src/org/sleuthkit/autopsy/communications/relationships/SummaryViewer.form @@ -41,16 +41,18 @@ - + + - + + - + @@ -74,10 +76,14 @@ - - + + + + + + @@ -104,17 +110,17 @@ - + - + - + - + @@ -139,6 +145,20 @@ + + + + + + + + + + + + + + diff --git a/Core/src/org/sleuthkit/autopsy/communications/relationships/SummaryViewer.java b/Core/src/org/sleuthkit/autopsy/communications/relationships/SummaryViewer.java index 5602152ba2..539c1eb2d3 100755 --- a/Core/src/org/sleuthkit/autopsy/communications/relationships/SummaryViewer.java +++ b/Core/src/org/sleuthkit/autopsy/communications/relationships/SummaryViewer.java @@ -104,10 +104,11 @@ public class SummaryViewer extends javax.swing.JPanel implements RelationshipsVi } else { SelectionSummary summaryDetails = info.getSummary(); - attachmentsDataLabel.setText(Integer.toString(summaryDetails.getAttachmentCnt())); + thumbnailsDataLabel.setText(Integer.toString(summaryDetails.getThumbnailCnt())); callLogsDataLabel.setText(Integer.toString(summaryDetails.getCallLogCnt())); contactsDataLabel.setText(Integer.toString(summaryDetails.getContactsCnt())); messagesDataLabel.setText(Integer.toString(summaryDetails.getMessagesCnt() + summaryDetails.getEmailCnt())); + attachmentDataLabel.setText(Integer.toString(summaryDetails.getAttachmentCnt())); fileReferencesPanel.showOutlineView(); @@ -131,7 +132,7 @@ public class SummaryViewer extends javax.swing.JPanel implements RelationshipsVi @Override public void setEnabled(boolean enabled) { super.setEnabled(enabled); - attachmentsLabel.setEnabled(enabled); + thumbnailCntLabel.setEnabled(enabled); callLogsLabel.setEnabled(enabled); contactsLabel.setEnabled(enabled); messagesLabel.setEnabled(enabled); @@ -144,10 +145,11 @@ public class SummaryViewer extends javax.swing.JPanel implements RelationshipsVi * Clears the text fields and OutlookViews. */ private void clearControls() { - attachmentsDataLabel.setText(""); + thumbnailsDataLabel.setText(""); callLogsDataLabel.setText(""); contactsDataLabel.setText(""); messagesDataLabel.setText(""); + attachmentDataLabel.setText(""); fileReferencesPanel.setNode(new AbstractNode(Children.LEAF)); caseReferencesPanel.setNode(new AbstractNode(Children.LEAF)); @@ -187,11 +189,13 @@ public class SummaryViewer extends javax.swing.JPanel implements RelationshipsVi contactsLabel = new javax.swing.JLabel(); messagesLabel = new javax.swing.JLabel(); callLogsLabel = new javax.swing.JLabel(); - attachmentsLabel = new javax.swing.JLabel(); - attachmentsDataLabel = new javax.swing.JLabel(); + thumbnailCntLabel = new javax.swing.JLabel(); + thumbnailsDataLabel = new javax.swing.JLabel(); messagesDataLabel = new javax.swing.JLabel(); callLogsDataLabel = new javax.swing.JLabel(); contactsDataLabel = new javax.swing.JLabel(); + attachmentsLable = new javax.swing.JLabel(); + attachmentDataLabel = new javax.swing.JLabel(); fileReferencesPanel = new org.sleuthkit.autopsy.communications.relationships.OutlineViewPanel(); caseReferencesPanel = new org.sleuthkit.autopsy.communications.relationships.OutlineViewPanel(); @@ -205,9 +209,9 @@ public class SummaryViewer extends javax.swing.JPanel implements RelationshipsVi org.openide.awt.Mnemonics.setLocalizedText(callLogsLabel, org.openide.util.NbBundle.getMessage(SummaryViewer.class, "SummaryViewer.callLogsLabel.text")); // NOI18N - org.openide.awt.Mnemonics.setLocalizedText(attachmentsLabel, org.openide.util.NbBundle.getMessage(SummaryViewer.class, "SummaryViewer.attachmentsLabel.text")); // NOI18N + org.openide.awt.Mnemonics.setLocalizedText(thumbnailCntLabel, org.openide.util.NbBundle.getMessage(SummaryViewer.class, "SummaryViewer.thumbnailCntLabel.text")); // NOI18N - org.openide.awt.Mnemonics.setLocalizedText(attachmentsDataLabel, org.openide.util.NbBundle.getMessage(SummaryViewer.class, "SummaryViewer.attachmentsDataLabel.text")); // NOI18N + org.openide.awt.Mnemonics.setLocalizedText(thumbnailsDataLabel, org.openide.util.NbBundle.getMessage(SummaryViewer.class, "SummaryViewer.thumbnailsDataLabel.text")); // NOI18N org.openide.awt.Mnemonics.setLocalizedText(messagesDataLabel, org.openide.util.NbBundle.getMessage(SummaryViewer.class, "SummaryViewer.messagesDataLabel.text")); // NOI18N @@ -215,6 +219,10 @@ public class SummaryViewer extends javax.swing.JPanel implements RelationshipsVi org.openide.awt.Mnemonics.setLocalizedText(contactsDataLabel, org.openide.util.NbBundle.getMessage(SummaryViewer.class, "SummaryViewer.contactsDataLabel.text")); // NOI18N + org.openide.awt.Mnemonics.setLocalizedText(attachmentsLable, org.openide.util.NbBundle.getMessage(SummaryViewer.class, "SummaryViewer.attachmentsLable.text")); // NOI18N + + org.openide.awt.Mnemonics.setLocalizedText(attachmentDataLabel, org.openide.util.NbBundle.getMessage(SummaryViewer.class, "SummaryViewer.attachmentDataLabel.text")); // NOI18N + javax.swing.GroupLayout countsPanelLayout = new javax.swing.GroupLayout(countsPanel); countsPanel.setLayout(countsPanelLayout); countsPanelLayout.setHorizontalGroup( @@ -225,14 +233,16 @@ public class SummaryViewer extends javax.swing.JPanel implements RelationshipsVi .addComponent(messagesLabel) .addComponent(callLogsLabel) .addComponent(contactsLabel) - .addComponent(attachmentsLabel)) + .addComponent(thumbnailCntLabel) + .addComponent(attachmentsLable)) .addGap(18, 18, 18) .addGroup(countsPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addComponent(attachmentsDataLabel) + .addComponent(attachmentDataLabel) + .addComponent(thumbnailsDataLabel) .addComponent(contactsDataLabel) .addComponent(callLogsDataLabel) .addComponent(messagesDataLabel)) - .addContainerGap(959, Short.MAX_VALUE)) + .addContainerGap(845, Short.MAX_VALUE)) ); countsPanelLayout.setVerticalGroup( countsPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) @@ -251,9 +261,12 @@ public class SummaryViewer extends javax.swing.JPanel implements RelationshipsVi .addComponent(contactsDataLabel)) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) .addGroup(countsPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) - .addComponent(attachmentsLabel) - .addComponent(attachmentsDataLabel)) - .addContainerGap(javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)) + .addComponent(thumbnailCntLabel) + .addComponent(thumbnailsDataLabel)) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) + .addGroup(countsPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) + .addComponent(attachmentsLable) + .addComponent(attachmentDataLabel))) ); gridBagConstraints = new java.awt.GridBagConstraints(); @@ -287,8 +300,8 @@ public class SummaryViewer extends javax.swing.JPanel implements RelationshipsVi // Variables declaration - do not modify//GEN-BEGIN:variables - private javax.swing.JLabel attachmentsDataLabel; - private javax.swing.JLabel attachmentsLabel; + private javax.swing.JLabel attachmentDataLabel; + private javax.swing.JLabel attachmentsLable; private javax.swing.JLabel callLogsDataLabel; private javax.swing.JLabel callLogsLabel; private org.sleuthkit.autopsy.communications.relationships.OutlineViewPanel caseReferencesPanel; @@ -298,6 +311,8 @@ public class SummaryViewer extends javax.swing.JPanel implements RelationshipsVi private org.sleuthkit.autopsy.communications.relationships.OutlineViewPanel fileReferencesPanel; private javax.swing.JLabel messagesDataLabel; private javax.swing.JLabel messagesLabel; + private javax.swing.JLabel thumbnailCntLabel; + private javax.swing.JLabel thumbnailsDataLabel; // End of variables declaration//GEN-END:variables } From fe0e1b4cf188fd08e1df7d0694a87c785d998df1 Mon Sep 17 00:00:00 2001 From: Joe Ho Date: Tue, 13 Aug 2019 10:14:59 -0400 Subject: [PATCH 014/102] allow adding directories as Logical File Set --- .../autopsy/casemodule/AddLocalFilesTask.java | 10 +-- .../dsp/AddLogicalImageTask.java | 17 +---- .../dsp/Bundle.properties-MERGED | 2 + .../dsp/LogicalImagerDSProcessor.java | 66 +++++++++++++++---- .../logicalimager/dsp/LogicalImagerPanel.java | 16 ++++- 5 files changed, 74 insertions(+), 37 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/AddLocalFilesTask.java b/Core/src/org/sleuthkit/autopsy/casemodule/AddLocalFilesTask.java index feae1ddb6b..775fc51d8c 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/AddLocalFilesTask.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/AddLocalFilesTask.java @@ -22,12 +22,12 @@ import java.util.ArrayList; import java.util.List; import java.util.logging.Level; import org.openide.util.NbBundle; +import org.sleuthkit.autopsy.casemodule.services.FileManager; import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorCallback; import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorProgressMonitor; +import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.Content; -import org.sleuthkit.autopsy.casemodule.services.FileManager; -import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.datamodel.LocalFilesDataSource; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskDataException; @@ -37,7 +37,7 @@ import org.sleuthkit.datamodel.TskDataException; * case database, grouped under a virtual directory that serves as the data * source. */ -class AddLocalFilesTask implements Runnable { +public class AddLocalFilesTask implements Runnable { private static final Logger LOGGER = Logger.getLogger(AddLocalFilesTask.class.getName()); private final String deviceId; @@ -68,7 +68,7 @@ class AddLocalFilesTask implements Runnable { * during processing. * @param callback Callback to call when processing is done. */ - AddLocalFilesTask(String deviceId, String rootVirtualDirectoryName, List localFilePaths, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callback) { + public AddLocalFilesTask(String deviceId, String rootVirtualDirectoryName, List localFilePaths, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callback) { this.deviceId = deviceId; this.rootVirtualDirectoryName = rootVirtualDirectoryName; this.localFilePaths = localFilePaths; @@ -88,7 +88,7 @@ class AddLocalFilesTask implements Runnable { try { progress.setIndeterminate(true); FileManager fileManager = Case.getCurrentCaseThrows().getServices().getFileManager(); - LocalFilesDataSource newDataSource = fileManager.addLocalFilesDataSource(deviceId, rootVirtualDirectoryName, "", localFilePaths, new ProgressUpdater()); + LocalFilesDataSource newDataSource = fileManager.addLocalFilesDataSource(deviceId, "", "", localFilePaths, new ProgressUpdater()); newDataSources.add(newDataSource); } catch (TskDataException | TskCoreException | NoCurrentCaseException ex) { errors.add(ex.getMessage()); diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java index 9070dfc197..cac1ea4345 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java @@ -19,13 +19,11 @@ package org.sleuthkit.autopsy.logicalimager.dsp; import java.io.File; -import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; import java.util.logging.Level; -import org.apache.commons.io.FileUtils; import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; @@ -80,20 +78,7 @@ final class AddLogicalImageTask extends AddMultipleImageTask { List errorList = new ArrayList<>(); List emptyDataSources = new ArrayList<>(); - try { - progressMonitor.setProgressText(Bundle.AddLogicalImageTask_copyingImageFromTo(src.toString(), dest.toString())); - FileUtils.copyDirectory(src, dest); - progressMonitor.setProgressText(Bundle.AddLogicalImageTask_doneCopying()); - } catch (IOException ex) { - // Copy directory failed - String msg = Bundle.AddLogicalImageTask_failedToCopyDirectory(src.toString(), dest.toString()); - errorList.add(msg); - logger.log(Level.SEVERE, String.format("Failed to copy directory %s to %s", src.toString(), dest.toString()), ex); - callback.done(DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS, errorList, emptyDataSources); - return; - } - - // Add the alert.txt and users.txt to the case report + // Add the alert.txt and users.txt to the case report progressMonitor.setProgressText(Bundle.AddLogicalImageTask_addingToReport(ALERT_TXT)); String status = addReport(Paths.get(dest.toString(), ALERT_TXT), ALERT_TXT + " " + src.getName()); if (status != null) { diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/Bundle.properties-MERGED index 32bd3c7868..2551162ab2 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/Bundle.properties-MERGED @@ -39,6 +39,8 @@ AddMultipleImageTask.nonCriticalErrorAdding=Non-critical error adding {0} for de LogicalImagerDSProcessor.dataSourceType=Autopsy Logical Imager Results # {0} - directory LogicalImagerDSProcessor.directoryAlreadyExists=Directory {0} already exists +# {0} - sparseImageDirectory +LogicalImagerDSProcessor.directoryDoesNotContainSparseImage=Directory {0} does not contain any images # {0} - directory LogicalImagerDSProcessor.failToCreateDirectory=Failed to create directory {0} # {0} - file diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/LogicalImagerDSProcessor.java b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/LogicalImagerDSProcessor.java index da9d5f7aa6..ae0f3d533f 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/LogicalImagerDSProcessor.java +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/LogicalImagerDSProcessor.java @@ -19,6 +19,7 @@ package org.sleuthkit.autopsy.logicalimager.dsp; import java.io.File; +import java.io.FilenameFilter; import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; @@ -27,9 +28,11 @@ import java.util.Calendar; import java.util.List; import java.util.UUID; import javax.swing.JPanel; +import org.apache.commons.io.FileUtils; import org.openide.util.NbBundle.Messages; import org.openide.util.lookup.ServiceProvider; import org.openide.util.lookup.ServiceProviders; +import org.sleuthkit.autopsy.casemodule.AddLocalFilesTask; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessor; @@ -131,7 +134,10 @@ public final class LogicalImagerDSProcessor implements DataSourceProcessor { "# {0} - directory", "LogicalImagerDSProcessor.failToCreateDirectory=Failed to create directory {0}", "# {0} - directory", "LogicalImagerDSProcessor.directoryAlreadyExists=Directory {0} already exists", "# {0} - file", "LogicalImagerDSProcessor.failToGetCanonicalPath=Fail to get canonical path for {0}", - "LogicalImagerDSProcessor.noCurrentCase=No current case",}) + "LogicalImagerDSProcessor.noCurrentCase=No current case", + "# {0} - sparseImageDirectory", "LogicalImagerDSProcessor.directoryDoesNotContainSparseImage=Directory {0} does not contain any images", + + }) @Override public void run(DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callback) { configPanel.storeSettings(); @@ -170,9 +176,21 @@ public final class LogicalImagerDSProcessor implements DataSourceProcessor { } File src = imageDirPath.toFile(); + try { + progressMonitor.setProgressText(Bundle.AddLogicalImageTask_copyingImageFromTo(src.toString(), dest.toString())); + FileUtils.copyDirectory(src, dest); + progressMonitor.setProgressText(Bundle.AddLogicalImageTask_doneCopying()); + } catch (IOException ex) { + // Copy directory failed + String msg = Bundle.AddLogicalImageTask_failedToCopyDirectory(src.toString(), dest.toString()); + errorList.add(msg); + callback.done(DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS, errorList, emptyDataSources); + return; + } + // Get all VHD files in the src directory List imagePaths = new ArrayList<>(); - for (File f : src.listFiles()) { + for (File f : dest.listFiles()) { if (f.getName().endsWith(".vhd")) { try { imagePaths.add(f.getCanonicalPath()); @@ -184,17 +202,39 @@ public final class LogicalImagerDSProcessor implements DataSourceProcessor { } } } - try { - String deviceId = UUID.randomUUID().toString(); - String timeZone = Calendar.getInstance().getTimeZone().getID(); - run(deviceId, imagePaths, - timeZone, src, dest, - progressMonitor, callback); - } catch (NoCurrentCaseException ex) { - String msg = Bundle.LogicalImagerDSProcessor_noCurrentCase(); - errorList.add(msg); - callback.done(DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS, errorList, emptyDataSources); - return; + String deviceId = UUID.randomUUID().toString(); + if (imagePaths.isEmpty()) { + // No VHD in src directory, try ingest directories using Logical File Set + String[] directories = dest.list(new FilenameFilter() { + @Override + public boolean accept(File dir, String name) { + return Paths.get(dir.toString(), name).toFile().isDirectory(); + } + }); + for (String dir : directories) { + imagePaths.add(Paths.get(dest.toString(), dir).toFile().getAbsolutePath()); + } + if (imagePaths.isEmpty()) { + String msg = Bundle.LogicalImagerDSProcessor_directoryDoesNotContainSparseImage(dest); + errorList.add(msg); + callback.done(DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS, errorList, emptyDataSources); + return; + } + + // ingest the directories + new Thread(new AddLocalFilesTask(deviceId, null, imagePaths, progressMonitor, callback)).start(); + + } else { + try { + String timeZone = Calendar.getInstance().getTimeZone().getID(); + run(deviceId, imagePaths, + timeZone, src, dest, + progressMonitor, callback); + } catch (NoCurrentCaseException ex) { + String msg = Bundle.LogicalImagerDSProcessor_noCurrentCase(); + errorList.add(msg); + callback.done(DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS, errorList, emptyDataSources); + } } } diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/LogicalImagerPanel.java b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/LogicalImagerPanel.java index 06722722d6..d491c77778 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/LogicalImagerPanel.java +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/LogicalImagerPanel.java @@ -333,9 +333,19 @@ final class LogicalImagerPanel extends JPanel implements DocumentListener { } }); if (vhdFiles.length == 0) { - setErrorMessage(Bundle.LogicalImagerPanel_messageLabel_directoryDoesNotContainSparseImage(path)); - firePropertyChange(DataSourceProcessor.DSP_PANEL_EVENT.UPDATE_UI.toString(), true, false); - return; + // No VHD files, try directories for individual files + String[] directories = dir.list(new FilenameFilter() { + @Override + public boolean accept(File dir, String name) { + return Paths.get(dir.toString(), name).toFile().isDirectory(); + } + }); + if (directories.length == 0) { + // No directories, bail + setErrorMessage(Bundle.LogicalImagerPanel_messageLabel_directoryDoesNotContainSparseImage(path)); + firePropertyChange(DataSourceProcessor.DSP_PANEL_EVENT.UPDATE_UI.toString(), true, false); + return; + } } manualImageDirPath = Paths.get(path); setNormalMessage(path); From bca13c0808c089a405fb9d69ca11c5a040950943 Mon Sep 17 00:00:00 2001 From: Joe Ho Date: Tue, 13 Aug 2019 14:04:54 -0400 Subject: [PATCH 015/102] Add Create VHD checkbox to configuration panel --- .../configuration/Bundle.properties | 1 + .../configuration/Bundle.properties-MERGED | 2 ++ .../configuration/ConfigVisualPanel2.form | 14 +++++++++++- .../configuration/ConfigVisualPanel2.java | 22 ++++++++++++++++--- .../configuration/LogicalImagerConfig.java | 17 ++++++++++++++ .../LogicalImagerConfigDeserializer.java | 8 ++++++- 6 files changed, 59 insertions(+), 5 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/Bundle.properties b/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/Bundle.properties index 65fa3dd72d..6a77371df1 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/Bundle.properties @@ -122,3 +122,4 @@ EditNonFullPathsRulePanel.fileNamesInfoLabel.text=File names are case insensitiv EditNonFullPathsRulePanel.extensionsInfoLabel.text=Extensions are case insensitive. ConfigVisualPanel2.promptBeforeExit.text=Prompt before exiting imager ConfigVisualPanel2.promptBeforeExit.actionCommand= +ConfigVisualPanel2.createVHDCheckBox.text=Create VHD diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/Bundle.properties-MERGED index 3e6aad34de..d0c05b6ef4 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/Bundle.properties-MERGED @@ -191,6 +191,8 @@ EditNonFullPathsRulePanel.fileNamesInfoLabel.text=File names are case insensitiv EditNonFullPathsRulePanel.extensionsInfoLabel.text=Extensions are case insensitive. ConfigVisualPanel2.promptBeforeExit.text=Prompt before exiting imager ConfigVisualPanel2.promptBeforeExit.actionCommand= +ConfigVisualPanel2.createVHDCheckBox.toolTipText= +ConfigVisualPanel2.createVHDCheckBox.text=Create VHD NewRuleSetPanel.attributeRule.description=Search for files based on one or more attributes or metadata fields. NewRuleSetPanel.attributeRule.name=Attribute NewRuleSetPanel.fullPathRule.description=Search for files based on full exact match path. diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/ConfigVisualPanel2.form b/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/ConfigVisualPanel2.form index b5725173df..5628bdb709 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/ConfigVisualPanel2.form +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/ConfigVisualPanel2.form @@ -103,6 +103,7 @@ + @@ -193,7 +194,8 @@ - + + @@ -582,5 +584,15 @@ + + + + + + + + + + diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/ConfigVisualPanel2.java b/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/ConfigVisualPanel2.java index 65fe93fcd3..cadd5e7cca 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/ConfigVisualPanel2.java +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/ConfigVisualPanel2.java @@ -111,6 +111,7 @@ final class ConfigVisualPanel2 extends JPanel { maxSizeLabel = new javax.swing.JLabel(); maxSizeTextField = new javax.swing.JFormattedTextField(); promptBeforeExit = new javax.swing.JCheckBox(); + createVHDCheckBox = new javax.swing.JCheckBox(); org.openide.awt.Mnemonics.setLocalizedText(modifiedDateLabel, org.openide.util.NbBundle.getMessage(ConfigVisualPanel2.class, "ConfigVisualPanel2.modifiedDateLabel.text")); // NOI18N @@ -264,6 +265,13 @@ final class ConfigVisualPanel2 extends JPanel { } }); + org.openide.awt.Mnemonics.setLocalizedText(createVHDCheckBox, org.openide.util.NbBundle.getMessage(ConfigVisualPanel2.class, "ConfigVisualPanel2.createVHDCheckBox.text")); // NOI18N + createVHDCheckBox.addActionListener(new java.awt.event.ActionListener() { + public void actionPerformed(java.awt.event.ActionEvent evt) { + createVHDCheckBoxActionPerformed(evt); + } + }); + javax.swing.GroupLayout layout = new javax.swing.GroupLayout(this); this.setLayout(layout); layout.setHorizontalGroup( @@ -338,7 +346,8 @@ final class ConfigVisualPanel2 extends JPanel { .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addComponent(flagEncryptionProgramsCheckBox) .addComponent(finalizeImageWriter) - .addComponent(promptBeforeExit)) + .addComponent(promptBeforeExit) + .addComponent(createVHDCheckBox)) .addGap(0, 0, Short.MAX_VALUE)) .addComponent(jSeparator1))))) ); @@ -412,7 +421,8 @@ final class ConfigVisualPanel2 extends JPanel { .addComponent(finalizeImageWriter) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) .addComponent(promptBeforeExit) - .addGap(21, 21, 21)))) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addComponent(createVHDCheckBox)))) ); }// //GEN-END:initComponents @@ -546,6 +556,10 @@ final class ConfigVisualPanel2 extends JPanel { config.setPromptBeforeExit(promptBeforeExit.isSelected()); }//GEN-LAST:event_promptBeforeExitActionPerformed + private void createVHDCheckBoxActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_createVHDCheckBoxActionPerformed + config.setCreateVHD(createVHDCheckBox.isSelected()); + }//GEN-LAST:event_createVHDCheckBoxActionPerformed + /** * Set the whether the a rule for detecting encryption programs will be * added to the rules in this config @@ -588,6 +602,7 @@ final class ConfigVisualPanel2 extends JPanel { // Variables declaration - do not modify//GEN-BEGIN:variables private javax.swing.JTextField configFileTextField; + private javax.swing.JCheckBox createVHDCheckBox; private javax.swing.JLabel daysIncludedLabel; private javax.swing.JButton deleteRuleButton; private javax.swing.JTextField descriptionEditTextField; @@ -638,13 +653,14 @@ final class ConfigVisualPanel2 extends JPanel { * Update the panel to reflect the rules in the current config * * @param configFilePath path of the config file being modified - * @param config contents of the config file being modifed + * @param config contents of the config file being modified * @param rowSelectionkey the name of the rule to select by default */ private void updatePanel(String configFilePath, LogicalImagerConfig config, String rowSelectionkey) { configFileTextField.setText(configFilePath); finalizeImageWriter.setSelected(config.isFinalizeImageWriter()); promptBeforeExit.setSelected(config.isPromptBeforeExit()); + createVHDCheckBox.setSelected(config.isCreateVHD()); LogicalImagerRuleSet ruleSet = getRuleSetFromCurrentConfig(); flagEncryptionProgramsCheckBox.setSelected(ruleSet.find(EncryptionProgramsRule.getName()) != null); RulesTableModel rulesTableModel = new RulesTableModel(); diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/LogicalImagerConfig.java b/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/LogicalImagerConfig.java index 9d1c175de3..0dd40a7d9a 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/LogicalImagerConfig.java +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/LogicalImagerConfig.java @@ -42,6 +42,10 @@ class LogicalImagerConfig { @Expose(serialize = true) private boolean promptBeforeExit; + @SerializedName("create-VHD") + @Expose(serialize = true) + private boolean createVHD; + @SerializedName("rule-sets") @Expose(serialize = true) private List ruleSets; @@ -50,6 +54,7 @@ class LogicalImagerConfig { this.version = CURRENT_VERSION; this.finalizeImageWriter = false; this.promptBeforeExit = true; + this.createVHD = false; this.ruleSets = new ArrayList<>(); } @@ -60,6 +65,7 @@ class LogicalImagerConfig { this.version = CURRENT_VERSION; this.finalizeImageWriter = finalizeImageWriter; this.promptBeforeExit = true; + this.createVHD = false; this.ruleSets = ruleSets; } @@ -71,6 +77,7 @@ class LogicalImagerConfig { this.version = version; this.finalizeImageWriter = finalizeImageWriter; this.promptBeforeExit = true; + this.createVHD = false; this.ruleSets = ruleSets; } @@ -78,11 +85,13 @@ class LogicalImagerConfig { String version, boolean finalizeImageWriter, boolean promptBeforeExit, + boolean creatVHD, List ruleSets ) { this.version = version; this.finalizeImageWriter = finalizeImageWriter; this.promptBeforeExit = promptBeforeExit; + this.createVHD = creatVHD; this.ruleSets = ruleSets; } @@ -114,6 +123,14 @@ class LogicalImagerConfig { this.promptBeforeExit = promptBeforeExit; } + boolean isCreateVHD() { + return createVHD; + } + + void setCreateVHD(boolean createVHD) { + this.createVHD = createVHD; + } + List getRuleSets() { return ruleSets; } diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/LogicalImagerConfigDeserializer.java b/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/LogicalImagerConfigDeserializer.java index dd433b68cf..d14fa31ccf 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/LogicalImagerConfigDeserializer.java +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/LogicalImagerConfigDeserializer.java @@ -46,6 +46,7 @@ class LogicalImagerConfigDeserializer implements JsonDeserializer parseRules(JsonArray asJsonArray) { From 774ef43a22ea4d34928886c1fa99726fddbba447 Mon Sep 17 00:00:00 2001 From: Kelly Kelly Date: Wed, 14 Aug 2019 11:29:36 -0400 Subject: [PATCH 016/102] cvt-Modified label on Summary Panel --- .../autopsy/communications/relationships/Bundle.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/communications/relationships/Bundle.properties b/Core/src/org/sleuthkit/autopsy/communications/relationships/Bundle.properties index 9eb64c45be..4d0b858691 100755 --- a/Core/src/org/sleuthkit/autopsy/communications/relationships/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/communications/relationships/Bundle.properties @@ -18,6 +18,6 @@ MessageViewer.backButton.AccessibleContext.accessibleDescription= MessageViewer.backButton.text=Threads MessageViewer.showAllButton.text=All Messages SummaryViewer.thumbnailCntLabel.text=Media Attachments: -SummaryViewer.attachmentsLable.text=Attachments: +SummaryViewer.attachmentsLable.text=Total Attachments: SummaryViewer.thumbnailsDataLabel.text=attachments -SummaryViewer.attachmentDataLabel.text=jLabel1 +SummaryViewer.attachmentDataLabel.text=count From 0f15e20fd3771e263c661b4699ffc047cc2387e8 Mon Sep 17 00:00:00 2001 From: Joe Ho Date: Wed, 14 Aug 2019 14:03:31 -0400 Subject: [PATCH 017/102] Copy the source directory in AddLogicalImageTask, so progress monitor works --- .../configuration/Bundle.properties-MERGED | 1 - .../dsp/AddLogicalImageTask.java | 96 +++++++++++++++++-- .../dsp/Bundle.properties-MERGED | 10 +- .../dsp/LogicalImagerDSProcessor.java | 74 ++------------ 4 files changed, 102 insertions(+), 79 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/Bundle.properties-MERGED index d0c05b6ef4..cec2da7dfb 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/Bundle.properties-MERGED @@ -191,7 +191,6 @@ EditNonFullPathsRulePanel.fileNamesInfoLabel.text=File names are case insensitiv EditNonFullPathsRulePanel.extensionsInfoLabel.text=Extensions are case insensitive. ConfigVisualPanel2.promptBeforeExit.text=Prompt before exiting imager ConfigVisualPanel2.promptBeforeExit.actionCommand= -ConfigVisualPanel2.createVHDCheckBox.toolTipText= ConfigVisualPanel2.createVHDCheckBox.text=Create VHD NewRuleSetPanel.attributeRule.description=Search for files based on one or more attributes or metadata fields. NewRuleSetPanel.attributeRule.name=Attribute diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java index cac1ea4345..01ae872c45 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java @@ -19,12 +19,16 @@ package org.sleuthkit.autopsy.logicalimager.dsp; import java.io.File; +import java.io.FilenameFilter; +import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; import java.util.logging.Level; +import org.apache.commons.io.FileUtils; import org.openide.util.NbBundle.Messages; +import org.sleuthkit.autopsy.casemodule.AddLocalFilesTask; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorCallback; @@ -38,24 +42,28 @@ import org.sleuthkit.datamodel.TskCoreException; * alert.txt and users.txt files to report - add an image data source to the * case database. */ -final class AddLogicalImageTask extends AddMultipleImageTask { +final class AddLogicalImageTask implements Runnable { - private final static Logger logger = Logger.getLogger(AddLogicalImageTask.class.getName()); + private final static Logger LOGGER = Logger.getLogger(AddLogicalImageTask.class.getName()); private final static String ALERT_TXT = "alert.txt"; //NON-NLS private final static String USERS_TXT = "users.txt"; //NON-NLS + private final String deviceId; + private final String timeZone; private final File src; private final File dest; private final DataSourceProcessorCallback callback; private final DataSourceProcessorProgressMonitor progressMonitor; + private volatile boolean cancelled; + AddLogicalImageTask(String deviceId, - List imagePaths, String timeZone, File src, File dest, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callback ) throws NoCurrentCaseException { - super(deviceId, imagePaths, timeZone, progressMonitor, callback); + this.deviceId = deviceId; + this.timeZone = timeZone; this.src = src; this.dest = dest; this.progressMonitor = progressMonitor; @@ -71,14 +79,34 @@ final class AddLogicalImageTask extends AddMultipleImageTask { "AddLogicalImageTask.doneCopying=Done copying", "# {0} - src", "# {1} - dest", "AddLogicalImageTask.failedToCopyDirectory=Failed to copy directory {0} to {1}", "# {0} - file", "AddLogicalImageTask.addingToReport=Adding {0} to report", - "# {0} - file", "AddLogicalImageTask.doneAddingToReport=Done adding {0} to report" + "# {0} - file", "AddLogicalImageTask.doneAddingToReport=Done adding {0} to report", + "AddLogicalImageTask.ingestionCancelled=Ingestion cancelled", + "# {0} - file", "AddLogicalImageTask.failToGetCanonicalPath=Fail to get canonical path for {0}", + "# {0} - sparseImageDirectory", "AddLogicalImageTask.directoryDoesNotContainSparseImage=Directory {0} does not contain any images", + "AddLogicalImageTask.noCurrentCase=No current case", }) @Override public void run() { List errorList = new ArrayList<>(); List emptyDataSources = new ArrayList<>(); - // Add the alert.txt and users.txt to the case report + try { + progressMonitor.setProgressText(Bundle.AddLogicalImageTask_copyingImageFromTo(src.toString(), dest.toString())); + FileUtils.copyDirectory(src, dest); + progressMonitor.setProgressText(Bundle.AddLogicalImageTask_doneCopying()); + } catch (IOException ex) { + // Copy directory failed + String msg = Bundle.AddLogicalImageTask_failedToCopyDirectory(src.toString(), dest.toString()); + errorList.add(msg); + callback.done(DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS, errorList, emptyDataSources); + return; + } + + if (cancelled) { + return; + } + + // Add the alert.txt and users.txt to the case report progressMonitor.setProgressText(Bundle.AddLogicalImageTask_addingToReport(ALERT_TXT)); String status = addReport(Paths.get(dest.toString(), ALERT_TXT), ALERT_TXT + " " + src.getName()); if (status != null) { @@ -97,7 +125,50 @@ final class AddLogicalImageTask extends AddMultipleImageTask { } progressMonitor.setProgressText(Bundle.AddLogicalImageTask_doneAddingToReport(USERS_TXT)); - super.run(); + // Get all VHD files in the dest directory + List imagePaths = new ArrayList<>(); + for (File f : dest.listFiles()) { + if (f.getName().endsWith(".vhd")) { + try { + imagePaths.add(f.getCanonicalPath()); + } catch (IOException ex) { + String msg = Bundle.AddLogicalImageTask_failToGetCanonicalPath(f.getName()); + errorList.add(msg); + callback.done(DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS, errorList, emptyDataSources); + return; + } + } + } + if (imagePaths.isEmpty()) { + // No VHD in src directory, try ingest directories using Logical File Set + String[] directories = dest.list(new FilenameFilter() { + @Override + public boolean accept(File dir, String name) { + return Paths.get(dir.toString(), name).toFile().isDirectory(); + } + }); + for (String dir : directories) { + imagePaths.add(Paths.get(dest.toString(), dir).toFile().getAbsolutePath()); + } + if (imagePaths.isEmpty()) { + String msg = Bundle.AddLogicalImageTask_directoryDoesNotContainSparseImage(dest); + errorList.add(msg); + callback.done(DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS, errorList, emptyDataSources); + return; + } + + // ingest the directories + new Thread(new AddLocalFilesTask(deviceId, null, imagePaths, progressMonitor, callback)).start(); + } else { + // ingest the VHDs + try { + new Thread(new AddMultipleImageTask(deviceId, imagePaths, timeZone , progressMonitor, callback)).start(); + } catch (NoCurrentCaseException ex) { + String msg = Bundle.AddLogicalImageTask_noCurrentCase(); + errorList.add(msg); + callback.done(DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS, errorList, emptyDataSources); + } + } } /** @@ -121,8 +192,17 @@ final class AddLogicalImageTask extends AddMultipleImageTask { return null; } catch (TskCoreException ex) { String msg = Bundle.AddLogicalImageTask_failedToAddReport(reportPath.toString(), ex.getMessage()); - logger.log(Level.SEVERE, String.format("Failed to add report %s. Reason= %s", reportPath.toString(), ex.getMessage()), ex); + LOGGER.log(Level.SEVERE, String.format("Failed to add report %s. Reason= %s", reportPath.toString(), ex.getMessage()), ex); return msg; } } + + /** + * Attempts to cancel the processing of the input image files. May result in + * partial processing of the input. + */ + void cancelTask() { + LOGGER.log(Level.WARNING, "AddLogicalImageTask cancelled, processing may be incomplete"); // NON-NLS + cancelled = true; + } } diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/Bundle.properties-MERGED index 2551162ab2..d9c56237be 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/Bundle.properties-MERGED @@ -7,6 +7,8 @@ AddLogicalImageTask.addingToReport=Adding {0} to report # {0} - src # {1} - dest AddLogicalImageTask.copyingImageFromTo=Copying image from {0} to {1} +# {0} - sparseImageDirectory +AddLogicalImageTask.directoryDoesNotContainSparseImage=Directory {0} does not contain any images # {0} - file AddLogicalImageTask.doneAddingToReport=Done adding {0} to report AddLogicalImageTask.doneCopying=Done copying @@ -16,6 +18,10 @@ AddLogicalImageTask.failedToAddReport=Failed to add report {0}. Reason= {1} # {0} - src # {1} - dest AddLogicalImageTask.failedToCopyDirectory=Failed to copy directory {0} to {1} +# {0} - file +AddLogicalImageTask.failToGetCanonicalPath=Fail to get canonical path for {0} +AddLogicalImageTask.ingestionCancelled=Ingestion cancelled +AddLogicalImageTask.noCurrentCase=No current case # {0} - imageFilePath AddMultipleImageTask.adding=Adding: {0} # {0} - file @@ -39,12 +45,8 @@ AddMultipleImageTask.nonCriticalErrorAdding=Non-critical error adding {0} for de LogicalImagerDSProcessor.dataSourceType=Autopsy Logical Imager Results # {0} - directory LogicalImagerDSProcessor.directoryAlreadyExists=Directory {0} already exists -# {0} - sparseImageDirectory -LogicalImagerDSProcessor.directoryDoesNotContainSparseImage=Directory {0} does not contain any images # {0} - directory LogicalImagerDSProcessor.failToCreateDirectory=Failed to create directory {0} -# {0} - file -LogicalImagerDSProcessor.failToGetCanonicalPath=Fail to get canonical path for {0} # {0} - imageDirPath LogicalImagerDSProcessor.imageDirPathNotFound={0} not found.\nUSB drive has been ejected. LogicalImagerDSProcessor.noCurrentCase=No current case diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/LogicalImagerDSProcessor.java b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/LogicalImagerDSProcessor.java index ae0f3d533f..8a66270f55 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/LogicalImagerDSProcessor.java +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/LogicalImagerDSProcessor.java @@ -19,8 +19,6 @@ package org.sleuthkit.autopsy.logicalimager.dsp; import java.io.File; -import java.io.FilenameFilter; -import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; @@ -28,11 +26,9 @@ import java.util.Calendar; import java.util.List; import java.util.UUID; import javax.swing.JPanel; -import org.apache.commons.io.FileUtils; import org.openide.util.NbBundle.Messages; import org.openide.util.lookup.ServiceProvider; import org.openide.util.lookup.ServiceProviders; -import org.sleuthkit.autopsy.casemodule.AddLocalFilesTask; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessor; @@ -133,10 +129,7 @@ public final class LogicalImagerDSProcessor implements DataSourceProcessor { "# {0} - imageDirPath", "LogicalImagerDSProcessor.imageDirPathNotFound={0} not found.\nUSB drive has been ejected.", "# {0} - directory", "LogicalImagerDSProcessor.failToCreateDirectory=Failed to create directory {0}", "# {0} - directory", "LogicalImagerDSProcessor.directoryAlreadyExists=Directory {0} already exists", - "# {0} - file", "LogicalImagerDSProcessor.failToGetCanonicalPath=Fail to get canonical path for {0}", "LogicalImagerDSProcessor.noCurrentCase=No current case", - "# {0} - sparseImageDirectory", "LogicalImagerDSProcessor.directoryDoesNotContainSparseImage=Directory {0} does not contain any images", - }) @Override public void run(DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callback) { @@ -177,64 +170,14 @@ public final class LogicalImagerDSProcessor implements DataSourceProcessor { File src = imageDirPath.toFile(); try { - progressMonitor.setProgressText(Bundle.AddLogicalImageTask_copyingImageFromTo(src.toString(), dest.toString())); - FileUtils.copyDirectory(src, dest); - progressMonitor.setProgressText(Bundle.AddLogicalImageTask_doneCopying()); - } catch (IOException ex) { - // Copy directory failed - String msg = Bundle.AddLogicalImageTask_failedToCopyDirectory(src.toString(), dest.toString()); + String deviceId = UUID.randomUUID().toString(); + String timeZone = Calendar.getInstance().getTimeZone().getID(); + run(deviceId, timeZone, src, dest, + progressMonitor, callback); + } catch (NoCurrentCaseException ex) { + String msg = Bundle.LogicalImagerDSProcessor_noCurrentCase(); errorList.add(msg); callback.done(DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS, errorList, emptyDataSources); - return; - } - - // Get all VHD files in the src directory - List imagePaths = new ArrayList<>(); - for (File f : dest.listFiles()) { - if (f.getName().endsWith(".vhd")) { - try { - imagePaths.add(f.getCanonicalPath()); - } catch (IOException ex) { - String msg = Bundle.LogicalImagerDSProcessor_failToGetCanonicalPath(f.getName()); - errorList.add(msg); - callback.done(DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS, errorList, emptyDataSources); - return; - } - } - } - String deviceId = UUID.randomUUID().toString(); - if (imagePaths.isEmpty()) { - // No VHD in src directory, try ingest directories using Logical File Set - String[] directories = dest.list(new FilenameFilter() { - @Override - public boolean accept(File dir, String name) { - return Paths.get(dir.toString(), name).toFile().isDirectory(); - } - }); - for (String dir : directories) { - imagePaths.add(Paths.get(dest.toString(), dir).toFile().getAbsolutePath()); - } - if (imagePaths.isEmpty()) { - String msg = Bundle.LogicalImagerDSProcessor_directoryDoesNotContainSparseImage(dest); - errorList.add(msg); - callback.done(DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS, errorList, emptyDataSources); - return; - } - - // ingest the directories - new Thread(new AddLocalFilesTask(deviceId, null, imagePaths, progressMonitor, callback)).start(); - - } else { - try { - String timeZone = Calendar.getInstance().getTimeZone().getID(); - run(deviceId, imagePaths, - timeZone, src, dest, - progressMonitor, callback); - } catch (NoCurrentCaseException ex) { - String msg = Bundle.LogicalImagerDSProcessor_noCurrentCase(); - errorList.add(msg); - callback.done(DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS, errorList, emptyDataSources); - } } } @@ -248,7 +191,6 @@ public final class LogicalImagerDSProcessor implements DataSourceProcessor { * @param deviceId An ASCII-printable identifier for the device * associated with the data source that is intended * to be unique across multiple cases (e.g., a UUID). - * @param imagePaths Paths to the image files. * @param timeZone The time zone to use when processing dates and * times for the image, obtained from * java.util.TimeZone.getID. @@ -258,11 +200,11 @@ public final class LogicalImagerDSProcessor implements DataSourceProcessor { * processing. * @param callback Callback to call when processing is done. */ - private void run(String deviceId, List imagePaths, String timeZone, + private void run(String deviceId, String timeZone, File src, File dest, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callback ) throws NoCurrentCaseException { - addLogicalImageTask = new AddLogicalImageTask(deviceId, imagePaths, timeZone, src, dest, + addLogicalImageTask = new AddLogicalImageTask(deviceId, timeZone, src, dest, progressMonitor, callback); new Thread(addLogicalImageTask).start(); } From 12870f0fd13afc78a2095f17fd3e5268fa0b86af Mon Sep 17 00:00:00 2001 From: Joe Ho Date: Fri, 16 Aug 2019 00:09:50 -0400 Subject: [PATCH 018/102] Fix adding interesting files for non-VHD source --- .../dsp/AddLogicalImageTask.java | 133 +++++++++++++----- .../dsp/Bundle.properties-MERGED | 5 + 2 files changed, 103 insertions(+), 35 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java index 81c85ebf17..0ac6dfb4ce 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java @@ -20,8 +20,8 @@ package org.sleuthkit.autopsy.logicalimager.dsp; import java.io.BufferedReader; import java.io.File; -import java.io.FilenameFilter; import java.io.FileInputStream; +import java.io.FilenameFilter; import java.io.IOException; import java.io.InputStreamReader; import java.nio.file.Path; @@ -32,21 +32,22 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.logging.Level; +import org.apache.commons.io.FileUtils; import org.openide.util.NbBundle.Messages; -import org.sleuthkit.autopsy.casemodule.AddLocalFilesTask; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; -import org.sleuthkit.autopsy.casemodule.services.Blackboard; +import org.sleuthkit.autopsy.casemodule.services.FileManager; import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorCallback; import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorProgressMonitor; import org.sleuthkit.autopsy.coreutils.Logger; -import org.sleuthkit.autopsy.ingest.IngestServices; -import org.sleuthkit.autopsy.ingest.ModuleDataEvent; import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.datamodel.Blackboard; import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardAttribute; import org.sleuthkit.datamodel.Content; +import org.sleuthkit.datamodel.LocalFilesDataSource; import org.sleuthkit.datamodel.TskCoreException; +import org.sleuthkit.datamodel.TskDataException; /** * A runnable that - copy the logical image folder to a destination folder - add @@ -67,7 +68,7 @@ final class AddLogicalImageTask implements Runnable { private final DataSourceProcessorProgressMonitor progressMonitor; private final Blackboard blackboard; private final Case currentCase; - + private volatile boolean cancelled; AddLogicalImageTask(String deviceId, @@ -83,7 +84,7 @@ final class AddLogicalImageTask implements Runnable { this.progressMonitor = progressMonitor; this.callback = callback; this.currentCase = Case.getCurrentCase(); - this.blackboard = this.currentCase.getServices().getBlackboard(); + this.blackboard = this.currentCase.getServices().getArtifactsBlackboard(); } /** @@ -118,6 +119,7 @@ final class AddLogicalImageTask implements Runnable { // Copy directory failed String msg = Bundle.AddLogicalImageTask_failedToCopyDirectory(src.toString(), dest.toString()); errorList.add(msg); + } // Add the SearchResults.txt and users.txt to the case report String resultsFilename; @@ -157,7 +159,7 @@ final class AddLogicalImageTask implements Runnable { if (f.getName().endsWith(".vhd")) { try { imagePaths.add(f.getCanonicalPath()); - } catch (IOException ex) { + } catch (IOException ioe) { String msg = Bundle.AddLogicalImageTask_failToGetCanonicalPath(f.getName()); errorList.add(msg); callback.done(DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS, errorList, emptyDataSources); @@ -165,7 +167,13 @@ final class AddLogicalImageTask implements Runnable { } } } + + AddMultipleImageTask addMultipleImageTask = null; + List newDataSources = new ArrayList<>(); + boolean createVHD; + if (imagePaths.isEmpty()) { + createVHD = false; // No VHD in src directory, try ingest directories using Logical File Set String[] directories = dest.list(new FilenameFilter() { @Override @@ -173,9 +181,9 @@ final class AddLogicalImageTask implements Runnable { return Paths.get(dir.toString(), name).toFile().isDirectory(); } }); - for (String dir : directories) { - imagePaths.add(Paths.get(dest.toString(), dir).toFile().getAbsolutePath()); - } +// for (String dir : directories) { + imagePaths.add(Paths.get(dest.toString(), "root").toFile().getAbsolutePath()); +// } if (imagePaths.isEmpty()) { String msg = Bundle.AddLogicalImageTask_directoryDoesNotContainSparseImage(dest); errorList.add(msg); @@ -184,11 +192,29 @@ final class AddLogicalImageTask implements Runnable { } // ingest the directories - new Thread(new AddLocalFilesTask(deviceId, null, imagePaths, progressMonitor, callback)).start(); + FileManager fileManager = Case.getCurrentCase().getServices().getFileManager(); + try { + LocalFilesDataSource newDataSource = fileManager.addLocalFilesDataSource(deviceId, "", "", imagePaths, new ProgressUpdater()); + newDataSources.add(newDataSource); + } catch (TskCoreException | TskDataException ex) { + errorList.add(ex.getMessage()); + LOGGER.log(Level.SEVERE, String.format("Failed to add datasource: %s", ex.getMessage()), ex); // NON-NLS + callback.done(DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS, errorList, emptyDataSources); + } + } else { + createVHD = true; + // ingest the VHDs try { - new Thread(new AddMultipleImageTask(deviceId, imagePaths, timeZone , progressMonitor, callback)).start(); + addMultipleImageTask = new AddMultipleImageTask(deviceId, imagePaths, timeZone , progressMonitor, callback); + addMultipleImageTask.run(); + + if (addMultipleImageTask.getResult() == DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS) { + callback.done(addMultipleImageTask.getResult(), addMultipleImageTask.getErrorMessages(), addMultipleImageTask.getNewDataSources()); + return; + } + } catch (NoCurrentCaseException ex) { String msg = Bundle.AddLogicalImageTask_noCurrentCase(); errorList.add(msg); @@ -196,16 +222,15 @@ final class AddLogicalImageTask implements Runnable { } } - if (super.getResult() == DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS) { - callback.done(super.getResult(), super.getErrorMessages(), super.getNewDataSources()); - return; - } - try { progressMonitor.setProgressText(Bundle.AddLogicalImageTask_addingInterestingFiles()); - addInterestingFiles(dest, Paths.get(dest.toString(), resultsFilename)); + addInterestingFiles(dest, Paths.get(dest.toString(), resultsFilename), createVHD); progressMonitor.setProgressText(Bundle.AddLogicalImageTask_doneAddingInterestingFiles()); - callback.done(super.getResult(), super.getErrorMessages(), super.getNewDataSources()); + if (addMultipleImageTask != null) { + callback.done(addMultipleImageTask.getResult(), addMultipleImageTask.getErrorMessages(), addMultipleImageTask.getNewDataSources()); + } else { + callback.done(DataSourceProcessorCallback.DataSourceProcessorResult.NO_ERRORS, errorList, newDataSources); + } } catch (IOException | TskCoreException ex) { errorList.add(Bundle.AddLogicalImageTask_failedToAddInterestingFiles(ex.getMessage())); LOGGER.log(Level.SEVERE, "Failed to add interesting files", ex); // NON-NLS @@ -264,7 +289,7 @@ final class AddLogicalImageTask implements Runnable { "# {0} - line number", "# {1} - fields length", "# {2} - expected length", "AddLogicalImageTask.notEnoughFields=File does not contain enough fields at line {0}, got {1}, expecting {2}", "# {0} - target image path", "AddLogicalImageTask.cannotFindDataSourceObjId=Cannot find obj_id in tsk_image_names for {0}" }) - private void addInterestingFiles(File src, Path resultsPath) throws IOException, TskCoreException { + private void addInterestingFiles(File src, Path resultsPath, boolean createVHD) throws IOException, TskCoreException { Map> imagePaths = currentCase.getSleuthkitCase().getImagePaths(); Map imagePathToObjIdMap = imagePathsToDataSourceObjId(imagePaths); @@ -279,13 +304,6 @@ final class AddLogicalImageTask implements Runnable { throw new IOException(Bundle.AddLogicalImageTask_notEnoughFields(lineNumber, fields.length, 9)); } String vhdFilename = fields[0]; - - String targetImagePath = Paths.get(src.toString(), vhdFilename).toString(); - Long dataSourceObjId = imagePathToObjIdMap.get(targetImagePath); - if (dataSourceObjId == null) { - throw new TskCoreException(Bundle.AddLogicalImageTask_cannotFindDataSourceObjId(targetImagePath)); - } - // String fileSystemOffsetStr = fields[1]; String fileMetaAddressStr = fields[2]; // String extractStatusStr = fields[3]; @@ -293,10 +311,26 @@ final class AddLogicalImageTask implements Runnable { String ruleName = fields[5]; // String description = fields[6]; String filename = fields[7]; -// String parentPath = fields[8]; - String query = String.format("data_source_obj_id = '%s' AND meta_addr = '%s' AND name = '%s'", // NON-NLS - dataSourceObjId.toString(), fileMetaAddressStr, filename); + String query; + String targetImagePath; + if (createVHD) { + targetImagePath = Paths.get(src.toString(), vhdFilename).toString(); + Long dataSourceObjId = imagePathToObjIdMap.get(targetImagePath); + if (dataSourceObjId == null) { + throw new TskCoreException(Bundle.AddLogicalImageTask_cannotFindDataSourceObjId(targetImagePath)); + } + query = String.format("data_source_obj_id = '%s' AND meta_addr = '%s' AND name = '%s'", // NON-NLS + dataSourceObjId.toString(), fileMetaAddressStr, filename.replace("'", "''")); + } else { + String parentPath = fields[8]; + targetImagePath = Paths.get("root", vhdFilename).toString(); + String tmpRootPath = targetImagePath.replace(".vhd", "").replace("\\", "/"); + String searchParentPath = "/" + tmpRootPath + "/" + parentPath; + query = String.format("name = '%s' AND parent_path = '%s'", // NON-NLS + filename.replace("'", "''"), searchParentPath.replace("'", "''")); + } + List matchedFiles = Case.getCurrentCase().getSleuthkitCase().findAllFilesWhere(query); for (AbstractFile file : matchedFiles) { addInterestingFile(file, ruleSetName, ruleName); @@ -304,8 +338,8 @@ final class AddLogicalImageTask implements Runnable { lineNumber++; } } - IngestServices.getInstance().fireModuleDataEvent(new ModuleDataEvent(MODULE_NAME, - BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT)); +// IngestServices.getInstance().fireModuleDataEvent(new ModuleDataEvent(MODULE_NAME, +// BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT)); } private void addInterestingFile(AbstractFile file, String ruleSetName, String ruleName) throws TskCoreException { @@ -314,16 +348,45 @@ final class AddLogicalImageTask implements Runnable { attributes.add(setNameAttribute); BlackboardAttribute ruleNameAttribute = new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_CATEGORY, MODULE_NAME, ruleName); attributes.add(ruleNameAttribute); - org.sleuthkit.datamodel.Blackboard tskBlackboard = Case.getCurrentCase().getSleuthkitCase().getBlackboard(); + Blackboard tskBlackboard = Case.getCurrentCase().getSleuthkitCase().getBlackboard(); if (!tskBlackboard.artifactExists(file, BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT, attributes)) { BlackboardArtifact artifact = file.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT); artifact.addAttributes(attributes); try { // index the artifact for keyword search - blackboard.indexArtifact(artifact); + blackboard.postArtifact(artifact, MODULE_NAME); } catch (Blackboard.BlackboardException ex) { LOGGER.log(Level.SEVERE, "Unable to index blackboard artifact " + artifact.getArtifactID(), ex); //NON-NLS } } } + + /** + * Updates task progress as the file manager adds the local/logical files + * and/or directories to the case database. + */ + @Messages({ + "# {0} - parent path", "# {1} - filename", "AddLogicalImageTask.localFileAddProgress=Adding: {0}/{1}", + }) + private class ProgressUpdater implements FileManager.FileAddProgressUpdater { + + private int count; + + /** + * Updates task progress (called by the file manager after it adds each + * local file/directory to the case database). + */ + @Override + public void fileAdded(final AbstractFile file) { + ++count; + if (count % 10 == 0) { + progressMonitor.setProgressText( + Bundle.AddLogicalImageTask_localFileAddProgress( + file.getParentPath(), + file.getName() + ) + ); + } + } + } } diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/Bundle.properties-MERGED index a25babfef6..3cecb6f2c7 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/Bundle.properties-MERGED @@ -5,6 +5,8 @@ AddLogicalImageTask.addingInterestingFiles=Adding search results as interesting files # {0} - file AddLogicalImageTask.addingToReport=Adding {0} to report +# {0} - target image path +AddLogicalImageTask.cannotFindDataSourceObjId=Cannot find obj_id in tsk_image_names for {0} # {0} - SearchResults.txt # {1} - directory AddLogicalImageTask.cannotFindFiles=Cannot find {0} in {1} @@ -28,6 +30,9 @@ AddLogicalImageTask.failedToCopyDirectory=Failed to copy directory {0} to {1} # {0} - file AddLogicalImageTask.failToGetCanonicalPath=Fail to get canonical path for {0} AddLogicalImageTask.ingestionCancelled=Ingestion cancelled +# {0} - parent path +# {1} - filename +AddLogicalImageTask.localFileAddProgress=Adding: {0}/{1} AddLogicalImageTask.noCurrentCase=No current case # {0} - line number # {1} - fields length From 0e45fb99a8ebe66fdd4f805d248bd5824ab6abdc Mon Sep 17 00:00:00 2001 From: Joe Ho Date: Fri, 16 Aug 2019 11:35:33 -0400 Subject: [PATCH 019/102] Fix blackboard code --- .../dsp/AddLogicalImageTask.java | 52 ++++++++----------- 1 file changed, 22 insertions(+), 30 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java index 0ac6dfb4ce..843315dbec 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java @@ -21,7 +21,6 @@ package org.sleuthkit.autopsy.logicalimager.dsp; import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; -import java.io.FilenameFilter; import java.io.IOException; import java.io.InputStreamReader; import java.nio.file.Path; @@ -60,6 +59,7 @@ final class AddLogicalImageTask implements Runnable { private final static String SEARCH_RESULTS_TXT = "SearchResults.txt"; //NON-NLS private final static String USERS_TXT = "users.txt"; //NON-NLS private final static String MODULE_NAME = "Logical Imager"; //NON-NLS + private final static String ROOT_STR = "root"; // NON-NLS private final String deviceId; private final String timeZone; private final File src; @@ -174,24 +174,18 @@ final class AddLogicalImageTask implements Runnable { if (imagePaths.isEmpty()) { createVHD = false; - // No VHD in src directory, try ingest directories using Logical File Set - String[] directories = dest.list(new FilenameFilter() { - @Override - public boolean accept(File dir, String name) { - return Paths.get(dir.toString(), name).toFile().isDirectory(); - } - }); -// for (String dir : directories) { - imagePaths.add(Paths.get(dest.toString(), "root").toFile().getAbsolutePath()); -// } - if (imagePaths.isEmpty()) { + // No VHD in src directory, try ingest the root directory using Logical File Set + File root = Paths.get(dest.toString(), ROOT_STR).toFile(); + if (root.exists() && root.isDirectory()) { + imagePaths.add(root.getAbsolutePath()); + } else { String msg = Bundle.AddLogicalImageTask_directoryDoesNotContainSparseImage(dest); errorList.add(msg); callback.done(DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS, errorList, emptyDataSources); return; } - // ingest the directories + // ingest the root directory FileManager fileManager = Case.getCurrentCase().getServices().getFileManager(); try { LocalFilesDataSource newDataSource = fileManager.addLocalFilesDataSource(deviceId, "", "", imagePaths, new ProgressUpdater()); @@ -204,17 +198,14 @@ final class AddLogicalImageTask implements Runnable { } else { createVHD = true; - // ingest the VHDs try { addMultipleImageTask = new AddMultipleImageTask(deviceId, imagePaths, timeZone , progressMonitor, callback); addMultipleImageTask.run(); - if (addMultipleImageTask.getResult() == DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS) { callback.done(addMultipleImageTask.getResult(), addMultipleImageTask.getErrorMessages(), addMultipleImageTask.getNewDataSources()); return; } - } catch (NoCurrentCaseException ex) { String msg = Bundle.AddLogicalImageTask_noCurrentCase(); errorList.add(msg); @@ -295,6 +286,7 @@ final class AddLogicalImageTask implements Runnable { try (BufferedReader br = new BufferedReader(new InputStreamReader( new FileInputStream(resultsPath.toFile()), "UTF8"))) { // NON-NLS + List artifacts = new ArrayList<>(); String line; br.readLine(); // skip the header line int lineNumber = 2; @@ -324,40 +316,40 @@ final class AddLogicalImageTask implements Runnable { dataSourceObjId.toString(), fileMetaAddressStr, filename.replace("'", "''")); } else { String parentPath = fields[8]; - targetImagePath = Paths.get("root", vhdFilename).toString(); + targetImagePath = Paths.get(ROOT_STR, vhdFilename).toString(); String tmpRootPath = targetImagePath.replace(".vhd", "").replace("\\", "/"); String searchParentPath = "/" + tmpRootPath + "/" + parentPath; - query = String.format("name = '%s' AND parent_path = '%s'", // NON-NLS + query = String.format("name = '%s' AND parent_path = '%s'", // NON-NLS filename.replace("'", "''"), searchParentPath.replace("'", "''")); } + // TODO - findAllFilesWhere should SQL-escape the query List matchedFiles = Case.getCurrentCase().getSleuthkitCase().findAllFilesWhere(query); for (AbstractFile file : matchedFiles) { - addInterestingFile(file, ruleSetName, ruleName); + addInterestingFileToArtifacts(file, ruleSetName, ruleName, artifacts); } lineNumber++; + } // end reading file + + try { + // index the artifact for keyword search + blackboard.postArtifacts(artifacts, MODULE_NAME); + } catch (Blackboard.BlackboardException ex) { + LOGGER.log(Level.SEVERE, "Unable to post artifacts to blackboard", ex); //NON-NLS } } -// IngestServices.getInstance().fireModuleDataEvent(new ModuleDataEvent(MODULE_NAME, -// BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT)); } - private void addInterestingFile(AbstractFile file, String ruleSetName, String ruleName) throws TskCoreException { + private void addInterestingFileToArtifacts(AbstractFile file, String ruleSetName, String ruleName, List artifacts) throws TskCoreException { Collection attributes = new ArrayList<>(); BlackboardAttribute setNameAttribute = new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME, MODULE_NAME, ruleSetName); attributes.add(setNameAttribute); BlackboardAttribute ruleNameAttribute = new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_CATEGORY, MODULE_NAME, ruleName); attributes.add(ruleNameAttribute); - Blackboard tskBlackboard = Case.getCurrentCase().getSleuthkitCase().getBlackboard(); - if (!tskBlackboard.artifactExists(file, BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT, attributes)) { + if (!blackboard.artifactExists(file, BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT, attributes)) { BlackboardArtifact artifact = file.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT); artifact.addAttributes(attributes); - try { - // index the artifact for keyword search - blackboard.postArtifact(artifact, MODULE_NAME); - } catch (Blackboard.BlackboardException ex) { - LOGGER.log(Level.SEVERE, "Unable to index blackboard artifact " + artifact.getArtifactID(), ex); //NON-NLS - } + artifacts.add(artifact); } } From 0b09d8ad3b23601a61488a9de542ceac17d5e455 Mon Sep 17 00:00:00 2001 From: Joe Ho Date: Fri, 16 Aug 2019 11:37:35 -0400 Subject: [PATCH 020/102] Fix VHD extension --- .../autopsy/logicalimager/dsp/AddLogicalImageTask.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java index 843315dbec..89887d7999 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java @@ -60,6 +60,7 @@ final class AddLogicalImageTask implements Runnable { private final static String USERS_TXT = "users.txt"; //NON-NLS private final static String MODULE_NAME = "Logical Imager"; //NON-NLS private final static String ROOT_STR = "root"; // NON-NLS + private final static String VHD_EXTENSION = ".vhd"; // NON-NLS private final String deviceId; private final String timeZone; private final File src; @@ -156,7 +157,7 @@ final class AddLogicalImageTask implements Runnable { // Get all VHD files in the dest directory List imagePaths = new ArrayList<>(); for (File f : dest.listFiles()) { - if (f.getName().endsWith(".vhd")) { + if (f.getName().endsWith(VHD_EXTENSION)) { try { imagePaths.add(f.getCanonicalPath()); } catch (IOException ioe) { @@ -317,7 +318,8 @@ final class AddLogicalImageTask implements Runnable { } else { String parentPath = fields[8]; targetImagePath = Paths.get(ROOT_STR, vhdFilename).toString(); - String tmpRootPath = targetImagePath.replace(".vhd", "").replace("\\", "/"); + // vhdFilename have .vhd extension, we don't + String tmpRootPath = targetImagePath.replace("\\", "/"); String searchParentPath = "/" + tmpRootPath + "/" + parentPath; query = String.format("name = '%s' AND parent_path = '%s'", // NON-NLS filename.replace("'", "''"), searchParentPath.replace("'", "''")); From 1bd04489db80a60e3206d7863251953b3b9dd245 Mon Sep 17 00:00:00 2001 From: Joe Ho Date: Mon, 19 Aug 2019 09:48:31 -0400 Subject: [PATCH 021/102] check directory for either VHD or subdir --- .../autopsy/logicalimager/dsp/LogicalImagerPanel.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/LogicalImagerPanel.java b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/LogicalImagerPanel.java index d491c77778..1cdf09286b 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/LogicalImagerPanel.java +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/LogicalImagerPanel.java @@ -370,11 +370,11 @@ final class LogicalImagerPanel extends JPanel implements DocumentListener { } } - private boolean dirHasVhdFiles(File dir) { - File[] fList = dir.listFiles(new FilenameFilter() { + private boolean dirHasImagerResult(File dir) { + String[] fList = dir.list(new FilenameFilter() { @Override public boolean accept(File dir, String name) { - return name.endsWith(".vhd"); + return name.endsWith(".vhd") || Paths.get(dir.toString(), name).toFile().isDirectory(); } }); return (fList != null && fList.length != 0); @@ -392,9 +392,9 @@ final class LogicalImagerPanel extends JPanel implements DocumentListener { if (fList != null) { imageTableModel = new ImageTableModel(); // Find all directories with name like Logical_Imager_HOSTNAME_yyyymmdd_HH_MM_SS - // and has vhd files in it + // and has Logical Imager result in it for (File file : fList) { - if (file.isDirectory() && dirHasVhdFiles(file)) { + if (file.isDirectory() && dirHasImagerResult(file)) { String dir = file.getName(); Matcher m = regex.matcher(dir); if (m.find()) { From 1944eb672e1d8f60c8d2414ab97c93cebbe257f1 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Tue, 20 Aug 2019 17:00:00 -0400 Subject: [PATCH 022/102] Removed the location column filter for FS files and reordered the location property in the AAFN --- .../sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java | 2 +- .../sleuthkit/autopsy/directorytree/DataResultFilterNode.java | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java index eeacf491bf..145906f785 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java @@ -336,12 +336,12 @@ public abstract class AbstractAbstractFileNode extends A backgroundTasksPool.submit(new GetSCOTask( new WeakReference<>(this), weakPcl)); - properties.add(new NodeProperty<>(LOCATION.toString(), LOCATION.toString(), NO_DESCR, getContentPath(content))); properties.add(new NodeProperty<>(MOD_TIME.toString(), MOD_TIME.toString(), NO_DESCR, ContentUtils.getStringTime(content.getMtime(), content))); properties.add(new NodeProperty<>(CHANGED_TIME.toString(), CHANGED_TIME.toString(), NO_DESCR, ContentUtils.getStringTime(content.getCtime(), content))); properties.add(new NodeProperty<>(ACCESS_TIME.toString(), ACCESS_TIME.toString(), NO_DESCR, ContentUtils.getStringTime(content.getAtime(), content))); properties.add(new NodeProperty<>(CREATED_TIME.toString(), CREATED_TIME.toString(), NO_DESCR, ContentUtils.getStringTime(content.getCrtime(), content))); properties.add(new NodeProperty<>(SIZE.toString(), SIZE.toString(), NO_DESCR, content.getSize())); + properties.add(new NodeProperty<>(LOCATION.toString(), LOCATION.toString(), NO_DESCR, getContentPath(content))); properties.add(new NodeProperty<>(FLAGS_DIR.toString(), FLAGS_DIR.toString(), NO_DESCR, content.getDirFlagAsString())); properties.add(new NodeProperty<>(FLAGS_META.toString(), FLAGS_META.toString(), NO_DESCR, content.getMetaFlagsAsString())); properties.add(new NodeProperty<>(KNOWN.toString(), KNOWN.toString(), NO_DESCR, content.getKnown().getName())); diff --git a/Core/src/org/sleuthkit/autopsy/directorytree/DataResultFilterNode.java b/Core/src/org/sleuthkit/autopsy/directorytree/DataResultFilterNode.java index 3c206049d6..14aceaf14f 100644 --- a/Core/src/org/sleuthkit/autopsy/directorytree/DataResultFilterNode.java +++ b/Core/src/org/sleuthkit/autopsy/directorytree/DataResultFilterNode.java @@ -179,9 +179,7 @@ public class DataResultFilterNode extends FilterNode { newPs.setShortDescription(ps.getShortDescription()); newPs.put(ps.getProperties()); - if (newPs.remove(AbstractFsContentNode.HIDE_PARENT) != null) { - newPs.remove(AbstractFilePropertyType.LOCATION.toString()); - } + newPs.remove(AbstractFsContentNode.HIDE_PARENT); propertySets[i] = newPs; } } From b79b5c5b7774c2dcd180f2c0f121d5540ab5ddd6 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Tue, 20 Aug 2019 17:02:26 -0400 Subject: [PATCH 023/102] Looks better before hash :) --- .../sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java index 145906f785..e02a4ff776 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java @@ -341,10 +341,10 @@ public abstract class AbstractAbstractFileNode extends A properties.add(new NodeProperty<>(ACCESS_TIME.toString(), ACCESS_TIME.toString(), NO_DESCR, ContentUtils.getStringTime(content.getAtime(), content))); properties.add(new NodeProperty<>(CREATED_TIME.toString(), CREATED_TIME.toString(), NO_DESCR, ContentUtils.getStringTime(content.getCrtime(), content))); properties.add(new NodeProperty<>(SIZE.toString(), SIZE.toString(), NO_DESCR, content.getSize())); - properties.add(new NodeProperty<>(LOCATION.toString(), LOCATION.toString(), NO_DESCR, getContentPath(content))); properties.add(new NodeProperty<>(FLAGS_DIR.toString(), FLAGS_DIR.toString(), NO_DESCR, content.getDirFlagAsString())); properties.add(new NodeProperty<>(FLAGS_META.toString(), FLAGS_META.toString(), NO_DESCR, content.getMetaFlagsAsString())); properties.add(new NodeProperty<>(KNOWN.toString(), KNOWN.toString(), NO_DESCR, content.getKnown().getName())); + properties.add(new NodeProperty<>(LOCATION.toString(), LOCATION.toString(), NO_DESCR, getContentPath(content))); properties.add(new NodeProperty<>(MD5HASH.toString(), MD5HASH.toString(), NO_DESCR, StringUtils.defaultString(content.getMd5Hash()))); properties.add(new NodeProperty<>(MIMETYPE.toString(), MIMETYPE.toString(), NO_DESCR, StringUtils.defaultString(content.getMIMEType()))); properties.add(new NodeProperty<>(EXTENSION.toString(), EXTENSION.toString(), NO_DESCR, content.getNameExtension())); From 54205589bbc27be31388e9df92aaae8dc3e6c2c2 Mon Sep 17 00:00:00 2001 From: Joe Ho Date: Wed, 21 Aug 2019 15:51:23 -0400 Subject: [PATCH 024/102] Use LocalFileImporter class --- .../datamodel/utils/LocalFileImporter.java | 209 ++++++++++++++++++ .../dsp/AddLogicalImageTask.java | 89 ++++++-- 2 files changed, 283 insertions(+), 15 deletions(-) create mode 100644 Core/src/org/sleuthkit/autopsy/datamodel/utils/LocalFileImporter.java diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/utils/LocalFileImporter.java b/Core/src/org/sleuthkit/autopsy/datamodel/utils/LocalFileImporter.java new file mode 100644 index 0000000000..155f167953 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/datamodel/utils/LocalFileImporter.java @@ -0,0 +1,209 @@ +/* + * + * Autopsy Forensic Browser + * + * Copyright 2019 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.datamodel.utils; + +import java.io.File; +import java.util.HashMap; +import java.util.Map; +import java.util.logging.Level; +import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.datamodel.Content; +import org.sleuthkit.datamodel.DataSource; +import org.sleuthkit.datamodel.SleuthkitCase; +import org.sleuthkit.datamodel.SpecialDirectory; +import org.sleuthkit.datamodel.TskCoreException; +import org.sleuthkit.datamodel.TskData; + +/** + * Utility class for adding local files with specified paths in the data source. + * It is currently assumed that the data source is empty to start or that at + * least the paths to the files being added do not exist; no checks will be done + * to see if folders exist prior to creating them through addLocalFile(). + */ +public class LocalFileImporter { + private static final Logger logger = Logger.getLogger(LocalFileImporter.class.getName()); + + SleuthkitCase.CaseDbTransaction globalTrans = null; + boolean useSingleTransaction = true; + SleuthkitCase sleuthkitCase; + private final Map localFileDirMap = new HashMap<>(); + + /** + * Create a LocalFileImporter. + * + * @param sleuthkitCase The current SleuthkitCase + */ + public LocalFileImporter(SleuthkitCase sleuthkitCase) { + this.sleuthkitCase = sleuthkitCase; + this.useSingleTransaction = false; + } + + /** + * Create a LocalFileImporter. The caller is responsible for committing + * or rolling back the transaction. + * + * @param sleuthkitCase The current SleuthkitCase + * @param trans The open CaseDbTransaction + */ + public LocalFileImporter(SleuthkitCase sleuthkitCase, SleuthkitCase.CaseDbTransaction trans) { + this.sleuthkitCase = sleuthkitCase; + this.globalTrans = trans; + this.useSingleTransaction = true; + } + + /** + * Add a local file to the database with the specified parameters. Will create + * any necessary parent folders. + * + * Will not fail if the fileOnDisk does not exist. + * + * @param fileOnDisk The local file on disk + * @param nameInImage The name to use in the data source + * @param pathInImage The path to use in the data source + * @param ctime Change time + * @param crtime Created time + * @param atime Access time + * @param mtime Modified time + * @param dataSource The data source to add the file to + * + * @return The AbstractFile that was just created + * + * @throws TskCoreException + */ + public AbstractFile addLocalFile(File fileOnDisk, String nameInImage, String pathInImage, + Long ctime, Long crtime, Long atime, Long mtime, + DataSource dataSource) throws TskCoreException { + + // Get the parent folder, creating it and any of its parent folders if necessary + SpecialDirectory parentDir = getLocalFilesDir(new File(pathInImage), dataSource); + + SleuthkitCase.CaseDbTransaction trans = null; + try { + if (useSingleTransaction) { + trans = globalTrans; + } else { + trans = sleuthkitCase.beginTransaction(); + } + + // Try to get the file size + long size = 0; + if (fileOnDisk.exists()) { + size = fileOnDisk.length(); + } + + // Create the new file + AbstractFile file = sleuthkitCase.addLocalFile(nameInImage, fileOnDisk.getAbsolutePath(), size, + ctime, crtime, atime, mtime, + true, TskData.EncodingType.NONE, parentDir, trans); + + if (! useSingleTransaction) { + trans.commit(); + } + return file; + } catch (TskCoreException ex) { + if ((!useSingleTransaction) && (null != trans)) { + try { + trans.rollback(); + } catch (TskCoreException ex2) { + logger.log(Level.SEVERE, String.format("Failed to rollback transaction after exception: %s", ex.getMessage()), ex2); + } + } + throw ex; + } + } + + /** + * Returns the SpecialDirectory object corresponding to the given file, creating + * it and its parents as needed. + * + * @param file The file to get the SpecialDirectory for + * @param dataSource The data source + * + * @return The SpecialDirectory object corresponding to the given file + * + * @throws TskCoreException + */ + private SpecialDirectory getLocalFilesDir(File file, Content dataSource) throws TskCoreException { + if ((file == null) || file.getPath().isEmpty()) { + throw new TskCoreException("Can not create directory from null path"); + } + + // Check if we've already created it + if (localFileDirMap.containsKey(file.toString())) { + return localFileDirMap.get(file.toString()); + } + + File parent = file.getParentFile(); + if (parent == null) { + // This is the root of the path and it isn't in the map, so create it + SpecialDirectory dir = createLocalFilesDir(dataSource.getId(), file.getName()); + localFileDirMap.put(file.getName(), dir); + return dir; + + } else { + // Create everything above this in the tree, and then add the parent folder + SpecialDirectory parentDir = getLocalFilesDir(parent, dataSource); + SpecialDirectory dir = createLocalFilesDir(parentDir.getId(), file.getName()); + localFileDirMap.put(file.getPath(), dir); + return dir; + } + } + + /** + * Create a new LocalDirectory + * + * @param parentId The object ID for parent + * @param name The name of the new local directory + * + * @return The new LocalDirectory + * + * @throws TskCoreException + */ + private SpecialDirectory createLocalFilesDir(long parentId, String name) throws TskCoreException { + SleuthkitCase.CaseDbTransaction trans = null; + + try { + if (useSingleTransaction) { + trans = globalTrans; + } else { + trans = sleuthkitCase.beginTransaction(); + } + SpecialDirectory dir; + + dir = sleuthkitCase.addLocalDirectory(parentId, name, trans); + + if (! useSingleTransaction) { + trans.commit(); + } + return dir; + } catch (TskCoreException ex) { + if (( !useSingleTransaction) && (null != trans)) { + try { + trans.rollback(); + } catch (TskCoreException ex2) { + logger.log(Level.SEVERE, String.format("Failed to rollback transaction after exception: %s", ex.getMessage()), ex2); + } + } + throw ex; + } + } + +} \ No newline at end of file diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java index 89887d7999..fc2e93697b 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java @@ -23,6 +23,7 @@ import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStreamReader; +import java.io.UnsupportedEncodingException; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; @@ -39,14 +40,15 @@ import org.sleuthkit.autopsy.casemodule.services.FileManager; import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorCallback; import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorProgressMonitor; import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.autopsy.datamodel.utils.LocalFileImporter; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.Blackboard; import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardAttribute; import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.LocalFilesDataSource; +import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.TskCoreException; -import org.sleuthkit.datamodel.TskDataException; /** * A runnable that - copy the logical image folder to a destination folder - add @@ -186,17 +188,13 @@ final class AddLogicalImageTask implements Runnable { return; } - // ingest the root directory - FileManager fileManager = Case.getCurrentCase().getServices().getFileManager(); try { - LocalFilesDataSource newDataSource = fileManager.addLocalFilesDataSource(deviceId, "", "", imagePaths, new ProgressUpdater()); - newDataSources.add(newDataSource); - } catch (TskCoreException | TskDataException ex) { + addExtractedFiles(dest, Paths.get(dest.toString(), resultsFilename), newDataSources); + } catch (TskCoreException | IOException ex) { errorList.add(ex.getMessage()); LOGGER.log(Level.SEVERE, String.format("Failed to add datasource: %s", ex.getMessage()), ex); // NON-NLS callback.done(DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS, errorList, emptyDataSources); } - } else { createVHD = true; // ingest the VHDs @@ -293,9 +291,9 @@ final class AddLogicalImageTask implements Runnable { int lineNumber = 2; while ((line = br.readLine()) != null) { String[] fields = line.split("\t", -1); // NON-NLS - if (fields.length != 9) { - throw new IOException(Bundle.AddLogicalImageTask_notEnoughFields(lineNumber, fields.length, 9)); - } +// if (fields.length != 9) { +// throw new IOException(Bundle.AddLogicalImageTask_notEnoughFields(lineNumber, fields.length, 9)); +// } String vhdFilename = fields[0]; // String fileSystemOffsetStr = fields[1]; String fileMetaAddressStr = fields[2]; @@ -317,12 +315,9 @@ final class AddLogicalImageTask implements Runnable { dataSourceObjId.toString(), fileMetaAddressStr, filename.replace("'", "''")); } else { String parentPath = fields[8]; - targetImagePath = Paths.get(ROOT_STR, vhdFilename).toString(); - // vhdFilename have .vhd extension, we don't - String tmpRootPath = targetImagePath.replace("\\", "/"); - String searchParentPath = "/" + tmpRootPath + "/" + parentPath; + parentPath = "/" + ROOT_STR + "/" + vhdFilename + "/" + parentPath; query = String.format("name = '%s' AND parent_path = '%s'", // NON-NLS - filename.replace("'", "''"), searchParentPath.replace("'", "''")); + filename.replace("'", "''"), parentPath.replace("'", "''")); } // TODO - findAllFilesWhere should SQL-escape the query @@ -355,6 +350,70 @@ final class AddLogicalImageTask implements Runnable { } } + private void addExtractedFiles(File src, Path resultsPath, List newDataSources) throws TskCoreException, UnsupportedEncodingException, IOException { + + SleuthkitCase skCase = Case.getCurrentCase().getSleuthkitCase(); + SleuthkitCase.CaseDbTransaction trans = null; + try { + trans = skCase.beginTransaction(); + LocalFilesDataSource localFilesDataSource = skCase.addLocalFilesDataSource(deviceId, this.src.getName(), timeZone, trans); + LocalFileImporter fileImporter = new LocalFileImporter(skCase, trans); + + try (BufferedReader br = new BufferedReader(new InputStreamReader( + new FileInputStream(resultsPath.toFile()), "UTF8"))) { // NON-NLS + String line; + br.readLine(); // skip the header line + int lineNumber = 2; + while ((line = br.readLine()) != null) { + String[] fields = line.split("\t", -1); // NON-NLS + if (fields.length != 14) { + throw new IOException(Bundle.AddLogicalImageTask_notEnoughFields(lineNumber, fields.length, 14)); + } + String vhdFilename = fields[0]; +// String fileSystemOffsetStr = fields[1]; + String fileMetaAddressStr = fields[2]; +// String extractStatusStr = fields[3]; + String ruleSetName = fields[4]; + String ruleName = fields[5]; +// String description = fields[6]; + String filename = fields[7]; + String parentPath = fields[8]; + String extractedFilePath = fields[9]; + String crtime = fields[10]; + String mtime = fields[11]; + String atime = fields[12]; + String ctime = fields[13]; + parentPath = ROOT_STR + "/" + vhdFilename + "/" + parentPath; + + //addLocalFile here + AbstractFile localFile = fileImporter.addLocalFile( + Paths.get(src.toString(), extractedFilePath).toFile(), + filename, + parentPath, + Long.parseLong(ctime), + Long.parseLong(crtime), + Long.parseLong(atime), + Long.parseLong(mtime), + localFilesDataSource); + + lineNumber++; + } // end reading file + + trans.commit(); + newDataSources.add(localFilesDataSource); + } + } catch (TskCoreException ex) { + LOGGER.log(Level.SEVERE, "Error adding local files", ex); // NON-NLS + if (null != trans) { + try { + trans.rollback(); + } catch (TskCoreException ex2) { + LOGGER.log(Level.SEVERE, String.format("Failed to rollback transaction after exception: %s", ex.getMessage()), ex2); // NON-NLS + } + } + } + } + /** * Updates task progress as the file manager adds the local/logical files * and/or directories to the case database. From ec82b0785cbb08fb3b3e1c92d5fb1e1fe07ab11d Mon Sep 17 00:00:00 2001 From: Joe Ho Date: Wed, 21 Aug 2019 16:10:22 -0400 Subject: [PATCH 025/102] Fix codacy errors --- .../autopsy/logicalimager/dsp/AddLogicalImageTask.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java index fc2e93697b..95283b4d30 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java @@ -371,10 +371,10 @@ final class AddLogicalImageTask implements Runnable { } String vhdFilename = fields[0]; // String fileSystemOffsetStr = fields[1]; - String fileMetaAddressStr = fields[2]; +// String fileMetaAddressStr = fields[2]; // String extractStatusStr = fields[3]; - String ruleSetName = fields[4]; - String ruleName = fields[5]; +// String ruleSetName = fields[4]; +// String ruleName = fields[5]; // String description = fields[6]; String filename = fields[7]; String parentPath = fields[8]; @@ -386,7 +386,7 @@ final class AddLogicalImageTask implements Runnable { parentPath = ROOT_STR + "/" + vhdFilename + "/" + parentPath; //addLocalFile here - AbstractFile localFile = fileImporter.addLocalFile( + fileImporter.addLocalFile( Paths.get(src.toString(), extractedFilePath).toFile(), filename, parentPath, From 28aac6ca11e36ed94f738ca3478ecee078fd1616 Mon Sep 17 00:00:00 2001 From: Joe Ho Date: Thu, 22 Aug 2019 14:26:31 -0400 Subject: [PATCH 026/102] Address PR comments --- .../autopsy/casemodule/AddLocalFilesTask.java | 6 +- .../configuration/LogicalImagerConfig.java | 4 +- .../dsp/AddLogicalImageTask.java | 87 ++++++++----------- .../dsp/Bundle.properties-MERGED | 5 +- 4 files changed, 45 insertions(+), 57 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/AddLocalFilesTask.java b/Core/src/org/sleuthkit/autopsy/casemodule/AddLocalFilesTask.java index 775fc51d8c..185f696a97 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/AddLocalFilesTask.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/AddLocalFilesTask.java @@ -37,7 +37,7 @@ import org.sleuthkit.datamodel.TskDataException; * case database, grouped under a virtual directory that serves as the data * source. */ -public class AddLocalFilesTask implements Runnable { +class AddLocalFilesTask implements Runnable { private static final Logger LOGGER = Logger.getLogger(AddLocalFilesTask.class.getName()); private final String deviceId; @@ -68,7 +68,7 @@ public class AddLocalFilesTask implements Runnable { * during processing. * @param callback Callback to call when processing is done. */ - public AddLocalFilesTask(String deviceId, String rootVirtualDirectoryName, List localFilePaths, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callback) { + AddLocalFilesTask(String deviceId, String rootVirtualDirectoryName, List localFilePaths, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callback) { this.deviceId = deviceId; this.rootVirtualDirectoryName = rootVirtualDirectoryName; this.localFilePaths = localFilePaths; @@ -88,7 +88,7 @@ public class AddLocalFilesTask implements Runnable { try { progress.setIndeterminate(true); FileManager fileManager = Case.getCurrentCaseThrows().getServices().getFileManager(); - LocalFilesDataSource newDataSource = fileManager.addLocalFilesDataSource(deviceId, "", "", localFilePaths, new ProgressUpdater()); + LocalFilesDataSource newDataSource = fileManager.addLocalFilesDataSource(deviceId, rootVirtualDirectoryName, "", localFilePaths, new ProgressUpdater()); newDataSources.add(newDataSource); } catch (TskDataException | TskCoreException | NoCurrentCaseException ex) { errors.add(ex.getMessage()); diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/LogicalImagerConfig.java b/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/LogicalImagerConfig.java index 0dd40a7d9a..d896c15001 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/LogicalImagerConfig.java +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/LogicalImagerConfig.java @@ -85,13 +85,13 @@ class LogicalImagerConfig { String version, boolean finalizeImageWriter, boolean promptBeforeExit, - boolean creatVHD, + boolean createVHD, List ruleSets ) { this.version = version; this.finalizeImageWriter = finalizeImageWriter; this.promptBeforeExit = promptBeforeExit; - this.createVHD = creatVHD; + this.createVHD = createVHD; this.ruleSets = ruleSets; } diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java index 95283b4d30..80098747ed 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java @@ -36,7 +36,6 @@ import org.apache.commons.io.FileUtils; import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; -import org.sleuthkit.autopsy.casemodule.services.FileManager; import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorCallback; import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorProgressMonitor; import org.sleuthkit.autopsy.coreutils.Logger; @@ -107,7 +106,9 @@ final class AddLogicalImageTask implements Runnable { "AddLogicalImageTask.addingInterestingFiles=Adding search results as interesting files", "AddLogicalImageTask.doneAddingInterestingFiles=Done adding search results as interesting files", "# {0} - SearchResults.txt", "# {1} - directory", "AddLogicalImageTask.cannotFindFiles=Cannot find {0} in {1}", - "# {0} - reason", "AddLogicalImageTask.failedToAddInterestingFiles=Failed to add interesting files: {0}" + "# {0} - reason", "AddLogicalImageTask.failedToAddInterestingFiles=Failed to add interesting files: {0}", + "AddLogicalImageTask.addingExtractedFiles=Adding extracted files", + "AddLogicalImageTask.doneAddingExtractedFiles=Done adding extracted files", }) @Override public void run() { @@ -189,11 +190,14 @@ final class AddLogicalImageTask implements Runnable { } try { + progressMonitor.setProgressText(Bundle.AddLogicalImageTask_addingExtractedFiles()); addExtractedFiles(dest, Paths.get(dest.toString(), resultsFilename), newDataSources); + progressMonitor.setProgressText(Bundle.AddLogicalImageTask_doneAddingExtractedFiles()); } catch (TskCoreException | IOException ex) { errorList.add(ex.getMessage()); LOGGER.log(Level.SEVERE, String.format("Failed to add datasource: %s", ex.getMessage()), ex); // NON-NLS callback.done(DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS, errorList, emptyDataSources); + return; } } else { createVHD = true; @@ -209,6 +213,7 @@ final class AddLogicalImageTask implements Runnable { String msg = Bundle.AddLogicalImageTask_noCurrentCase(); errorList.add(msg); callback.done(DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS, errorList, emptyDataSources); + return; } } @@ -290,10 +295,13 @@ final class AddLogicalImageTask implements Runnable { br.readLine(); // skip the header line int lineNumber = 2; while ((line = br.readLine()) != null) { + if (cancelled) { + return; + } String[] fields = line.split("\t", -1); // NON-NLS -// if (fields.length != 9) { -// throw new IOException(Bundle.AddLogicalImageTask_notEnoughFields(lineNumber, fields.length, 9)); -// } + if (fields.length != 14) { + throw new IOException(Bundle.AddLogicalImageTask_notEnoughFields(lineNumber, fields.length, 9)); + } String vhdFilename = fields[0]; // String fileSystemOffsetStr = fields[1]; String fileMetaAddressStr = fields[2]; @@ -344,14 +352,13 @@ final class AddLogicalImageTask implements Runnable { BlackboardAttribute ruleNameAttribute = new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_CATEGORY, MODULE_NAME, ruleName); attributes.add(ruleNameAttribute); if (!blackboard.artifactExists(file, BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT, attributes)) { - BlackboardArtifact artifact = file.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT); + BlackboardArtifact artifact = this.currentCase.getSleuthkitCase().newBlackboardArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT, file.getId()); artifact.addAttributes(attributes); artifacts.add(artifact); } } private void addExtractedFiles(File src, Path resultsPath, List newDataSources) throws TskCoreException, UnsupportedEncodingException, IOException { - SleuthkitCase skCase = Case.getCurrentCase().getSleuthkitCase(); SleuthkitCase.CaseDbTransaction trans = null; try { @@ -365,16 +372,20 @@ final class AddLogicalImageTask implements Runnable { br.readLine(); // skip the header line int lineNumber = 2; while ((line = br.readLine()) != null) { + if (cancelled) { + rollbackTransaction(trans); + return; + } String[] fields = line.split("\t", -1); // NON-NLS if (fields.length != 14) { throw new IOException(Bundle.AddLogicalImageTask_notEnoughFields(lineNumber, fields.length, 14)); } String vhdFilename = fields[0]; // String fileSystemOffsetStr = fields[1]; -// String fileMetaAddressStr = fields[2]; +// String fileMetaAddressStr = fields[2]; // String extractStatusStr = fields[3]; -// String ruleSetName = fields[4]; -// String ruleName = fields[5]; +// String ruleSetName = fields[4]; +// String ruleName = fields[5]; // String description = fields[6]; String filename = fields[7]; String parentPath = fields[8]; @@ -398,48 +409,26 @@ final class AddLogicalImageTask implements Runnable { lineNumber++; } // end reading file - - trans.commit(); - newDataSources.add(localFilesDataSource); } - } catch (TskCoreException ex) { - LOGGER.log(Level.SEVERE, "Error adding local files", ex); // NON-NLS - if (null != trans) { - try { - trans.rollback(); - } catch (TskCoreException ex2) { - LOGGER.log(Level.SEVERE, String.format("Failed to rollback transaction after exception: %s", ex.getMessage()), ex2); // NON-NLS - } + trans.commit(); + newDataSources.add(localFilesDataSource); + + } catch (IOException | NumberFormatException | TskCoreException ex) { + LOGGER.log(Level.SEVERE, "Error adding extracted files", ex); // NON-NLS + rollbackTransaction(trans); + throw new TskCoreException("Error adding extracted files", ex); + } + } + + private void rollbackTransaction(SleuthkitCase.CaseDbTransaction trans) throws TskCoreException { + if (null != trans) { + try { + trans.rollback(); + } catch (TskCoreException ex) { + LOGGER.log(Level.SEVERE, String.format("Failed to rollback transaction: %s", ex.getMessage()), ex); // NON-NLS + throw new TskCoreException("Error cancelling", ex); } } } - /** - * Updates task progress as the file manager adds the local/logical files - * and/or directories to the case database. - */ - @Messages({ - "# {0} - parent path", "# {1} - filename", "AddLogicalImageTask.localFileAddProgress=Adding: {0}/{1}", - }) - private class ProgressUpdater implements FileManager.FileAddProgressUpdater { - - private int count; - - /** - * Updates task progress (called by the file manager after it adds each - * local file/directory to the case database). - */ - @Override - public void fileAdded(final AbstractFile file) { - ++count; - if (count % 10 == 0) { - progressMonitor.setProgressText( - Bundle.AddLogicalImageTask_localFileAddProgress( - file.getParentPath(), - file.getName() - ) - ); - } - } - } } diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/Bundle.properties-MERGED index 3cecb6f2c7..fd08147b08 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/Bundle.properties-MERGED @@ -2,6 +2,7 @@ # To change this template file, choose Tools | Templates # and open the template in the editor. +AddLogicalImageTask.addingExtractedFiles=Adding extracted files AddLogicalImageTask.addingInterestingFiles=Adding search results as interesting files # {0} - file AddLogicalImageTask.addingToReport=Adding {0} to report @@ -15,6 +16,7 @@ AddLogicalImageTask.cannotFindFiles=Cannot find {0} in {1} AddLogicalImageTask.copyingImageFromTo=Copying image from {0} to {1} # {0} - sparseImageDirectory AddLogicalImageTask.directoryDoesNotContainSparseImage=Directory {0} does not contain any images +AddLogicalImageTask.doneAddingExtractedFiles=Done adding extracted files AddLogicalImageTask.doneAddingInterestingFiles=Done adding search results as interesting files # {0} - file AddLogicalImageTask.doneAddingToReport=Done adding {0} to report @@ -30,9 +32,6 @@ AddLogicalImageTask.failedToCopyDirectory=Failed to copy directory {0} to {1} # {0} - file AddLogicalImageTask.failToGetCanonicalPath=Fail to get canonical path for {0} AddLogicalImageTask.ingestionCancelled=Ingestion cancelled -# {0} - parent path -# {1} - filename -AddLogicalImageTask.localFileAddProgress=Adding: {0}/{1} AddLogicalImageTask.noCurrentCase=No current case # {0} - line number # {1} - fields length From c58b602fa20ec893562c71f84d99227da638f129 Mon Sep 17 00:00:00 2001 From: Joe Ho Date: Thu, 22 Aug 2019 14:47:08 -0400 Subject: [PATCH 027/102] Fix more PR comments. --- .../autopsy/logicalimager/dsp/AddLogicalImageTask.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java index 80098747ed..48488dd82a 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java @@ -23,7 +23,6 @@ import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStreamReader; -import java.io.UnsupportedEncodingException; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; @@ -193,7 +192,7 @@ final class AddLogicalImageTask implements Runnable { progressMonitor.setProgressText(Bundle.AddLogicalImageTask_addingExtractedFiles()); addExtractedFiles(dest, Paths.get(dest.toString(), resultsFilename), newDataSources); progressMonitor.setProgressText(Bundle.AddLogicalImageTask_doneAddingExtractedFiles()); - } catch (TskCoreException | IOException ex) { + } catch (TskCoreException ex) { errorList.add(ex.getMessage()); LOGGER.log(Level.SEVERE, String.format("Failed to add datasource: %s", ex.getMessage()), ex); // NON-NLS callback.done(DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS, errorList, emptyDataSources); @@ -358,7 +357,7 @@ final class AddLogicalImageTask implements Runnable { } } - private void addExtractedFiles(File src, Path resultsPath, List newDataSources) throws TskCoreException, UnsupportedEncodingException, IOException { + private void addExtractedFiles(File src, Path resultsPath, List newDataSources) throws TskCoreException { SleuthkitCase skCase = Case.getCurrentCase().getSleuthkitCase(); SleuthkitCase.CaseDbTransaction trans = null; try { @@ -426,7 +425,6 @@ final class AddLogicalImageTask implements Runnable { trans.rollback(); } catch (TskCoreException ex) { LOGGER.log(Level.SEVERE, String.format("Failed to rollback transaction: %s", ex.getMessage()), ex); // NON-NLS - throw new TskCoreException("Error cancelling", ex); } } } From 9170d59cf5eff7e79cf17783eee8dfb2d4bcb56a Mon Sep 17 00:00:00 2001 From: Kelly Kelly Date: Mon, 26 Aug 2019 11:29:37 -0400 Subject: [PATCH 028/102] Inital commit of EMLParser --- .../Bundle.properties-MERGED | 1 + .../autopsy/thunderbirdparser/EMLParser.java | 77 ++++ .../autopsy/thunderbirdparser/MboxParser.java | 430 +++++++++--------- .../MimeJ4MessageParser.java | 305 +++++++++++++ .../ThunderbirdMboxFileIngestModule.java | 71 +++ 5 files changed, 669 insertions(+), 215 deletions(-) create mode 100755 thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/EMLParser.java create mode 100755 thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/MimeJ4MessageParser.java diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/Bundle.properties-MERGED b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/Bundle.properties-MERGED index b39a22c484..00b58bd2c7 100755 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/Bundle.properties-MERGED +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/Bundle.properties-MERGED @@ -1,3 +1,4 @@ +EMLParser.handleAttch.noOpenCase.errMsg=Exception while getting open case. MboxParser.handleAttch.noOpenCase.errMsg=Exception while getting open case. OpenIDE-Module-Display-Category=Ingest Module OpenIDE-Module-Long-Description=Email Parser ingest module.\n\nThe module extracts MBOX and PST e-mail files and posts the results to the blackboard.\nIt knows about the Thunderbird folder structure for MBOX files. diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/EMLParser.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/EMLParser.java new file mode 100755 index 0000000000..bf788c7c4c --- /dev/null +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/EMLParser.java @@ -0,0 +1,77 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ +package org.sleuthkit.autopsy.thunderbirdparser; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.logging.Level; +import org.apache.james.mime4j.MimeException; +import org.apache.james.mime4j.dom.BinaryBody; +import org.apache.james.mime4j.dom.Body; +import org.apache.james.mime4j.dom.Entity; +import org.apache.james.mime4j.dom.Message; +import org.apache.james.mime4j.dom.Multipart; +import org.apache.james.mime4j.dom.TextBody; +import org.apache.james.mime4j.dom.address.AddressList; +import org.apache.james.mime4j.dom.address.Mailbox; +import org.apache.james.mime4j.dom.address.MailboxList; +import org.apache.james.mime4j.dom.field.ContentDispositionField; +import org.apache.james.mime4j.dom.field.ContentTypeField; +import org.apache.james.mime4j.message.DefaultMessageBuilder; +import org.apache.james.mime4j.stream.Field; +import org.apache.james.mime4j.stream.MimeConfig; +import org.openide.util.NbBundle.Messages; +import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; +import org.sleuthkit.autopsy.coreutils.FileUtil; +import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.datamodel.EncodedFileOutputStream; +import org.sleuthkit.datamodel.TskData; + +/** + * + * @author kelly + */ +public class EMLParser extends MimeJ4MessageParser{ + + private static final Logger logger = Logger.getLogger(EMLParser.class.getName()); + + + static boolean isEMLFile(AbstractFile abFile, byte[] buffer) { + String ext = abFile.getNameExtension(); + boolean isEMLFile = ext != null ? ext.equals("eml") : false; + if(isEMLFile) { + isEMLFile = (new String(buffer)).contains("To:"); //NON-NLS + } + + return isEMLFile; + } + + static EmailMessage parse(AbstractFile sourceFile, String localPath) throws FileNotFoundException, IOException, MimeException { + try (FileInputStream fis = new FileInputStream(localPath)){ + //Create message with stream from file + //If you want to parse String, you can use: + //Message mimeMsg = new Message(new ByteArrayInputStream(mimeSource.getBytes())); + + DefaultMessageBuilder messageBuilder = new DefaultMessageBuilder(); + MimeConfig config = MimeConfig.custom().setMaxLineLen(-1).setMaxHeaderLen(-1).setMaxHeaderCount(-1).build(); + // disable line length checks. + messageBuilder.setMimeEntityConfig(config); + Message mimeMsg = messageBuilder.parseMessage(fis); + + + return (new EMLParser()).extractEmail(mimeMsg, localPath, sourceFile.getId()); + } + } + +} diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/MboxParser.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/MboxParser.java index 270e0ecdf8..164fcc62d4 100644 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/MboxParser.java +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/MboxParser.java @@ -65,7 +65,7 @@ import org.sleuthkit.datamodel.EncodedFileOutputStream; /** * An Iterator for parsing mbox files. Wraps an instance of MBoxEmailIterator. */ -class MboxParser implements Iterator { +class MboxParser extends MimeJ4MessageParser implements Iterator { private static final Logger logger = Logger.getLogger(MboxParser.class.getName()); private final DefaultMessageBuilder messageBuilder; @@ -186,87 +186,87 @@ class MboxParser implements Iterator { * * @return */ - private EmailMessage extractEmail(Message msg, long fileID) { - EmailMessage email = new EmailMessage(); - // Basic Info - email.setSender(getAddresses(msg.getFrom())); - email.setRecipients(getAddresses(msg.getTo())); - email.setBcc(getAddresses(msg.getBcc())); - email.setCc(getAddresses(msg.getCc())); - email.setSubject(msg.getSubject()); - email.setSentDate(msg.getDate()); - email.setLocalPath(localPath); - email.setMessageID(msg.getMessageId()); - - Field field = msg.getHeader().getField("in-reply-to"); //NON-NLS - String inReplyTo = null; - - if (field != null) { - inReplyTo = field.getBody(); - email.setInReplyToID(inReplyTo); - } - - field = msg.getHeader().getField("references"); - if (field != null) { - List references = new ArrayList<>(); - for (String id : field.getBody().split(">")) { - references.add(id.trim() + ">"); - } - - if (!references.contains(inReplyTo)) { - references.add(inReplyTo); - } - - email.setReferences(references); - } - - // Body - if (msg.isMultipart()) { - handleMultipart(email, (Multipart) msg.getBody(), fileID); - } else { - handleTextBody(email, (TextBody) msg.getBody(), msg.getMimeType(), msg.getHeader().getFields()); - } - - return email; - } - - /** - * Extract the subject, inReplyTo, message-ID and references from the - * Message object and returns them in a new EmailMessage object. - * - * @param msg Message object - * - * @return EmailMessage instance with only some of the message information - */ - private EmailMessage extractPartialEmail(Message msg) { - EmailMessage email = new EmailMessage(); - email.setSubject(msg.getSubject()); - email.setMessageID(msg.getMessageId()); - - Field field = msg.getHeader().getField("in-reply-to"); //NON-NLS - String inReplyTo = null; - - if (field != null) { - inReplyTo = field.getBody(); - email.setInReplyToID(inReplyTo); - } - - field = msg.getHeader().getField("references"); - if (field != null) { - List references = new ArrayList<>(); - for (String id : field.getBody().split(">")) { - references.add(id.trim() + ">"); - } - - if (!references.contains(inReplyTo)) { - references.add(inReplyTo); - } - - email.setReferences(references); - } - - return email; - } +// private EmailMessage extractEmail(Message msg, long fileID) { +// EmailMessage email = new EmailMessage(); +// // Basic Info +// email.setSender(getAddresses(msg.getFrom())); +// email.setRecipients(getAddresses(msg.getTo())); +// email.setBcc(getAddresses(msg.getBcc())); +// email.setCc(getAddresses(msg.getCc())); +// email.setSubject(msg.getSubject()); +// email.setSentDate(msg.getDate()); +// email.setLocalPath(localPath); +// email.setMessageID(msg.getMessageId()); +// +// Field field = msg.getHeader().getField("in-reply-to"); //NON-NLS +// String inReplyTo = null; +// +// if (field != null) { +// inReplyTo = field.getBody(); +// email.setInReplyToID(inReplyTo); +// } +// +// field = msg.getHeader().getField("references"); +// if (field != null) { +// List references = new ArrayList<>(); +// for (String id : field.getBody().split(">")) { +// references.add(id.trim() + ">"); +// } +// +// if (!references.contains(inReplyTo)) { +// references.add(inReplyTo); +// } +// +// email.setReferences(references); +// } +// +// // Body +// if (msg.isMultipart()) { +// handleMultipart(email, (Multipart) msg.getBody(), fileID); +// } else { +// handleTextBody(email, (TextBody) msg.getBody(), msg.getMimeType(), msg.getHeader().getFields()); +// } +// +// return email; +// } +// +// /** +// * Extract the subject, inReplyTo, message-ID and references from the +// * Message object and returns them in a new EmailMessage object. +// * +// * @param msg Message object +// * +// * @return EmailMessage instance with only some of the message information +// */ +// private EmailMessage extractPartialEmail(Message msg) { +// EmailMessage email = new EmailMessage(); +// email.setSubject(msg.getSubject()); +// email.setMessageID(msg.getMessageId()); +// +// Field field = msg.getHeader().getField("in-reply-to"); //NON-NLS +// String inReplyTo = null; +// +// if (field != null) { +// inReplyTo = field.getBody(); +// email.setInReplyToID(inReplyTo); +// } +// +// field = msg.getHeader().getField("references"); +// if (field != null) { +// List references = new ArrayList<>(); +// for (String id : field.getBody().split(">")) { +// references.add(id.trim() + ">"); +// } +// +// if (!references.contains(inReplyTo)) { +// references.add(inReplyTo); +// } +// +// email.setReferences(references); +// } +// +// return email; +// } /** * Handle a multipart mime message. Recursively calls handleMultipart if one @@ -276,138 +276,138 @@ class MboxParser implements Iterator { * @param email * @param multi */ - private void handleMultipart(EmailMessage email, Multipart multi, long fileID) { - List entities = multi.getBodyParts(); - for (int index = 0; index < entities.size(); index++) { - Entity e = entities.get(index); - if (e.isMultipart()) { - handleMultipart(email, (Multipart) e.getBody(), fileID); - } else if (e.getDispositionType() != null - && e.getDispositionType().equals(ContentDispositionField.DISPOSITION_TYPE_ATTACHMENT)) { - handleAttachment(email, e, fileID, index); - } else if (e.getMimeType().equals(HTML_TYPE) - || e.getMimeType().equals(ContentTypeField.TYPE_TEXT_PLAIN)) { - handleTextBody(email, (TextBody) e.getBody(), e.getMimeType(), e.getHeader().getFields()); - } else { - // Ignore other types. - } - } - } - - /** - * Extract text out of a body part of the message. - * - * Handles text and html mime types. Throws away all other types. (only - * other example I've seen is text/calendar) - * - * @param email - * @param tb - * @param type The Mime type of the body. - */ - private void handleTextBody(EmailMessage email, TextBody tb, String type, List fields) { - BufferedReader r; - try { - r = new BufferedReader(tb.getReader()); - StringBuilder bodyString = new StringBuilder(); - StringBuilder headersString = new StringBuilder(); - String line; - while ((line = r.readLine()) != null) { - bodyString.append(line).append("\n"); - } - - headersString.append("\n-----HEADERS-----\n"); - for (Field field : fields) { - String nextLine = field.getName() + ": " + field.getBody(); - headersString.append("\n").append(nextLine); - } - headersString.append("\n\n---END HEADERS--\n\n"); - - email.setHeaders(headersString.toString()); - - switch (type) { - case ContentTypeField.TYPE_TEXT_PLAIN: - email.setTextBody(bodyString.toString()); - break; - case HTML_TYPE: - email.setHtmlBody(bodyString.toString()); - break; - default: - // Not interested in other text types. - break; - } - } catch (IOException ex) { - logger.log(Level.WARNING, "Error getting text body of mbox message", ex); //NON-NLS - } - } - - /** - * Extract the attachment out of the given entity. Should only be called if - * e.getDispositionType() == "attachment" - * - * @param email - * @param e - */ - @NbBundle.Messages({"MboxParser.handleAttch.noOpenCase.errMsg=Exception while getting open case."}) - private void handleAttachment(EmailMessage email, Entity e, long fileID, int index) { - String outputDirPath; - String relModuleOutputPath; - try { - outputDirPath = ThunderbirdMboxFileIngestModule.getModuleOutputPath() + File.separator; - relModuleOutputPath = ThunderbirdMboxFileIngestModule.getRelModuleOutputPath() + File.separator; - } catch (NoCurrentCaseException ex) { - addErrorMessage(Bundle.MboxParser_handleAttch_noOpenCase_errMsg()); - logger.log(Level.SEVERE, Bundle.MboxParser_handleAttch_noOpenCase_errMsg(), ex); //NON-NLS - return; - } - String filename = FileUtil.escapeFileName(e.getFilename()); - - // also had some crazy long names, so make random one if we get those. - // also from Japanese image that had encoded name - if (filename.length() > 64) { - filename = UUID.randomUUID().toString(); - } - - String uniqueFilename = fileID + "-" + index + "-" + email.getSentDate() + "-" + filename; - String outPath = outputDirPath + uniqueFilename; - EncodedFileOutputStream fos; - BinaryBody bb; - try { - fos = new EncodedFileOutputStream(new FileOutputStream(outPath), TskData.EncodingType.XOR1); - } catch (IOException ex) { - addErrorMessage( - NbBundle.getMessage(this.getClass(), - "MboxParser.handleAttch.errMsg.failedToCreateOnDisk", outPath)); - logger.log(Level.WARNING, "Failed to create file output stream for: " + outPath, ex); //NON-NLS - return; - } - - try { - Body b = e.getBody(); - if (b instanceof BinaryBody) { - bb = (BinaryBody) b; - bb.writeTo(fos); - } else { - // This could potentially be other types. Only seen this once. - } - } catch (IOException ex) { - logger.log(Level.WARNING, "Failed to write mbox email attachment to disk.", ex); //NON-NLS - addErrorMessage(NbBundle.getMessage(this.getClass(), "MboxParser.handleAttch.failedWriteToDisk", filename)); - return; - } finally { - try { - fos.close(); - } catch (IOException ex) { - logger.log(Level.WARNING, "Failed to close file output stream", ex); //NON-NLS - } - } - - EmailMessage.Attachment attach = new EmailMessage.Attachment(); - attach.setName(filename); - attach.setLocalPath(relModuleOutputPath + uniqueFilename); - attach.setSize(new File(outPath).length()); - attach.setEncodingType(TskData.EncodingType.XOR1); - email.addAttachment(attach); - } +// private void handleMultipart(EmailMessage email, Multipart multi, long fileID) { +// List entities = multi.getBodyParts(); +// for (int index = 0; index < entities.size(); index++) { +// Entity e = entities.get(index); +// if (e.isMultipart()) { +// handleMultipart(email, (Multipart) e.getBody(), fileID); +// } else if (e.getDispositionType() != null +// && e.getDispositionType().equals(ContentDispositionField.DISPOSITION_TYPE_ATTACHMENT)) { +// handleAttachment(email, e, fileID, index); +// } else if (e.getMimeType().equals(HTML_TYPE) +// || e.getMimeType().equals(ContentTypeField.TYPE_TEXT_PLAIN)) { +// handleTextBody(email, (TextBody) e.getBody(), e.getMimeType(), e.getHeader().getFields()); +// } else { +// // Ignore other types. +// } +// } +// } +// +// /** +// * Extract text out of a body part of the message. +// * +// * Handles text and html mime types. Throws away all other types. (only +// * other example I've seen is text/calendar) +// * +// * @param email +// * @param tb +// * @param type The Mime type of the body. +// */ +// private void handleTextBody(EmailMessage email, TextBody tb, String type, List fields) { +// BufferedReader r; +// try { +// r = new BufferedReader(tb.getReader()); +// StringBuilder bodyString = new StringBuilder(); +// StringBuilder headersString = new StringBuilder(); +// String line; +// while ((line = r.readLine()) != null) { +// bodyString.append(line).append("\n"); +// } +// +// headersString.append("\n-----HEADERS-----\n"); +// for (Field field : fields) { +// String nextLine = field.getName() + ": " + field.getBody(); +// headersString.append("\n").append(nextLine); +// } +// headersString.append("\n\n---END HEADERS--\n\n"); +// +// email.setHeaders(headersString.toString()); +// +// switch (type) { +// case ContentTypeField.TYPE_TEXT_PLAIN: +// email.setTextBody(bodyString.toString()); +// break; +// case HTML_TYPE: +// email.setHtmlBody(bodyString.toString()); +// break; +// default: +// // Not interested in other text types. +// break; +// } +// } catch (IOException ex) { +// logger.log(Level.WARNING, "Error getting text body of mbox message", ex); //NON-NLS +// } +// } +// +// /** +// * Extract the attachment out of the given entity. Should only be called if +// * e.getDispositionType() == "attachment" +// * +// * @param email +// * @param e +// */ +// @NbBundle.Messages({"MboxParser.handleAttch.noOpenCase.errMsg=Exception while getting open case."}) +// private void handleAttachment(EmailMessage email, Entity e, long fileID, int index) { +// String outputDirPath; +// String relModuleOutputPath; +// try { +// outputDirPath = ThunderbirdMboxFileIngestModule.getModuleOutputPath() + File.separator; +// relModuleOutputPath = ThunderbirdMboxFileIngestModule.getRelModuleOutputPath() + File.separator; +// } catch (NoCurrentCaseException ex) { +// addErrorMessage(Bundle.MboxParser_handleAttch_noOpenCase_errMsg()); +// logger.log(Level.SEVERE, Bundle.MboxParser_handleAttch_noOpenCase_errMsg(), ex); //NON-NLS +// return; +// } +// String filename = FileUtil.escapeFileName(e.getFilename()); +// +// // also had some crazy long names, so make random one if we get those. +// // also from Japanese image that had encoded name +// if (filename.length() > 64) { +// filename = UUID.randomUUID().toString(); +// } +// +// String uniqueFilename = fileID + "-" + index + "-" + email.getSentDate() + "-" + filename; +// String outPath = outputDirPath + uniqueFilename; +// EncodedFileOutputStream fos; +// BinaryBody bb; +// try { +// fos = new EncodedFileOutputStream(new FileOutputStream(outPath), TskData.EncodingType.XOR1); +// } catch (IOException ex) { +// addErrorMessage( +// NbBundle.getMessage(this.getClass(), +// "MboxParser.handleAttch.errMsg.failedToCreateOnDisk", outPath)); +// logger.log(Level.WARNING, "Failed to create file output stream for: " + outPath, ex); //NON-NLS +// return; +// } +// +// try { +// Body b = e.getBody(); +// if (b instanceof BinaryBody) { +// bb = (BinaryBody) b; +// bb.writeTo(fos); +// } else { +// // This could potentially be other types. Only seen this once. +// } +// } catch (IOException ex) { +// logger.log(Level.WARNING, "Failed to write mbox email attachment to disk.", ex); //NON-NLS +// addErrorMessage(NbBundle.getMessage(this.getClass(), "MboxParser.handleAttch.failedWriteToDisk", filename)); +// return; +// } finally { +// try { +// fos.close(); +// } catch (IOException ex) { +// logger.log(Level.WARNING, "Failed to close file output stream", ex); //NON-NLS +// } +// } +// +// EmailMessage.Attachment attach = new EmailMessage.Attachment(); +// attach.setName(filename); +// attach.setLocalPath(relModuleOutputPath + uniqueFilename); +// attach.setSize(new File(outPath).length()); +// attach.setEncodingType(TskData.EncodingType.XOR1); +// email.addAttachment(attach); +// } /** * Get a String representation of the MailboxList (which is a list of email @@ -523,7 +523,7 @@ class MboxParser implements Iterator { try { Message msg = messageBuilder.parseMessage(messageBuffer.asInputStream(encoder.charset())); if (wholeMsg) { - return extractEmail(msg, fileID); + return extractEmail(msg, localPath, fileID); } else { return extractPartialEmail(msg); } diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/MimeJ4MessageParser.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/MimeJ4MessageParser.java new file mode 100755 index 0000000000..de3c4b11ec --- /dev/null +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/MimeJ4MessageParser.java @@ -0,0 +1,305 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ +package org.sleuthkit.autopsy.thunderbirdparser; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.logging.Level; +import org.apache.james.mime4j.dom.BinaryBody; +import org.apache.james.mime4j.dom.Body; +import org.apache.james.mime4j.dom.Entity; +import org.apache.james.mime4j.dom.Message; +import org.apache.james.mime4j.dom.Multipart; +import org.apache.james.mime4j.dom.TextBody; +import org.apache.james.mime4j.dom.address.AddressList; +import org.apache.james.mime4j.dom.address.Mailbox; +import org.apache.james.mime4j.dom.address.MailboxList; +import org.apache.james.mime4j.dom.field.ContentDispositionField; +import org.apache.james.mime4j.dom.field.ContentTypeField; +import org.apache.james.mime4j.stream.Field; +import org.openide.util.NbBundle; +import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; +import org.sleuthkit.autopsy.coreutils.FileUtil; +import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.datamodel.EncodedFileOutputStream; +import org.sleuthkit.datamodel.TskData; + +/** + * + * @author kelly + */ +public abstract class MimeJ4MessageParser { + private static final Logger logger = Logger.getLogger(MimeJ4MessageParser.class.getName()); + + /** + * The mime type string for html text. + */ + + private static final String HTML_TYPE = "text/html"; //NON-NLS + + /** + * Use the information stored in the given mime4j message to populate an + * EmailMessage. + * + * @param msg + * + * @return + */ + EmailMessage extractEmail(Message msg, String localPath, long sourceFileID) { + EmailMessage email = new EmailMessage(); + // Basic Info + email.setSender(getAddresses(msg.getFrom())); + email.setRecipients(getAddresses(msg.getTo())); + email.setBcc(getAddresses(msg.getBcc())); + email.setCc(getAddresses(msg.getCc())); + email.setSubject(msg.getSubject()); + email.setSentDate(msg.getDate()); + email.setLocalPath(localPath); + email.setMessageID(msg.getMessageId()); + + Field field = msg.getHeader().getField("in-reply-to"); //NON-NLS + String inReplyTo = null; + + if (field != null) { + inReplyTo = field.getBody(); + email.setInReplyToID(inReplyTo); + } + + field = msg.getHeader().getField("references"); + if (field != null) { + List references = new ArrayList<>(); + for (String id : field.getBody().split(">")) { + references.add(id.trim() + ">"); + } + + if (!references.contains(inReplyTo)) { + references.add(inReplyTo); + } + + email.setReferences(references); + } + + // Body + if (msg.isMultipart()) { + handleMultipart(email, (Multipart) msg.getBody(), sourceFileID); + } else { + handleTextBody(email, (TextBody) msg.getBody(), msg.getMimeType(), msg.getHeader().getFields()); + } + + return email; + } + + + /** + * Extract the subject, inReplyTo, message-ID and references from the + * Message object and returns them in a new EmailMessage object. + * + * @param msg Message object + * + * @return EmailMessage instance with only some of the message information + */ + EmailMessage extractPartialEmail(Message msg) { + EmailMessage email = new EmailMessage(); + email.setSubject(msg.getSubject()); + email.setMessageID(msg.getMessageId()); + + Field field = msg.getHeader().getField("in-reply-to"); //NON-NLS + String inReplyTo = null; + + if (field != null) { + inReplyTo = field.getBody(); + email.setInReplyToID(inReplyTo); + } + + field = msg.getHeader().getField("references"); + if (field != null) { + List references = new ArrayList<>(); + for (String id : field.getBody().split(">")) { + references.add(id.trim() + ">"); + } + + if (!references.contains(inReplyTo)) { + references.add(inReplyTo); + } + + email.setReferences(references); + } + + return email; + } + + /** + * Handle a multipart mime message. Recursively calls handleMultipart if one + * of the body parts is another multipart. Otherwise, calls the correct + * method to extract information out of each part of the body. + * + * @param email + * @param multi + */ + private void handleMultipart(EmailMessage email, Multipart multi, long fileID) { + List entities = multi.getBodyParts(); + for (int index = 0; index < entities.size(); index++) { + Entity e = entities.get(index); + if (e.isMultipart()) { + handleMultipart(email, (Multipart) e.getBody(), fileID); + } else if (e.getDispositionType() != null + && e.getDispositionType().equals(ContentDispositionField.DISPOSITION_TYPE_ATTACHMENT)) { + handleAttachment(email, e, fileID, index); + } else if (e.getMimeType().equals(HTML_TYPE) + || e.getMimeType().equals(ContentTypeField.TYPE_TEXT_PLAIN)) { + handleTextBody(email, (TextBody) e.getBody(), e.getMimeType(), e.getHeader().getFields()); + } else { + // Ignore other types. + } + } + } + + /** + * Extract text out of a body part of the message. + * + * Handles text and html mime types. Throws away all other types. (only + * other example I've seen is text/calendar) + * + * @param email + * @param tb + * @param type The Mime type of the body. + */ + private void handleTextBody(EmailMessage email, TextBody tb, String type, List fields) { + BufferedReader r; + try { + r = new BufferedReader(tb.getReader()); + StringBuilder bodyString = new StringBuilder(); + StringBuilder headersString = new StringBuilder(); + String line; + while ((line = r.readLine()) != null) { + bodyString.append(line).append("\n"); + } + + headersString.append("\n-----HEADERS-----\n"); + for (Field field : fields) { + String nextLine = field.getName() + ": " + field.getBody(); + headersString.append("\n").append(nextLine); + } + headersString.append("\n\n---END HEADERS--\n\n"); + + email.setHeaders(headersString.toString()); + + switch (type) { + case ContentTypeField.TYPE_TEXT_PLAIN: + email.setTextBody(bodyString.toString()); + break; + case HTML_TYPE: + email.setHtmlBody(bodyString.toString()); + break; + default: + // Not interested in other text types. + break; + } + } catch (IOException ex) { + logger.log(Level.WARNING, "Error getting text body of mbox message", ex); //NON-NLS + } + } + + /** + * Extract the attachment out of the given entity. Should only be called if + * e.getDispositionType() == "attachment" + * + * @param email + * @param e + */ + @NbBundle.Messages({"EMLParser.handleAttch.noOpenCase.errMsg=Exception while getting open case."}) + private static void handleAttachment(EmailMessage email, Entity e, long fileID, int index) { + String outputDirPath; + String relModuleOutputPath; + try { + outputDirPath = ThunderbirdMboxFileIngestModule.getModuleOutputPath() + File.separator; + relModuleOutputPath = ThunderbirdMboxFileIngestModule.getRelModuleOutputPath() + File.separator; + } catch (NoCurrentCaseException ex) { +// logger.log(Level.SEVERE, Bundle.MboxParser_handleAttch_noOpenCase_errMsg(), ex); //NON-NLS + return; + } + String filename = FileUtil.escapeFileName(e.getFilename()); + + // also had some crazy long names, so make random one if we get those. + // also from Japanese image that had encoded name + if (filename.length() > 64) { + filename = UUID.randomUUID().toString(); + } + + String uniqueFilename = fileID + "-" + index + "-" + email.getSentDate() + "-" + filename; + String outPath = outputDirPath + uniqueFilename; + EncodedFileOutputStream fos; + BinaryBody bb; + try { + fos = new EncodedFileOutputStream(new FileOutputStream(outPath), TskData.EncodingType.XOR1); + } catch (IOException ex) { + logger.log(Level.WARNING, "Failed to create file output stream for: " + outPath, ex); //NON-NLS + return; + } + + try { + Body b = e.getBody(); + if (b instanceof BinaryBody) { + bb = (BinaryBody) b; + bb.writeTo(fos); + } else { + // This could potentially be other types. Only seen this once. + } + } catch (IOException ex) { + logger.log(Level.WARNING, "Failed to write mbox email attachment to disk.", ex); //NON-NLS + return; + } finally { + try { + fos.close(); + } catch (IOException ex) { + logger.log(Level.WARNING, "Failed to close file output stream", ex); //NON-NLS + } + } + + EmailMessage.Attachment attach = new EmailMessage.Attachment(); + attach.setName(filename); + attach.setLocalPath(relModuleOutputPath + uniqueFilename); + attach.setSize(new File(outPath).length()); + attach.setEncodingType(TskData.EncodingType.XOR1); + email.addAttachment(attach); + } + + /** + * Get a String representation of the MailboxList (which is a list of email + * addresses). + * + * @param mailboxList + * + * @return + */ + private static String getAddresses(MailboxList mailboxList) { + if (mailboxList == null) { + return ""; + } + StringBuilder addresses = new StringBuilder(); + for (Mailbox m : mailboxList) { + addresses.append(m.toString()).append("; "); + } + return addresses.toString(); + } + + /** + * Get a String representation of the AddressList (which is a list of email + * addresses). + * + * @param addressList + * + * @return + */ + private static String getAddresses(AddressList addressList) { + return (addressList == null) ? "" : getAddresses(addressList.flatten()); + } +} diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java index 9687df82d1..5f00d96bb5 100644 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java @@ -29,6 +29,7 @@ import java.util.Set; import java.util.logging.Level; import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.james.mime4j.MimeException; import org.openide.util.NbBundle; import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.casemodule.Case; @@ -113,12 +114,15 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { // check its signature boolean isMbox = false; + boolean isEMLFile = false; + try { byte[] t = new byte[64]; if (abstractFile.getSize() > 64) { int byteRead = abstractFile.read(t, 0, 64); if (byteRead > 0) { isMbox = MboxParser.isValidMimeTypeMbox(t); + isEMLFile = EMLParser.isEMLFile(abstractFile, t); } } } catch (TskException ex) { @@ -128,6 +132,10 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { if (isMbox) { return processMBox(abstractFile); } + + if(isEMLFile) { + return processEMLFile(abstractFile); + } if (PstParser.isPstFile(abstractFile)) { return processPst(abstractFile); @@ -352,6 +360,69 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { return ProcessResult.OK; } + + private ProcessResult processEMLFile(AbstractFile abstractFile) { + String fileName; + try { + fileName = getTempPath() + File.separator + abstractFile.getName() + + "-" + String.valueOf(abstractFile.getId()); + } catch (NoCurrentCaseException ex) { + logger.log(Level.SEVERE, "Exception while getting open case.", ex); //NON-NLS + return ProcessResult.ERROR; + } + File file = new File(fileName); + + long freeSpace = services.getFreeDiskSpace(); + if ((freeSpace != IngestMonitor.DISK_FREE_SPACE_UNKNOWN) && (abstractFile.getSize() >= freeSpace)) { + logger.log(Level.WARNING, String.format("Not enough disk space to write file '%s' (id=%d) to disk.", + abstractFile.getName(), abstractFile.getId())); //NON-NLS +// IngestMessage msg = IngestMessage.createErrorMessage(EmailParserModuleFactory.getModuleName(), EmailParserModuleFactory.getModuleName(), +// Bundle.ThunderbirdMboxFileIngestModule_errorMessage_outOfDiskSpace(abstractFile.getName(), abstractFile.getId())); +// services.postMessage(msg); + return ProcessResult.OK; + } + + try { + ContentUtils.writeToFile(abstractFile, file, context::fileIngestIsCancelled); + } catch (IOException ex) { + logger.log(Level.WARNING, String.format("Failed writing the vCard file '%s' (id=%d) to disk.", + abstractFile.getName(), abstractFile.getId()), ex); //NON-NLS + return ProcessResult.OK; + } + + try { + EmailMessage message = EMLParser.parse(abstractFile, fileName); + + if(message == null) { + return ProcessResult.OK; + } + + List derivedFiles = new ArrayList<>(); + + BlackboardArtifact msgArtifact = addEmailArtifact(message, abstractFile); + + if ((msgArtifact != null) && (message.hasAttachment())) { + derivedFiles.addAll(handleAttachments(message.getAttachments(), abstractFile, msgArtifact )); + } + + + if (derivedFiles.isEmpty() == false) { + for (AbstractFile derived : derivedFiles) { + services.fireModuleContentEvent(new ModuleContentEvent(derived)); + } + } + context.addFilesToJob(derivedFiles); + + } catch (IOException ex) { + logger.log(Level.WARNING, String.format("Error reading eml file %s", abstractFile.getName()), ex); + return ProcessResult.ERROR; + } catch (MimeException ex) { + logger.log(Level.WARNING, String.format("Error reading eml file %s", abstractFile.getName()), ex); + return ProcessResult.ERROR; + } + + return ProcessResult.OK; + } /** * Get a path to a temporary folder. From 229110ef765a7d6335e3aaa9d7be1d84c954f428 Mon Sep 17 00:00:00 2001 From: Kelly Kelly Date: Mon, 26 Aug 2019 15:10:33 -0400 Subject: [PATCH 029/102] Completed support for EML files. --- .../Bundle.properties-MERGED | 2 +- .../autopsy/thunderbirdparser/EMLParser.java | 85 +++-- .../autopsy/thunderbirdparser/MboxParser.java | 318 +----------------- .../MimeJ4MessageParser.java | 101 +++++- 4 files changed, 134 insertions(+), 372 deletions(-) diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/Bundle.properties-MERGED b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/Bundle.properties-MERGED index 00b58bd2c7..aa01a19072 100755 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/Bundle.properties-MERGED +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/Bundle.properties-MERGED @@ -1,5 +1,5 @@ -EMLParser.handleAttch.noOpenCase.errMsg=Exception while getting open case. MboxParser.handleAttch.noOpenCase.errMsg=Exception while getting open case. +MimeJ4MessageParser.handleAttch.noOpenCase.errMsg=Exception while getting open case. OpenIDE-Module-Display-Category=Ingest Module OpenIDE-Module-Long-Description=Email Parser ingest module.\n\nThe module extracts MBOX and PST e-mail files and posts the results to the blackboard.\nIt knows about the Thunderbird folder structure for MBOX files. OpenIDE-Module-Name=Email Parser diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/EMLParser.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/EMLParser.java index bf788c7c4c..eea1b992e7 100755 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/EMLParser.java +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/EMLParser.java @@ -1,52 +1,48 @@ /* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. + * Autopsy Forensic Browser + * + * Copyright 2019 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.sleuthkit.autopsy.thunderbirdparser; -import java.io.BufferedReader; -import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; -import java.io.FileOutputStream; import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.UUID; -import java.util.logging.Level; import org.apache.james.mime4j.MimeException; -import org.apache.james.mime4j.dom.BinaryBody; -import org.apache.james.mime4j.dom.Body; -import org.apache.james.mime4j.dom.Entity; import org.apache.james.mime4j.dom.Message; -import org.apache.james.mime4j.dom.Multipart; -import org.apache.james.mime4j.dom.TextBody; -import org.apache.james.mime4j.dom.address.AddressList; -import org.apache.james.mime4j.dom.address.Mailbox; -import org.apache.james.mime4j.dom.address.MailboxList; -import org.apache.james.mime4j.dom.field.ContentDispositionField; -import org.apache.james.mime4j.dom.field.ContentTypeField; -import org.apache.james.mime4j.message.DefaultMessageBuilder; -import org.apache.james.mime4j.stream.Field; -import org.apache.james.mime4j.stream.MimeConfig; -import org.openide.util.NbBundle.Messages; -import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; -import org.sleuthkit.autopsy.coreutils.FileUtil; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.datamodel.AbstractFile; -import org.sleuthkit.datamodel.EncodedFileOutputStream; -import org.sleuthkit.datamodel.TskData; /** - * - * @author kelly + * EML file parser. An .eml file contains a single email message. + * */ public class EMLParser extends MimeJ4MessageParser{ private static final Logger logger = Logger.getLogger(EMLParser.class.getName()); - + /** + * If the extention of the AbstractFile is eml and 'To:' is found close to the + * beginning of the file, then its probably an eml file. + * + * @param abFile AbstractFile to test + * @param buffer A byte buffer of the begining of the file. + * + * @return True, if we think this is an eml file, false otherwise. + */ static boolean isEMLFile(AbstractFile abFile, byte[] buffer) { String ext = abFile.getNameExtension(); boolean isEMLFile = ext != null ? ext.equals("eml") : false; @@ -57,21 +53,22 @@ public class EMLParser extends MimeJ4MessageParser{ return isEMLFile; } + /** + * + * @param sourceFile AbstractFile source file for eml message + * @param localPath The local path to the eml file + * + * @return EmailMessage object for message in eml file + * + * @throws FileNotFoundException + * @throws IOException + * @throws MimeException + */ static EmailMessage parse(AbstractFile sourceFile, String localPath) throws FileNotFoundException, IOException, MimeException { try (FileInputStream fis = new FileInputStream(localPath)){ - //Create message with stream from file - //If you want to parse String, you can use: - //Message mimeMsg = new Message(new ByteArrayInputStream(mimeSource.getBytes())); - - DefaultMessageBuilder messageBuilder = new DefaultMessageBuilder(); - MimeConfig config = MimeConfig.custom().setMaxLineLen(-1).setMaxHeaderLen(-1).setMaxHeaderCount(-1).build(); - // disable line length checks. - messageBuilder.setMimeEntityConfig(config); - Message mimeMsg = messageBuilder.parseMessage(fis); - - - return (new EMLParser()).extractEmail(mimeMsg, localPath, sourceFile.getId()); + EMLParser parser = new EMLParser(); + Message mimeMsg = parser.getMessageBuilder().parseMessage(fis); + return parser.extractEmail(mimeMsg, localPath, sourceFile.getId()); } } - } diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/MboxParser.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/MboxParser.java index 164fcc62d4..a17294b15b 100644 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/MboxParser.java +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/MboxParser.java @@ -19,12 +19,10 @@ package org.sleuthkit.autopsy.thunderbirdparser; import java.io.BufferedInputStream; -import java.io.BufferedReader; import java.io.CharConversionException; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; -import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.nio.charset.Charset; @@ -35,32 +33,14 @@ import java.nio.charset.UnsupportedCharsetException; import java.util.ArrayList; import java.util.Iterator; import java.util.List; -import java.util.UUID; import java.util.logging.Level; import org.sleuthkit.autopsy.coreutils.Logger; -import org.apache.james.mime4j.dom.BinaryBody; -import org.apache.james.mime4j.dom.Body; -import org.apache.james.mime4j.dom.Entity; import org.apache.james.mime4j.dom.Message; -import org.apache.james.mime4j.dom.Multipart; -import org.apache.james.mime4j.dom.TextBody; -import org.apache.james.mime4j.dom.address.AddressList; -import org.apache.james.mime4j.dom.address.Mailbox; -import org.apache.james.mime4j.dom.address.MailboxList; -import org.apache.james.mime4j.dom.field.ContentDispositionField; -import org.apache.james.mime4j.dom.field.ContentTypeField; import org.apache.james.mime4j.mboxiterator.CharBufferWrapper; import org.apache.james.mime4j.mboxiterator.MboxIterator; -import org.apache.james.mime4j.message.DefaultMessageBuilder; -import org.apache.james.mime4j.stream.Field; -import org.apache.james.mime4j.stream.MimeConfig; import org.apache.tika.parser.txt.CharsetDetector; import org.apache.tika.parser.txt.CharsetMatch; import org.openide.util.NbBundle; -import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; -import org.sleuthkit.autopsy.coreutils.FileUtil; -import org.sleuthkit.datamodel.TskData; -import org.sleuthkit.datamodel.EncodedFileOutputStream; /** * An Iterator for parsing mbox files. Wraps an instance of MBoxEmailIterator. @@ -68,27 +48,11 @@ import org.sleuthkit.datamodel.EncodedFileOutputStream; class MboxParser extends MimeJ4MessageParser implements Iterator { private static final Logger logger = Logger.getLogger(MboxParser.class.getName()); - private final DefaultMessageBuilder messageBuilder; - private final List errorList = new ArrayList<>(); - - /** - * The mime type string for html text. - */ - private static final String HTML_TYPE = "text/html"; //NON-NLS - - /** - * The local path of the mbox file. - */ - private String localPath; private Iterator emailIterator = null; private MboxParser(String localPath) { - this.localPath = localPath; - messageBuilder = new DefaultMessageBuilder(); - MimeConfig config = MimeConfig.custom().setMaxLineLen(-1).build(); - // disable line length checks. - messageBuilder.setMimeEntityConfig(config); + setLocalPath(localPath); } static boolean isValidMimeTypeMbox(byte[] buffer) { @@ -170,276 +134,6 @@ class MboxParser extends MimeJ4MessageParser implements Iterator { return emailIterator != null ? emailIterator.next() : null; } - String getErrors() { - String result = ""; - for (String msg: errorList) { - result += "
  • " + msg + "
    • "; - } - return result; - } - - /** - * Use the information stored in the given mime4j message to populate an - * EmailMessage. - * - * @param msg - * - * @return - */ -// private EmailMessage extractEmail(Message msg, long fileID) { -// EmailMessage email = new EmailMessage(); -// // Basic Info -// email.setSender(getAddresses(msg.getFrom())); -// email.setRecipients(getAddresses(msg.getTo())); -// email.setBcc(getAddresses(msg.getBcc())); -// email.setCc(getAddresses(msg.getCc())); -// email.setSubject(msg.getSubject()); -// email.setSentDate(msg.getDate()); -// email.setLocalPath(localPath); -// email.setMessageID(msg.getMessageId()); -// -// Field field = msg.getHeader().getField("in-reply-to"); //NON-NLS -// String inReplyTo = null; -// -// if (field != null) { -// inReplyTo = field.getBody(); -// email.setInReplyToID(inReplyTo); -// } -// -// field = msg.getHeader().getField("references"); -// if (field != null) { -// List references = new ArrayList<>(); -// for (String id : field.getBody().split(">")) { -// references.add(id.trim() + ">"); -// } -// -// if (!references.contains(inReplyTo)) { -// references.add(inReplyTo); -// } -// -// email.setReferences(references); -// } -// -// // Body -// if (msg.isMultipart()) { -// handleMultipart(email, (Multipart) msg.getBody(), fileID); -// } else { -// handleTextBody(email, (TextBody) msg.getBody(), msg.getMimeType(), msg.getHeader().getFields()); -// } -// -// return email; -// } -// -// /** -// * Extract the subject, inReplyTo, message-ID and references from the -// * Message object and returns them in a new EmailMessage object. -// * -// * @param msg Message object -// * -// * @return EmailMessage instance with only some of the message information -// */ -// private EmailMessage extractPartialEmail(Message msg) { -// EmailMessage email = new EmailMessage(); -// email.setSubject(msg.getSubject()); -// email.setMessageID(msg.getMessageId()); -// -// Field field = msg.getHeader().getField("in-reply-to"); //NON-NLS -// String inReplyTo = null; -// -// if (field != null) { -// inReplyTo = field.getBody(); -// email.setInReplyToID(inReplyTo); -// } -// -// field = msg.getHeader().getField("references"); -// if (field != null) { -// List references = new ArrayList<>(); -// for (String id : field.getBody().split(">")) { -// references.add(id.trim() + ">"); -// } -// -// if (!references.contains(inReplyTo)) { -// references.add(inReplyTo); -// } -// -// email.setReferences(references); -// } -// -// return email; -// } - - /** - * Handle a multipart mime message. Recursively calls handleMultipart if one - * of the body parts is another multipart. Otherwise, calls the correct - * method to extract information out of each part of the body. - * - * @param email - * @param multi - */ -// private void handleMultipart(EmailMessage email, Multipart multi, long fileID) { -// List entities = multi.getBodyParts(); -// for (int index = 0; index < entities.size(); index++) { -// Entity e = entities.get(index); -// if (e.isMultipart()) { -// handleMultipart(email, (Multipart) e.getBody(), fileID); -// } else if (e.getDispositionType() != null -// && e.getDispositionType().equals(ContentDispositionField.DISPOSITION_TYPE_ATTACHMENT)) { -// handleAttachment(email, e, fileID, index); -// } else if (e.getMimeType().equals(HTML_TYPE) -// || e.getMimeType().equals(ContentTypeField.TYPE_TEXT_PLAIN)) { -// handleTextBody(email, (TextBody) e.getBody(), e.getMimeType(), e.getHeader().getFields()); -// } else { -// // Ignore other types. -// } -// } -// } -// -// /** -// * Extract text out of a body part of the message. -// * -// * Handles text and html mime types. Throws away all other types. (only -// * other example I've seen is text/calendar) -// * -// * @param email -// * @param tb -// * @param type The Mime type of the body. -// */ -// private void handleTextBody(EmailMessage email, TextBody tb, String type, List fields) { -// BufferedReader r; -// try { -// r = new BufferedReader(tb.getReader()); -// StringBuilder bodyString = new StringBuilder(); -// StringBuilder headersString = new StringBuilder(); -// String line; -// while ((line = r.readLine()) != null) { -// bodyString.append(line).append("\n"); -// } -// -// headersString.append("\n-----HEADERS-----\n"); -// for (Field field : fields) { -// String nextLine = field.getName() + ": " + field.getBody(); -// headersString.append("\n").append(nextLine); -// } -// headersString.append("\n\n---END HEADERS--\n\n"); -// -// email.setHeaders(headersString.toString()); -// -// switch (type) { -// case ContentTypeField.TYPE_TEXT_PLAIN: -// email.setTextBody(bodyString.toString()); -// break; -// case HTML_TYPE: -// email.setHtmlBody(bodyString.toString()); -// break; -// default: -// // Not interested in other text types. -// break; -// } -// } catch (IOException ex) { -// logger.log(Level.WARNING, "Error getting text body of mbox message", ex); //NON-NLS -// } -// } -// -// /** -// * Extract the attachment out of the given entity. Should only be called if -// * e.getDispositionType() == "attachment" -// * -// * @param email -// * @param e -// */ -// @NbBundle.Messages({"MboxParser.handleAttch.noOpenCase.errMsg=Exception while getting open case."}) -// private void handleAttachment(EmailMessage email, Entity e, long fileID, int index) { -// String outputDirPath; -// String relModuleOutputPath; -// try { -// outputDirPath = ThunderbirdMboxFileIngestModule.getModuleOutputPath() + File.separator; -// relModuleOutputPath = ThunderbirdMboxFileIngestModule.getRelModuleOutputPath() + File.separator; -// } catch (NoCurrentCaseException ex) { -// addErrorMessage(Bundle.MboxParser_handleAttch_noOpenCase_errMsg()); -// logger.log(Level.SEVERE, Bundle.MboxParser_handleAttch_noOpenCase_errMsg(), ex); //NON-NLS -// return; -// } -// String filename = FileUtil.escapeFileName(e.getFilename()); -// -// // also had some crazy long names, so make random one if we get those. -// // also from Japanese image that had encoded name -// if (filename.length() > 64) { -// filename = UUID.randomUUID().toString(); -// } -// -// String uniqueFilename = fileID + "-" + index + "-" + email.getSentDate() + "-" + filename; -// String outPath = outputDirPath + uniqueFilename; -// EncodedFileOutputStream fos; -// BinaryBody bb; -// try { -// fos = new EncodedFileOutputStream(new FileOutputStream(outPath), TskData.EncodingType.XOR1); -// } catch (IOException ex) { -// addErrorMessage( -// NbBundle.getMessage(this.getClass(), -// "MboxParser.handleAttch.errMsg.failedToCreateOnDisk", outPath)); -// logger.log(Level.WARNING, "Failed to create file output stream for: " + outPath, ex); //NON-NLS -// return; -// } -// -// try { -// Body b = e.getBody(); -// if (b instanceof BinaryBody) { -// bb = (BinaryBody) b; -// bb.writeTo(fos); -// } else { -// // This could potentially be other types. Only seen this once. -// } -// } catch (IOException ex) { -// logger.log(Level.WARNING, "Failed to write mbox email attachment to disk.", ex); //NON-NLS -// addErrorMessage(NbBundle.getMessage(this.getClass(), "MboxParser.handleAttch.failedWriteToDisk", filename)); -// return; -// } finally { -// try { -// fos.close(); -// } catch (IOException ex) { -// logger.log(Level.WARNING, "Failed to close file output stream", ex); //NON-NLS -// } -// } -// -// EmailMessage.Attachment attach = new EmailMessage.Attachment(); -// attach.setName(filename); -// attach.setLocalPath(relModuleOutputPath + uniqueFilename); -// attach.setSize(new File(outPath).length()); -// attach.setEncodingType(TskData.EncodingType.XOR1); -// email.addAttachment(attach); -// } - - /** - * Get a String representation of the MailboxList (which is a list of email - * addresses). - * - * @param mailboxList - * - * @return - */ - private String getAddresses(MailboxList mailboxList) { - if (mailboxList == null) { - return ""; - } - StringBuilder addresses = new StringBuilder(); - for (Mailbox m : mailboxList) { - addresses.append(m.toString()).append("; "); - } - return addresses.toString(); - } - - /** - * Get a String representation of the AddressList (which is a list of email - * addresses). - * - * @param addressList - * - * @return - */ - private String getAddresses(AddressList addressList) { - return (addressList == null) ? "" : getAddresses(addressList.flatten()); - } - /** * Get a list of the possible encoders for the given mboxFile using Tika's * CharsetDetector. At a minimum, returns the standard built in charsets. @@ -489,11 +183,7 @@ class MboxParser extends MimeJ4MessageParser implements Iterator { } } } - - private void addErrorMessage(String msg) { - errorList.add(msg); - } - + /** * An Interator for mbox email messages. */ @@ -521,9 +211,9 @@ class MboxParser extends MimeJ4MessageParser implements Iterator { CharBufferWrapper messageBuffer = mboxIterator.next(); try { - Message msg = messageBuilder.parseMessage(messageBuffer.asInputStream(encoder.charset())); + Message msg = getMessageBuilder().parseMessage(messageBuffer.asInputStream(encoder.charset())); if (wholeMsg) { - return extractEmail(msg, localPath, fileID); + return extractEmail(msg, getLocalPath(), fileID); } else { return extractPartialEmail(msg); } diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/MimeJ4MessageParser.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/MimeJ4MessageParser.java index de3c4b11ec..d922449d20 100755 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/MimeJ4MessageParser.java +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/MimeJ4MessageParser.java @@ -1,7 +1,20 @@ /* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. + * Autopsy Forensic Browser + * + * Copyright 2019 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.sleuthkit.autopsy.thunderbirdparser; @@ -24,7 +37,9 @@ import org.apache.james.mime4j.dom.address.Mailbox; import org.apache.james.mime4j.dom.address.MailboxList; import org.apache.james.mime4j.dom.field.ContentDispositionField; import org.apache.james.mime4j.dom.field.ContentTypeField; +import org.apache.james.mime4j.message.DefaultMessageBuilder; import org.apache.james.mime4j.stream.Field; +import org.apache.james.mime4j.stream.MimeConfig; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; import org.sleuthkit.autopsy.coreutils.FileUtil; @@ -33,25 +48,83 @@ import org.sleuthkit.datamodel.EncodedFileOutputStream; import org.sleuthkit.datamodel.TskData; /** - * - * @author kelly + * Super class for email parsers that can use the james.mime4J.Message objects. */ -public abstract class MimeJ4MessageParser { +abstract class MimeJ4MessageParser { private static final Logger logger = Logger.getLogger(MimeJ4MessageParser.class.getName()); /** * The mime type string for html text. */ - private static final String HTML_TYPE = "text/html"; //NON-NLS + private DefaultMessageBuilder messageBuilder = null; + private final List errorList = new ArrayList<>(); + + /** + * The local path of the email message(s) file. + */ + private String localPath; + + DefaultMessageBuilder getMessageBuilder() { + if(messageBuilder == null) { + messageBuilder = new DefaultMessageBuilder(); + MimeConfig config = MimeConfig.custom().setMaxLineLen(-1).setMaxHeaderLen(-1).setMaxHeaderCount(-1).build(); + // disable line length checks. + messageBuilder.setMimeEntityConfig(config); + } + + return messageBuilder; + } + + /** + * Sets the local path of the email messages file. + * + * @param localPath Local path of the file the email messages + */ + final void setLocalPath(String localPath) { + this.localPath = localPath; + } + + /** + * Gets the local path. + * + * @return + */ + String getLocalPath() { + return localPath; + } + + /** + * Get a list of the parsing error message. + * + * @return String containing all of the parse error message. Empty string + * is returned if there are no error messages. + */ + String getErrors() { + String result = ""; + for (String msg: errorList) { + result += "
  • " + msg + "
    • "; + } + return result; + } + + /** + * Adds a message to the error Message list. + * + * @param msg Message to add to the list. + */ + void addErrorMessage(String msg) { + errorList.add(msg); + } + /** * Use the information stored in the given mime4j message to populate an * EmailMessage. * - * @param msg + * @param msg The Message to extract data from. * - * @return + * @return EmailMessage for the Message. */ EmailMessage extractEmail(Message msg, String localPath, long sourceFileID) { EmailMessage email = new EmailMessage(); @@ -215,7 +288,7 @@ public abstract class MimeJ4MessageParser { * @param email * @param e */ - @NbBundle.Messages({"EMLParser.handleAttch.noOpenCase.errMsg=Exception while getting open case."}) + @NbBundle.Messages({"MimeJ4MessageParser.handleAttch.noOpenCase.errMsg=Exception while getting open case."}) private static void handleAttachment(EmailMessage email, Entity e, long fileID, int index) { String outputDirPath; String relModuleOutputPath; @@ -223,7 +296,7 @@ public abstract class MimeJ4MessageParser { outputDirPath = ThunderbirdMboxFileIngestModule.getModuleOutputPath() + File.separator; relModuleOutputPath = ThunderbirdMboxFileIngestModule.getRelModuleOutputPath() + File.separator; } catch (NoCurrentCaseException ex) { -// logger.log(Level.SEVERE, Bundle.MboxParser_handleAttch_noOpenCase_errMsg(), ex); //NON-NLS + logger.log(Level.SEVERE, Bundle.MimeJ4MessageParser_handleAttch_noOpenCase_errMsg(), ex); //NON-NLS return; } String filename = FileUtil.escapeFileName(e.getFilename()); @@ -278,7 +351,8 @@ public abstract class MimeJ4MessageParser { * * @param mailboxList * - * @return + * @return String list of email addresses separated by a ; or empty string + * if no addresses were found. */ private static String getAddresses(MailboxList mailboxList) { if (mailboxList == null) { @@ -297,7 +371,8 @@ public abstract class MimeJ4MessageParser { * * @param addressList * - * @return + * @return String list of email addresses separated by a ; or empty string + * if no addresses were found. */ private static String getAddresses(AddressList addressList) { return (addressList == null) ? "" : getAddresses(addressList.flatten()); From dd6f0ba0a3ef6bbceefbff98013302ad1ded3049 Mon Sep 17 00:00:00 2001 From: Mark McKinnon Date: Mon, 26 Aug 2019 21:25:20 -0400 Subject: [PATCH 030/102] address comments Address comments, --- .../datasourcesummary/DataSourceSummaryDetailsPanel.form | 2 +- CoreLibs/src/org/sleuthkit/autopsy/corelibs/OpenCvLoader.java | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/datasourcesummary/DataSourceSummaryDetailsPanel.form b/Core/src/org/sleuthkit/autopsy/casemodule/datasourcesummary/DataSourceSummaryDetailsPanel.form index 80a8a890c7..1924193213 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/datasourcesummary/DataSourceSummaryDetailsPanel.form +++ b/Core/src/org/sleuthkit/autopsy/casemodule/datasourcesummary/DataSourceSummaryDetailsPanel.form @@ -40,7 +40,7 @@ - + diff --git a/CoreLibs/src/org/sleuthkit/autopsy/corelibs/OpenCvLoader.java b/CoreLibs/src/org/sleuthkit/autopsy/corelibs/OpenCvLoader.java index 0a46afd8f9..79e4678e91 100644 --- a/CoreLibs/src/org/sleuthkit/autopsy/corelibs/OpenCvLoader.java +++ b/CoreLibs/src/org/sleuthkit/autopsy/corelibs/OpenCvLoader.java @@ -28,6 +28,7 @@ import org.opencv.core.Core; */ public final class OpenCvLoader { + // Uses java logger since the Autopsy class logger (Autopsy-core) is not part of this module private static final Logger logger = Logger.getLogger(OpenCvLoader.class.getName()); private static boolean openCvLoaded; private static UnsatisfiedLinkError exception = null; // Deprecated From c7207f575c106f0d16685e8df7232cbd4bf4065e Mon Sep 17 00:00:00 2001 From: Mark McKinnon Date: Mon, 26 Aug 2019 22:09:25 -0400 Subject: [PATCH 031/102] Update TikaTextExtractor.java Add comment why it uses java logger and not Autopsy logger --- .../org/sleuthkit/autopsy/textextractors/TikaTextExtractor.java | 1 + 1 file changed, 1 insertion(+) diff --git a/Core/src/org/sleuthkit/autopsy/textextractors/TikaTextExtractor.java b/Core/src/org/sleuthkit/autopsy/textextractors/TikaTextExtractor.java index 2d923cc719..a8bf0591fb 100644 --- a/Core/src/org/sleuthkit/autopsy/textextractors/TikaTextExtractor.java +++ b/Core/src/org/sleuthkit/autopsy/textextractors/TikaTextExtractor.java @@ -136,6 +136,7 @@ final class TikaTextExtractor implements TextExtractor { "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", //NON-NLS "application/pdf"); //NON-NLS + // Used to log to the tika file that is why it uses the java.util.logging.logger class instead of the Autopsy one private static final java.util.logging.Logger TIKA_LOGGER = java.util.logging.Logger.getLogger("Tika"); //NON-NLS private static final Logger AUTOPSY_LOGGER = Logger.getLogger(TikaTextExtractor.class.getName()); From 5e19718065668faa11cdfa32565d5ed7b6829bb3 Mon Sep 17 00:00:00 2001 From: Kelly Kelly Date: Tue, 27 Aug 2019 13:17:11 -0400 Subject: [PATCH 032/102] Changed vcard & eml parse code to use ReadContentInputStream --- .../autopsy/thunderbirdparser/EMLParser.java | 48 +++++----- .../MimeJ4MessageParser.java | 57 ++++++------ .../ThunderbirdMboxFileIngestModule.java | 88 +++---------------- .../thunderbirdparser/VcardParser.java | 5 +- 4 files changed, 68 insertions(+), 130 deletions(-) diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/EMLParser.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/EMLParser.java index eea1b992e7..cf37d9ee36 100755 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/EMLParser.java +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/EMLParser.java @@ -18,57 +18,57 @@ */ package org.sleuthkit.autopsy.thunderbirdparser; -import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; import org.apache.james.mime4j.MimeException; import org.apache.james.mime4j.dom.Message; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.datamodel.ReadContentInputStream; /** - * EML file parser. An .eml file contains a single email message. - * + * EML file parser. An .eml file contains a single email message. + * */ -public class EMLParser extends MimeJ4MessageParser{ - +class EMLParser extends MimeJ4MessageParser { + private static final Logger logger = Logger.getLogger(EMLParser.class.getName()); - + /** - * If the extention of the AbstractFile is eml and 'To:' is found close to the - * beginning of the file, then its probably an eml file. - * + * If the extention of the AbstractFile is eml and 'To:' is found close to + * the beginning of the file, then its probably an eml file. + * * @param abFile AbstractFile to test - * @param buffer A byte buffer of the begining of the file. - * - * @return True, if we think this is an eml file, false otherwise. + * @param buffer A byte buffer of the beginning of the file. + * + * @return True, if we think this is an eml file, false otherwise. */ static boolean isEMLFile(AbstractFile abFile, byte[] buffer) { String ext = abFile.getNameExtension(); boolean isEMLFile = ext != null ? ext.equals("eml") : false; - if(isEMLFile) { - isEMLFile = (new String(buffer)).contains("To:"); //NON-NLS + if (isEMLFile) { + isEMLFile = (new String(buffer)).contains("To:"); //NON-NLS } - + return isEMLFile; } - + /** - * + * * @param sourceFile AbstractFile source file for eml message - * @param localPath The local path to the eml file - * + * @param localPath The local path to the eml file + * * @return EmailMessage object for message in eml file - * + * * @throws FileNotFoundException * @throws IOException - * @throws MimeException + * @throws MimeException */ - static EmailMessage parse(AbstractFile sourceFile, String localPath) throws FileNotFoundException, IOException, MimeException { - try (FileInputStream fis = new FileInputStream(localPath)){ + static EmailMessage parse(AbstractFile sourceFile) throws FileNotFoundException, IOException, MimeException { + try (ReadContentInputStream fis = new ReadContentInputStream(sourceFile)) { EMLParser parser = new EMLParser(); Message mimeMsg = parser.getMessageBuilder().parseMessage(fis); - return parser.extractEmail(mimeMsg, localPath, sourceFile.getId()); + return parser.extractEmail(mimeMsg, "", sourceFile.getId()); } } } diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/MimeJ4MessageParser.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/MimeJ4MessageParser.java index d922449d20..14482008a9 100755 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/MimeJ4MessageParser.java +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/MimeJ4MessageParser.java @@ -51,74 +51,74 @@ import org.sleuthkit.datamodel.TskData; * Super class for email parsers that can use the james.mime4J.Message objects. */ abstract class MimeJ4MessageParser { - private static final Logger logger = Logger.getLogger(MimeJ4MessageParser.class.getName()); - + + private static final Logger logger = Logger.getLogger(MimeJ4MessageParser.class.getName()); + /** * The mime type string for html text. */ private static final String HTML_TYPE = "text/html"; //NON-NLS private DefaultMessageBuilder messageBuilder = null; private final List errorList = new ArrayList<>(); - + /** * The local path of the email message(s) file. */ private String localPath; - + DefaultMessageBuilder getMessageBuilder() { - if(messageBuilder == null) { + if (messageBuilder == null) { messageBuilder = new DefaultMessageBuilder(); MimeConfig config = MimeConfig.custom().setMaxLineLen(-1).setMaxHeaderLen(-1).setMaxHeaderCount(-1).build(); // disable line length checks. messageBuilder.setMimeEntityConfig(config); } - + return messageBuilder; } - + /** * Sets the local path of the email messages file. - * + * * @param localPath Local path of the file the email messages */ final void setLocalPath(String localPath) { this.localPath = localPath; } - + /** * Gets the local path. - * - * @return + * + * @return */ String getLocalPath() { return localPath; } - + /** * Get a list of the parsing error message. - * - * @return String containing all of the parse error message. Empty string - * is returned if there are no error messages. + * + * @return String containing all of the parse error message. Empty string is + * returned if there are no error messages. */ String getErrors() { String result = ""; - for (String msg: errorList) { - result += "
  • " + msg + "
    • "; + for (String msg : errorList) { + result += "
  • " + msg + "
    • "; } return result; } - + /** * Adds a message to the error Message list. - * + * * @param msg Message to add to the list. */ void addErrorMessage(String msg) { errorList.add(msg); } - - /** + /** * Use the information stored in the given mime4j message to populate an * EmailMessage. * @@ -169,8 +169,7 @@ abstract class MimeJ4MessageParser { return email; } - - + /** * Extract the subject, inReplyTo, message-ID and references from the * Message object and returns them in a new EmailMessage object. @@ -208,8 +207,8 @@ abstract class MimeJ4MessageParser { return email; } - - /** + + /** * Handle a multipart mime message. Recursively calls handleMultipart if one * of the body parts is another multipart. Otherwise, calls the correct * method to extract information out of each part of the body. @@ -344,15 +343,15 @@ abstract class MimeJ4MessageParser { attach.setEncodingType(TskData.EncodingType.XOR1); email.addAttachment(attach); } - - /** + + /** * Get a String representation of the MailboxList (which is a list of email * addresses). * * @param mailboxList * * @return String list of email addresses separated by a ; or empty string - * if no addresses were found. + * if no addresses were found. */ private static String getAddresses(MailboxList mailboxList) { if (mailboxList == null) { @@ -372,7 +371,7 @@ abstract class MimeJ4MessageParser { * @param addressList * * @return String list of email addresses separated by a ; or empty string - * if no addresses were found. + * if no addresses were found. */ private static String getAddresses(AddressList addressList) { return (addressList == null) ? "" : getAddresses(addressList.flatten()); diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java index 5f00d96bb5..f428b455cf 100644 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java @@ -133,7 +133,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { return processMBox(abstractFile); } - if(isEMLFile) { + if (isEMLFile) { return processEMLFile(abstractFile); } @@ -318,93 +318,31 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { "ThunderbirdMboxFileIngestModule.errorMessage.outOfDiskSpace=Out of disk space. Cannot copy '{0}' (id={1}) to parse." }) private ProcessResult processVcard(AbstractFile abstractFile) { - String fileName; - try { - fileName = getTempPath() + File.separator + abstractFile.getName() - + "-" + String.valueOf(abstractFile.getId()); - } catch (NoCurrentCaseException ex) { - logger.log(Level.SEVERE, "Exception while getting open case.", ex); //NON-NLS - return ProcessResult.ERROR; - } - File file = new File(fileName); - - long freeSpace = services.getFreeDiskSpace(); - if ((freeSpace != IngestMonitor.DISK_FREE_SPACE_UNKNOWN) && (abstractFile.getSize() >= freeSpace)) { - logger.log(Level.WARNING, String.format("Not enough disk space to write file '%s' (id=%d) to disk.", - abstractFile.getName(), abstractFile.getId())); //NON-NLS - IngestMessage msg = IngestMessage.createErrorMessage(EmailParserModuleFactory.getModuleName(), EmailParserModuleFactory.getModuleName(), - Bundle.ThunderbirdMboxFileIngestModule_errorMessage_outOfDiskSpace(abstractFile.getName(), abstractFile.getId())); - services.postMessage(msg); - return ProcessResult.OK; - } - - try { - ContentUtils.writeToFile(abstractFile, file, context::fileIngestIsCancelled); - } catch (IOException ex) { - logger.log(Level.WARNING, String.format("Failed writing the vCard file '%s' (id=%d) to disk.", - abstractFile.getName(), abstractFile.getId()), ex); //NON-NLS - return ProcessResult.OK; - } - try { VcardParser parser = new VcardParser(currentCase, context); - parser.parse(file, abstractFile); + parser.parse(abstractFile); } catch (IOException | NoCurrentCaseException ex) { - logger.log(Level.WARNING, String.format("Exception while parsing the file '%s' (id=%d).", file.getName(), abstractFile.getId()), ex); //NON-NLS + logger.log(Level.WARNING, String.format("Exception while parsing the file '%s' (id=%d).", abstractFile.getName(), abstractFile.getId()), ex); //NON-NLS return ProcessResult.OK; } - - if (file.delete() == false) { - logger.log(Level.INFO, "Failed to delete temp file: {0}", file.getName()); //NON-NLS - } - return ProcessResult.OK; } - private ProcessResult processEMLFile(AbstractFile abstractFile) { - String fileName; + private ProcessResult processEMLFile(AbstractFile abstractFile) { try { - fileName = getTempPath() + File.separator + abstractFile.getName() - + "-" + String.valueOf(abstractFile.getId()); - } catch (NoCurrentCaseException ex) { - logger.log(Level.SEVERE, "Exception while getting open case.", ex); //NON-NLS - return ProcessResult.ERROR; - } - File file = new File(fileName); + EmailMessage message = EMLParser.parse(abstractFile); - long freeSpace = services.getFreeDiskSpace(); - if ((freeSpace != IngestMonitor.DISK_FREE_SPACE_UNKNOWN) && (abstractFile.getSize() >= freeSpace)) { - logger.log(Level.WARNING, String.format("Not enough disk space to write file '%s' (id=%d) to disk.", - abstractFile.getName(), abstractFile.getId())); //NON-NLS -// IngestMessage msg = IngestMessage.createErrorMessage(EmailParserModuleFactory.getModuleName(), EmailParserModuleFactory.getModuleName(), -// Bundle.ThunderbirdMboxFileIngestModule_errorMessage_outOfDiskSpace(abstractFile.getName(), abstractFile.getId())); -// services.postMessage(msg); - return ProcessResult.OK; - } - - try { - ContentUtils.writeToFile(abstractFile, file, context::fileIngestIsCancelled); - } catch (IOException ex) { - logger.log(Level.WARNING, String.format("Failed writing the vCard file '%s' (id=%d) to disk.", - abstractFile.getName(), abstractFile.getId()), ex); //NON-NLS - return ProcessResult.OK; - } - - try { - EmailMessage message = EMLParser.parse(abstractFile, fileName); - - if(message == null) { + if (message == null) { return ProcessResult.OK; } - + List derivedFiles = new ArrayList<>(); BlackboardArtifact msgArtifact = addEmailArtifact(message, abstractFile); - - if ((msgArtifact != null) && (message.hasAttachment())) { - derivedFiles.addAll(handleAttachments(message.getAttachments(), abstractFile, msgArtifact )); + + if ((msgArtifact != null) && (message.hasAttachment())) { + derivedFiles.addAll(handleAttachments(message.getAttachments(), abstractFile, msgArtifact)); } - if (derivedFiles.isEmpty() == false) { for (AbstractFile derived : derivedFiles) { @@ -412,7 +350,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { } } context.addFilesToJob(derivedFiles); - + } catch (IOException ex) { logger.log(Level.WARNING, String.format("Error reading eml file %s", abstractFile.getName()), ex); return ProcessResult.ERROR; @@ -420,7 +358,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { logger.log(Level.WARNING, String.format("Error reading eml file %s", abstractFile.getName()), ex); return ProcessResult.ERROR; } - + return ProcessResult.OK; } @@ -651,7 +589,7 @@ public final class ThunderbirdMboxFileIngestModule implements FileIngestModule { addArtifactAttribute(((id < 0L) ? NbBundle.getMessage(this.getClass(), "ThunderbirdMboxFileIngestModule.notAvail") : String.valueOf(id)), ATTRIBUTE_TYPE.TSK_MSG_ID, bbattributes); - addArtifactAttribute(((localPath.isEmpty() == false) ? localPath : "/foo/bar"), + addArtifactAttribute(((localPath.isEmpty() == false) ? localPath : ""), ATTRIBUTE_TYPE.TSK_PATH, bbattributes); addArtifactAttribute(cc, ATTRIBUTE_TYPE.TSK_EMAIL_CC, bbattributes); diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/VcardParser.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/VcardParser.java index facbce4794..84f4cd92c8 100755 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/VcardParser.java +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/VcardParser.java @@ -57,6 +57,7 @@ import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardAttribute; import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.DataSource; +import org.sleuthkit.datamodel.ReadContentInputStream; import org.sleuthkit.datamodel.Relationship; import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.TskCoreException; @@ -141,8 +142,8 @@ final class VcardParser { * file. * @throws NoCurrentCaseException If there is no open case. */ - void parse(File vcardFile, AbstractFile abstractFile) throws IOException, NoCurrentCaseException { - for (VCard vcard: Ezvcard.parse(vcardFile).all()) { + void parse(AbstractFile abstractFile) throws IOException, NoCurrentCaseException { + for (VCard vcard: Ezvcard.parse(new ReadContentInputStream(abstractFile)).all()) { addContactArtifact(vcard, abstractFile); } } From ccd387bb562fa1892a2f0f1081006b585b2b2b1b Mon Sep 17 00:00:00 2001 From: Mark McKinnon Date: Tue, 27 Aug 2019 15:32:42 -0400 Subject: [PATCH 033/102] Changed logging to use Autopsy logger Changed logging to use Autopsy Logger from java logger --- .../autopsy/report/PortableCaseInterestingItemsListPanel.java | 2 +- .../org/sleuthkit/autopsy/report/PortableCaseTagsListPanel.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/report/PortableCaseInterestingItemsListPanel.java b/Core/src/org/sleuthkit/autopsy/report/PortableCaseInterestingItemsListPanel.java index afd49dbe33..da3a45db07 100644 --- a/Core/src/org/sleuthkit/autopsy/report/PortableCaseInterestingItemsListPanel.java +++ b/Core/src/org/sleuthkit/autopsy/report/PortableCaseInterestingItemsListPanel.java @@ -206,7 +206,7 @@ class PortableCaseInterestingItemsListPanel extends javax.swing.JPanel { */ private static class GetInterestingItemSetNamesCallback implements CaseDbAccessManager.CaseDbAccessQueryCallback { - private static final java.util.logging.Logger logger = java.util.logging.Logger.getLogger(GetInterestingItemSetNamesCallback.class.getName()); + private static final Logger logger = Logger.getLogger(GetInterestingItemSetNamesCallback.class.getName()); private final Map setCounts = new HashMap<>(); @Override diff --git a/Core/src/org/sleuthkit/autopsy/report/PortableCaseTagsListPanel.java b/Core/src/org/sleuthkit/autopsy/report/PortableCaseTagsListPanel.java index db40932835..af2fc729ad 100644 --- a/Core/src/org/sleuthkit/autopsy/report/PortableCaseTagsListPanel.java +++ b/Core/src/org/sleuthkit/autopsy/report/PortableCaseTagsListPanel.java @@ -205,7 +205,7 @@ class PortableCaseTagsListPanel extends javax.swing.JPanel { */ static class GetTagCountsCallback implements CaseDbAccessManager.CaseDbAccessQueryCallback { - private static final java.util.logging.Logger logger = java.util.logging.Logger.getLogger(GetTagCountsCallback.class.getName()); + private static final Logger logger = Logger.getLogger(GetTagCountsCallback.class.getName()); private final Map tagCounts = new HashMap<>(); @Override From fe3eeb77dca047fcc7d722e49d237295bd57e888 Mon Sep 17 00:00:00 2001 From: Joe Ho Date: Tue, 27 Aug 2019 16:28:30 -0400 Subject: [PATCH 034/102] Initial changes --- .../dsp/AddLogicalImageTask.java | 117 +++++++++-- .../dsp/AddMultipleImageTask.java | 181 +++++++++++------- .../dsp/LogicalImagerDSProcessor.java | 26 ++- 3 files changed, 232 insertions(+), 92 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java index 48488dd82a..8d56b676cc 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java @@ -71,7 +71,11 @@ final class AddLogicalImageTask implements Runnable { private final Case currentCase; private volatile boolean cancelled; - + private boolean addingInterestingFiles; + private AddMultipleImageTask addMultipleImageTask; + private Thread multipleImageThread; + private boolean createVHD; + AddLogicalImageTask(String deviceId, String timeZone, File src, File dest, @@ -134,10 +138,6 @@ final class AddLogicalImageTask implements Runnable { return; } - if (cancelled) { - return; - } - progressMonitor.setProgressText(Bundle.AddLogicalImageTask_addingToReport(resultsFilename)); String status = addReport(Paths.get(dest.toString(), resultsFilename), resultsFilename + " " + src.getName()); if (status != null) { @@ -171,13 +171,13 @@ final class AddLogicalImageTask implements Runnable { } } - AddMultipleImageTask addMultipleImageTask = null; + addMultipleImageTask = null; + AddDataSourceCallback privateCallback = null; List newDataSources = new ArrayList<>(); - boolean createVHD; if (imagePaths.isEmpty()) { createVHD = false; - // No VHD in src directory, try ingest the root directory using Logical File Set + // No VHD in src directory, try ingest the root directory as local files File root = Paths.get(dest.toString(), ROOT_STR).toFile(); if (root.exists() && root.isDirectory()) { imagePaths.add(root.getAbsolutePath()); @@ -202,12 +202,10 @@ final class AddLogicalImageTask implements Runnable { createVHD = true; // ingest the VHDs try { - addMultipleImageTask = new AddMultipleImageTask(deviceId, imagePaths, timeZone , progressMonitor, callback); - addMultipleImageTask.run(); - if (addMultipleImageTask.getResult() == DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS) { - callback.done(addMultipleImageTask.getResult(), addMultipleImageTask.getErrorMessages(), addMultipleImageTask.getNewDataSources()); - return; - } + privateCallback = new AddDataSourceCallback(); + addMultipleImageTask = new AddMultipleImageTask(deviceId, imagePaths, timeZone , progressMonitor, privateCallback); + multipleImageThread = new Thread(addMultipleImageTask); + multipleImageThread.start(); } catch (NoCurrentCaseException ex) { String msg = Bundle.AddLogicalImageTask_noCurrentCase(); errorList.add(msg); @@ -217,11 +215,40 @@ final class AddLogicalImageTask implements Runnable { } try { + if (createVHD) { + // Wait for addMultipleImageTask to finish, via its privateCallback + while (privateCallback.inProgress()) { + try { + Thread.sleep(1000); + } catch (InterruptedException ex) { + // Got the addMultipleImageTask stop (cancel) + LOGGER.log(Level.INFO, "AddMultipleImageTask interrupted", ex); // NON-NLS + // now wait for addMultipleImageTask to revert and finally callback + while (privateCallback.inProgress()) { + try { + Thread.sleep(1000); + } catch (InterruptedException ex2) { + LOGGER.log(Level.INFO, "AddMultipleImageTask interrupted 2", ex2); // NON-NLS + } + } + } + } + // TODO: Delete destination directory when 5453 (VHD file is not closed upon revert) is fixed. + // if (cancelled) { + // deleteDestinationDirectory(); + // } + if (privateCallback.getResult() == DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS) { + // bait out + callback.done(privateCallback.getResult(), privateCallback.getErrorMessages(), privateCallback.getNewDataSources()); + return; + } + } progressMonitor.setProgressText(Bundle.AddLogicalImageTask_addingInterestingFiles()); + addingInterestingFiles = true; addInterestingFiles(dest, Paths.get(dest.toString(), resultsFilename), createVHD); progressMonitor.setProgressText(Bundle.AddLogicalImageTask_doneAddingInterestingFiles()); - if (addMultipleImageTask != null) { - callback.done(addMultipleImageTask.getResult(), addMultipleImageTask.getErrorMessages(), addMultipleImageTask.getNewDataSources()); + if (addMultipleImageTask != null && privateCallback != null) { + callback.done(privateCallback.getResult(), privateCallback.getErrorMessages(), privateCallback.getNewDataSources()); } else { callback.done(DataSourceProcessorCallback.DataSourceProcessorResult.NO_ERRORS, errorList, newDataSources); } @@ -265,6 +292,17 @@ final class AddLogicalImageTask implements Runnable { void cancelTask() { LOGGER.log(Level.WARNING, "AddLogicalImageTask cancelled, processing may be incomplete"); // NON-NLS cancelled = true; + if (addMultipleImageTask != null) { + multipleImageThread.interrupt(); + addMultipleImageTask.cancelTask(); + } + if (!createVHD) { + // Don't delete destination directory once we started adding interesting files. + // At this point the database and destination directory are complete. + if (!addingInterestingFiles) { + deleteDestinationDirectory(); + } + } } private Map imagePathsToDataSourceObjId(Map> imagePaths) { @@ -429,4 +467,51 @@ final class AddLogicalImageTask implements Runnable { } } + private boolean deleteDestinationDirectory() { + try { + FileUtils.deleteDirectory(dest); + LOGGER.log(Level.INFO, String.format("Cancellation: Deleted directory %s", dest.toString())); // NON-NLS + return true; + } catch (IOException ex) { + LOGGER.log(Level.WARNING, String.format("Cancellation: Failed to delete directory %s", dest.toString()), ex); // NON-NLS + return false; + } + } + + private class AddDataSourceCallback extends DataSourceProcessorCallback { + private List errorMessages; + private List newDataSources; + private DataSourceProcessorResult result; + private boolean inProgress = true; + + @Override + public void done(DataSourceProcessorCallback.DataSourceProcessorResult result, List errorMessages, List newDataSources) { + LOGGER.log(Level.INFO, "privateCallback done"); // NON-NLS + this.errorMessages = errorMessages; + this.newDataSources = newDataSources; + this.result = result; + this.inProgress = false; + } + + @Override + public void doneEDT(DataSourceProcessorResult result, List errorMessages, List newDataSources) { + done(result, errorMessages, newDataSources); + } + + public List getNewDataSources() { + return newDataSources; + } + + public List getErrorMessages() { + return errorMessages; + } + + public DataSourceProcessorResult getResult() { + return result; + } + + private boolean inProgress() { + return inProgress; + } + } } diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddMultipleImageTask.java b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddMultipleImageTask.java index 5bba555647..b27b1e266d 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddMultipleImageTask.java +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddMultipleImageTask.java @@ -21,6 +21,7 @@ package org.sleuthkit.autopsy.logicalimager.dsp; import java.util.ArrayList; import java.util.List; import java.util.logging.Level; +import javax.annotation.concurrent.GuardedBy; import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; @@ -56,13 +57,18 @@ class AddMultipleImageTask implements Runnable { private final DataSourceProcessorProgressMonitor progressMonitor; private final DataSourceProcessorCallback callback; private final Case currentCase; - private boolean criticalErrorOccurred; - private volatile boolean cancelled; + private SleuthkitJNI.CaseDbHandle.AddImageProcess addImageProcess = null; - private List newDataSources; - private List errorMessages; - private DataSourceProcessorResult result; + /* + * The cancellation requested flag and SleuthKit add image process are + * guarded by a lock to synchronize cancelling the process (setting the flag + * and calling its stop method) and calling either its commit or revert + * method. + */ + private final Object tskAddImageProcessLock; + @GuardedBy("tskAddImageProcessLock") + private boolean tskAddImageProcessStopped; /** * Constructs a runnable that adds multiple image files to a case database. @@ -95,36 +101,53 @@ class AddMultipleImageTask implements Runnable { this.progressMonitor = progressMonitor; currentCase = Case.getCurrentCaseThrows(); this.criticalErrorOccurred = false; - this.result = DataSourceProcessorResult.NO_ERRORS; + tskAddImageProcessLock = new Object(); } + @Messages({ + "AddMultipleImageTask.cancelled=Cancellation: Add image process reverted", + }) @Override public void run() { - newDataSources = new ArrayList<>(); - errorMessages = new ArrayList<>(); + List errorMessages = new ArrayList<>(); + List newDataSources = new ArrayList<>(); + List emptyDataSources = new ArrayList<>(); /* * Try to add the input image files as images. */ List corruptedImageFilePaths = new ArrayList<>(); - currentCase.getSleuthkitCase().acquireSingleUserCaseWriteLock(); try { + currentCase.getSleuthkitCase().acquireSingleUserCaseWriteLock(); progressMonitor.setIndeterminate(true); for (String imageFilePath : imageFilePaths) { - if (!cancelled) { - addImageToCase(imageFilePath, newDataSources, corruptedImageFilePaths, errorMessages); + synchronized (tskAddImageProcessLock) { + if (!tskAddImageProcessStopped) { + addImageProcess = currentCase.getSleuthkitCase().makeAddImageProcess(timeZone, false, false, ""); + } else { + return; + } + } + run(imageFilePath, corruptedImageFilePaths, errorMessages); + commitOrRevertAddImageProcess(imageFilePath, errorMessages, newDataSources); + synchronized (tskAddImageProcessLock) { + if (tskAddImageProcessStopped) { + errorMessages.add(Bundle.AddMultipleImageTask_cancelled()); + callback.done(DataSourceProcessorResult.CRITICAL_ERRORS, errorMessages, emptyDataSources); + return; + } } } } finally { currentCase.getSleuthkitCase().releaseSingleUserCaseWriteLock(); } - + /* * Try to add any input image files that did not have file systems as a * single an unallocated space file with the device id as the root virtual * directory name. */ - if (!cancelled && !corruptedImageFilePaths.isEmpty()) { + if (!tskAddImageProcessStopped && !corruptedImageFilePaths.isEmpty()) { SleuthkitCase caseDatabase; caseDatabase = currentCase.getSleuthkitCase(); try { @@ -146,7 +169,6 @@ class AddMultipleImageTask implements Runnable { start += TWO_GB; sequence++; } - } double leftoverSize = imageSize - sequence * TWO_GB; fileRanges.add(new TskFileRange(start, (long)leftoverSize, sequence)); @@ -170,6 +192,7 @@ class AddMultipleImageTask implements Runnable { /* * Pass the results back via the callback. */ + DataSourceProcessorCallback.DataSourceProcessorResult result; if (criticalErrorOccurred) { result = DataSourceProcessorResult.CRITICAL_ERRORS; } else if (!errorMessages.isEmpty()) { @@ -177,6 +200,7 @@ class AddMultipleImageTask implements Runnable { } else { result = DataSourceProcessorResult.NO_ERRORS; } + callback.done(result, errorMessages, newDataSources); } /** @@ -185,16 +209,30 @@ class AddMultipleImageTask implements Runnable { */ void cancelTask() { LOGGER.log(Level.WARNING, "AddMultipleImageTask cancelled, processing may be incomplete"); // NON-NLS - cancelled = true; + synchronized (tskAddImageProcessLock) { + tskAddImageProcessStopped = true; + if (addImageProcess != null) { + try { + /* + * All this does is set a flag that will make the TSK add + * image process exit when the flag is checked between + * processing steps. The state of the flag is not + * accessible, so record it here so that it is known that + * the revert method of the process object needs to be + * called. + */ + addImageProcess.stop(); + } catch (TskCoreException ex) { + LOGGER.log(Level.SEVERE, "Cancellation: addImagePRocess.stop failed", ex); // NON-NLS + } + } + } } /** * Attempts to add an input image to the case. * * @param imageFilePath The image file path. - * @param newDataSources If the image is added, a data source is - * added to this list for eventual return to - * the caller via the callback. * @param corruptedImageFilePaths If the image cannot be added because * Sleuth Kit cannot detect a filesystem, * the image file path is added to this list @@ -209,13 +247,11 @@ class AddMultipleImageTask implements Runnable { "# {0} - imageFilePath", "# {1} - deviceId", "# {2} - exceptionMessage", "AddMultipleImageTask.criticalErrorAdding=Critical error adding {0} for device {1}: {2}", "# {0} - imageFilePath", "# {1} - deviceId", "# {2} - exceptionMessage", "AddMultipleImageTask.criticalErrorReverting=Critical error reverting add image process for {0} for device {1}: {2}", "# {0} - imageFilePath", "# {1} - deviceId", "# {2} - exceptionMessage", "AddMultipleImageTask.nonCriticalErrorAdding=Non-critical error adding {0} for device {1}: {2}",}) - private void addImageToCase(String imageFilePath, List newDataSources, List corruptedImageFilePaths, List errorMessages) { + private void run(String imageFilePath, List corruptedImageFilePaths, List errorMessages) { /* * Try to add the image to the case database as a data source. */ progressMonitor.setProgressText(Bundle.AddMultipleImageTask_adding(imageFilePath)); - SleuthkitCase caseDatabase = currentCase.getSleuthkitCase(); - SleuthkitJNI.CaseDbHandle.AddImageProcess addImageProcess = caseDatabase.makeAddImageProcess(timeZone, false, false, ""); try { addImageProcess.run(deviceId, new String[]{imageFilePath}); } catch (TskCoreException ex) { @@ -231,59 +267,64 @@ class AddMultipleImageTask implements Runnable { errorMessages.add(Bundle.AddMultipleImageTask_criticalErrorAdding(imageFilePath, deviceId, ex.getLocalizedMessage())); criticalErrorOccurred = true; } - /* - * Either way, the add image process needs to be reverted. - */ - try { - addImageProcess.revert(); - } catch (TskCoreException e) { - errorMessages.add(Bundle.AddMultipleImageTask_criticalErrorReverting(imageFilePath, deviceId, e.getLocalizedMessage())); - criticalErrorOccurred = true; - } - return; } catch (TskDataException ex) { errorMessages.add(Bundle.AddMultipleImageTask_nonCriticalErrorAdding(imageFilePath, deviceId, ex.getLocalizedMessage())); } - - /* - * Try to commit the results of the add image process, retrieve the new - * image from the case database, and add it to the list of new data - * sources to be returned via the callback. - */ - try { - long imageId = addImageProcess.commit(); - Image dataSource = caseDatabase.getImageById(imageId); - newDataSources.add(dataSource); - - /* - * Verify the size of the new image. Note that it may not be what is - * expected, but at least part of it was added to the case. - */ - String verificationError = dataSource.verifyImageSize(); - if (!verificationError.isEmpty()) { - errorMessages.add(Bundle.AddMultipleImageTask_nonCriticalErrorAdding(imageFilePath, deviceId, verificationError)); + } + + /** + * Commits or reverts the results of the TSK add image process. If the + * process was stopped before it completed or there was a critical error the + * results are reverted, otherwise they are committed. + * + * @param imageFilePath The image file path. + * @param errorMessages Error messages, if any, are added to this list for + * eventual return via the callback. + * @param newDataSources If the new image is successfully committed, it is + * added to this list for eventual return via the + * callback. + */ + private void commitOrRevertAddImageProcess(String imageFilePath, List errorMessages, List newDataSources) { + synchronized (tskAddImageProcessLock) { + if (tskAddImageProcessStopped || criticalErrorOccurred) { + try { + addImageProcess.revert(); + } catch (TskCoreException ex) { + errorMessages.add(Bundle.AddMultipleImageTask_criticalErrorReverting(imageFilePath, deviceId, ex.getLocalizedMessage())); + criticalErrorOccurred = true; + } + return; + } + + if (!tskAddImageProcessStopped) { + /* + * Try to commit the results of the add image process, retrieve the new + * image from the case database, and add it to the list of new data + * sources to be returned via the callback. + */ + try { + long imageId = addImageProcess.commit(); + Image dataSource = currentCase.getSleuthkitCase().getImageById(imageId); + newDataSources.add(dataSource); + + /* + * Verify the size of the new image. Note that it may not be what is + * expected, but at least part of it was added to the case. + */ + String verificationError = dataSource.verifyImageSize(); + if (!verificationError.isEmpty()) { + errorMessages.add(Bundle.AddMultipleImageTask_nonCriticalErrorAdding(imageFilePath, deviceId, verificationError)); + } + } catch (TskCoreException ex) { + /* + * The add image process commit failed or querying the case database + * for the newly added image failed. Either way, this is a critical + * error. + */ + errorMessages.add(Bundle.AddMultipleImageTask_criticalErrorAdding(imageFilePath, deviceId, ex.getLocalizedMessage())); + criticalErrorOccurred = true; + } } - } catch (TskCoreException ex) { - /* - * The add image process commit failed or querying the case database - * for the newly added image failed. Either way, this is a critical - * error. - */ - errorMessages.add(Bundle.AddMultipleImageTask_criticalErrorAdding(imageFilePath, deviceId, ex.getLocalizedMessage())); - criticalErrorOccurred = true; } } - - public List getNewDataSources() { - return newDataSources; - } - - public List getErrorMessages() { - return errorMessages; - } - - public DataSourceProcessorResult getResult() { - return result; - } - } diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/LogicalImagerDSProcessor.java b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/LogicalImagerDSProcessor.java index 058c0fa2ee..c5753d9240 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/LogicalImagerDSProcessor.java +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/LogicalImagerDSProcessor.java @@ -25,8 +25,9 @@ import java.util.ArrayList; import java.util.Calendar; import java.util.List; import java.util.UUID; +import javax.swing.JOptionPane; +import static javax.swing.JOptionPane.YES_OPTION; import javax.swing.JPanel; -import org.apache.commons.io.FileUtils; import org.openide.util.NbBundle.Messages; import org.openide.util.lookup.ServiceProvider; import org.openide.util.lookup.ServiceProviders; @@ -51,6 +52,7 @@ public final class LogicalImagerDSProcessor implements DataSourceProcessor { private static final String LOGICAL_IMAGER_DIR = "LogicalImager"; //NON-NLS private final LogicalImagerPanel configPanel; private AddLogicalImageTask addLogicalImageTask; + private Thread thread; /* * Constructs a Logical Imager data source processor that implements the @@ -163,10 +165,20 @@ public final class LogicalImagerDSProcessor implements DataSourceProcessor { File dest = Paths.get(logicalImagerDir.toString(), imageDirPath.getFileName().toString()).toFile(); if (dest.exists()) { // Destination directory already exists - String msg = Bundle.LogicalImagerDSProcessor_directoryAlreadyExists(dest.toString()); - errorList.add(msg); - callback.done(DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS, errorList, emptyDataSources); - return; + int showConfirmDialog = JOptionPane.showConfirmDialog(configPanel, + String.format("The logical imager folder %s already exists,\ndo you want to add it again using a new folder name?", dest.toString()), + "Destination directory confirmation", + JOptionPane.YES_NO_OPTION); + if (showConfirmDialog == YES_OPTION) { + // Get unique dest directory + String uniqueDirectory = imageDirPath.getFileName() + "_" + UUID.randomUUID(); + dest = Paths.get(logicalImagerDir.toString(), uniqueDirectory).toFile(); + } else { + String msg = Bundle.LogicalImagerDSProcessor_directoryAlreadyExists(dest.toString()); + errorList.add(msg); + callback.done(DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS, errorList, emptyDataSources); + return; + } } File src = imageDirPath.toFile(); @@ -207,12 +219,14 @@ public final class LogicalImagerDSProcessor implements DataSourceProcessor { ) throws NoCurrentCaseException { addLogicalImageTask = new AddLogicalImageTask(deviceId, timeZone, src, dest, progressMonitor, callback); - new Thread(addLogicalImageTask).start(); + thread = new Thread(addLogicalImageTask); + thread.start(); } @Override public void cancel() { if (addLogicalImageTask != null) { + thread.interrupt(); addLogicalImageTask.cancelTask(); } } From 21a096ce0e2e9fb279e936cad57aa5013be10ee8 Mon Sep 17 00:00:00 2001 From: Joe Ho Date: Tue, 27 Aug 2019 17:35:55 -0400 Subject: [PATCH 035/102] Fix messages --- .../autopsy/logicalimager/dsp/LogicalImagerDSProcessor.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/LogicalImagerDSProcessor.java b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/LogicalImagerDSProcessor.java index c5753d9240..706a1be47c 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/LogicalImagerDSProcessor.java +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/LogicalImagerDSProcessor.java @@ -132,6 +132,8 @@ public final class LogicalImagerDSProcessor implements DataSourceProcessor { "# {0} - imageDirPath", "LogicalImagerDSProcessor.imageDirPathNotFound={0} not found.\nUSB drive has been ejected.", "# {0} - directory", "LogicalImagerDSProcessor.failToCreateDirectory=Failed to create directory {0}", "# {0} - directory", "LogicalImagerDSProcessor.directoryAlreadyExists=Directory {0} already exists", + "LogicalImagerDSProcessor.destinationDirectoryConfirmation=Destination directory confirmation", + "# {0} - directory", "LogicalImagerDSProcessor.destinationDirectoryConfirmationMsg=The logical imager folder %s already exists,\ndo you want to add it again using a new folder name?", "LogicalImagerDSProcessor.noCurrentCase=No current case", }) @Override @@ -166,8 +168,8 @@ public final class LogicalImagerDSProcessor implements DataSourceProcessor { if (dest.exists()) { // Destination directory already exists int showConfirmDialog = JOptionPane.showConfirmDialog(configPanel, - String.format("The logical imager folder %s already exists,\ndo you want to add it again using a new folder name?", dest.toString()), - "Destination directory confirmation", + Bundle.LogicalImagerDSProcessor_destinationDirectoryConfirmationMsg(dest.toString()), + Bundle.LogicalImagerDSProcessor_destinationDirectoryConfirmation(), JOptionPane.YES_NO_OPTION); if (showConfirmDialog == YES_OPTION) { // Get unique dest directory From a1d9c7c1262753f6ba4276bf43a954db64fd9126 Mon Sep 17 00:00:00 2001 From: Kelly Kelly Date: Wed, 28 Aug 2019 10:40:37 -0400 Subject: [PATCH 036/102] Added localPath for eml files --- .../sleuthkit/autopsy/thunderbirdparser/Bundle.properties-MERGED | 1 - .../src/org/sleuthkit/autopsy/thunderbirdparser/EMLParser.java | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/Bundle.properties-MERGED b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/Bundle.properties-MERGED index aa01a19072..cdfd241886 100755 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/Bundle.properties-MERGED +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/Bundle.properties-MERGED @@ -1,4 +1,3 @@ -MboxParser.handleAttch.noOpenCase.errMsg=Exception while getting open case. MimeJ4MessageParser.handleAttch.noOpenCase.errMsg=Exception while getting open case. OpenIDE-Module-Display-Category=Ingest Module OpenIDE-Module-Long-Description=Email Parser ingest module.\n\nThe module extracts MBOX and PST e-mail files and posts the results to the blackboard.\nIt knows about the Thunderbird folder structure for MBOX files. diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/EMLParser.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/EMLParser.java index cf37d9ee36..f7f456dd50 100755 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/EMLParser.java +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/EMLParser.java @@ -67,6 +67,7 @@ class EMLParser extends MimeJ4MessageParser { static EmailMessage parse(AbstractFile sourceFile) throws FileNotFoundException, IOException, MimeException { try (ReadContentInputStream fis = new ReadContentInputStream(sourceFile)) { EMLParser parser = new EMLParser(); + parser.setLocalPath(sourceFile.getParentPath()); Message mimeMsg = parser.getMessageBuilder().parseMessage(fis); return parser.extractEmail(mimeMsg, "", sourceFile.getId()); } From 9a65a9629bf44f0478b15aad111d28b86f8839d9 Mon Sep 17 00:00:00 2001 From: Joe Ho Date: Thu, 29 Aug 2019 10:15:33 -0400 Subject: [PATCH 037/102] Fix messages. Prevent currentPage going negative. Add wait cursor. --- .../autopsy/corecomponents/DataResultViewerTable.java | 11 ++++++++--- .../logicalimager/dsp/AddMultipleImageTask.java | 2 +- .../logicalimager/dsp/Bundle.properties-MERGED | 6 +++++- .../logicalimager/dsp/LogicalImagerDSProcessor.java | 2 +- 4 files changed, 15 insertions(+), 6 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/corecomponents/DataResultViewerTable.java b/Core/src/org/sleuthkit/autopsy/corecomponents/DataResultViewerTable.java index 71f801282b..1c3a371d9f 100644 --- a/Core/src/org/sleuthkit/autopsy/corecomponents/DataResultViewerTable.java +++ b/Core/src/org/sleuthkit/autopsy/corecomponents/DataResultViewerTable.java @@ -81,12 +81,12 @@ import org.sleuthkit.autopsy.corecomponentinterfaces.DataResultViewer; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; import org.sleuthkit.autopsy.coreutils.ThreadConfined; -import org.sleuthkit.autopsy.datamodel.NodeProperty; -import org.sleuthkit.autopsy.datamodel.NodeSelectionInfo; import org.sleuthkit.autopsy.datamodel.BaseChildFactory; import org.sleuthkit.autopsy.datamodel.BaseChildFactory.PageChangeEvent; import org.sleuthkit.autopsy.datamodel.BaseChildFactory.PageCountChangeEvent; import org.sleuthkit.autopsy.datamodel.BaseChildFactory.PageSizeChangeEvent; +import org.sleuthkit.autopsy.datamodel.NodeProperty; +import org.sleuthkit.autopsy.datamodel.NodeSelectionInfo; /** * A tabular result viewer that displays the children of the given root node @@ -880,18 +880,23 @@ public class DataResultViewerTable extends AbstractDataResultViewer { void nextPage() { currentPage++; + DataResultViewerTable.this.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR)); postPageChangeEvent(); + DataResultViewerTable.this.setCursor(null); } void previousPage() { currentPage--; + DataResultViewerTable.this.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR)); postPageChangeEvent(); + DataResultViewerTable.this.setCursor(null); } @NbBundle.Messages({"# {0} - totalPages", "DataResultViewerTable.goToPageTextField.msgDlg=Please enter a valid page number between 1 and {0}", "DataResultViewerTable.goToPageTextField.err=Invalid page number"}) void gotoPage() { + int saveCurrentPage = currentPage; try { currentPage = Integer.decode(gotoPageTextField.getText()); } catch (NumberFormatException e) { @@ -900,7 +905,7 @@ public class DataResultViewerTable extends AbstractDataResultViewer { } if (currentPage > totalPages || currentPage < 1) { - currentPage = 1; + currentPage = saveCurrentPage; JOptionPane.showMessageDialog(DataResultViewerTable.this, Bundle.DataResultViewerTable_goToPageTextField_msgDlg(totalPages), Bundle.DataResultViewerTable_goToPageTextField_err(), diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddMultipleImageTask.java b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddMultipleImageTask.java index b27b1e266d..681f7733da 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddMultipleImageTask.java +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddMultipleImageTask.java @@ -91,7 +91,7 @@ class AddMultipleImageTask implements Runnable { @Messages({ "# {0} - file", "AddMultipleImageTask.addingFileAsLogicalFile=Adding: {0} as an unallocated space file.", "# {0} - deviceId", "# {1} - exceptionMessage", - "AddMultipleImageTask.errorAddingImgWithoutFileSystem=Error adding images without file systems for device %s: %s",}) + "AddMultipleImageTask.errorAddingImgWithoutFileSystem=Error adding images without file systems for device {0}: {1}",}) AddMultipleImageTask(String deviceId, List imageFilePaths, String timeZone, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callback) throws NoCurrentCaseException { this.deviceId = deviceId; diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/Bundle.properties-MERGED index fd08147b08..0b4b6498cf 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/Bundle.properties-MERGED @@ -41,6 +41,7 @@ AddLogicalImageTask.notEnoughFields=File does not contain enough fields at line AddMultipleImageTask.adding=Adding: {0} # {0} - file AddMultipleImageTask.addingFileAsLogicalFile=Adding: {0} as an unallocated space file. +AddMultipleImageTask.cancelled=Cancellation: Add image process reverted # {0} - imageFilePath # {1} - deviceId # {2} - exceptionMessage @@ -51,13 +52,16 @@ AddMultipleImageTask.criticalErrorAdding=Critical error adding {0} for device {1 AddMultipleImageTask.criticalErrorReverting=Critical error reverting add image process for {0} for device {1}: {2} # {0} - deviceId # {1} - exceptionMessage -AddMultipleImageTask.errorAddingImgWithoutFileSystem=Error adding images without file systems for device %s: %s +AddMultipleImageTask.errorAddingImgWithoutFileSystem=Error adding images without file systems for device {0}: {1} AddMultipleImageTask.fsTypeUnknownErr=Cannot determine file system type # {0} - imageFilePath # {1} - deviceId # {2} - exceptionMessage AddMultipleImageTask.nonCriticalErrorAdding=Non-critical error adding {0} for device {1}: {2} LogicalImagerDSProcessor.dataSourceType=Autopsy Logical Imager Results +LogicalImagerDSProcessor.destinationDirectoryConfirmation=Destination directory confirmation +# {0} - directory +LogicalImagerDSProcessor.destinationDirectoryConfirmationMsg=The logical imager folder {0} already exists,\ndo you want to add it again using a new folder name? # {0} - directory LogicalImagerDSProcessor.directoryAlreadyExists=Directory {0} already exists # {0} - directory diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/LogicalImagerDSProcessor.java b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/LogicalImagerDSProcessor.java index 706a1be47c..4cd8ff162e 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/LogicalImagerDSProcessor.java +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/LogicalImagerDSProcessor.java @@ -133,7 +133,7 @@ public final class LogicalImagerDSProcessor implements DataSourceProcessor { "# {0} - directory", "LogicalImagerDSProcessor.failToCreateDirectory=Failed to create directory {0}", "# {0} - directory", "LogicalImagerDSProcessor.directoryAlreadyExists=Directory {0} already exists", "LogicalImagerDSProcessor.destinationDirectoryConfirmation=Destination directory confirmation", - "# {0} - directory", "LogicalImagerDSProcessor.destinationDirectoryConfirmationMsg=The logical imager folder %s already exists,\ndo you want to add it again using a new folder name?", + "# {0} - directory", "LogicalImagerDSProcessor.destinationDirectoryConfirmationMsg=The logical imager folder {0} already exists,\ndo you want to add it again using a new folder name?", "LogicalImagerDSProcessor.noCurrentCase=No current case", }) @Override From 13df8bf57c76ec752abbd64ba42e9971e58d46c1 Mon Sep 17 00:00:00 2001 From: Mark McKinnon Date: Thu, 29 Aug 2019 15:28:44 -0400 Subject: [PATCH 038/102] Update IngestSearchRunner.java Make it so the search status bar disapears when it is cancelled --- .../org/sleuthkit/autopsy/keywordsearch/IngestSearchRunner.java | 1 + 1 file changed, 1 insertion(+) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IngestSearchRunner.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IngestSearchRunner.java index cea9fd1a82..937c9567fd 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IngestSearchRunner.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IngestSearchRunner.java @@ -483,6 +483,7 @@ final class IngestSearchRunner { if (progressGroup != null) { progressGroup.setDisplayName(displayName + " " + NbBundle.getMessage(this.getClass(), "SearchRunner.doInBackGround.cancelMsg")); } + progressGroup.finish(); return IngestSearchRunner.Searcher.this.cancel(true); } }, null); From 93539197776de6dd0d9d963d2278dbb4413bcd22 Mon Sep 17 00:00:00 2001 From: Joe Ho Date: Fri, 30 Aug 2019 16:15:20 -0400 Subject: [PATCH 039/102] Initial changes --- .../configuration/Bundle.properties | 2 +- .../configuration/Bundle.properties-MERGED | 2 +- .../configuration/ConfigVisualPanel1.java | 61 ++++++++++++++++--- .../logicalimager/configuration/Kernel32.java | 56 +++++++++++++++++ 4 files changed, 111 insertions(+), 10 deletions(-) create mode 100644 Core/src/org/sleuthkit/autopsy/logicalimager/configuration/Kernel32.java diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/Bundle.properties b/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/Bundle.properties index 65fa3dd72d..fde2c2b893 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/Bundle.properties @@ -105,7 +105,7 @@ EditNonFullPathsRulePanel.minSizeCheckbox.text=Minimum size: NewRulePanel.chooseLabel.text=Choose the type of rule ConfigVisualPanel1.configureDriveRadioButton.text_1=Configure selected external drive: ConfigVisualPanel1.configureFolderRadioButton.text_1=Configure in a folder: -ConfigVisualPanel1.descriptionTextArea.text=Select a location for the Logical Imager. This location will contain the imaging program and a configuration file. If that location already contains a configuration file, it will be loaded to edit. Imaging results will be saved to this location, so ensure it has enough free space. +ConfigVisualPanel1.descriptionTextArea.text=Select a location for the Logical Imager. This location will contain the imaging program and a configuration file. If that location already contains a configuration file, it will be loaded to edit. Imaging results will be saved to this location, so ensure it has enough free space. Drives with FAT format are not supported. ConfigVisualPanel1.refreshButton.text=Refresh ConfigVisualPanel3.saveButton.text=Save ConfigVisualPanel3.configLabel.text=Logical Imager config file save status: diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/Bundle.properties-MERGED index 3e6aad34de..2216c2e69a 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/Bundle.properties-MERGED @@ -174,7 +174,7 @@ LogicalImagerConfigDeserializer.unsupportedKeyException=Unsupported key: {0} NewRulePanel.chooseLabel.text=Choose the type of rule ConfigVisualPanel1.configureDriveRadioButton.text_1=Configure selected external drive: ConfigVisualPanel1.configureFolderRadioButton.text_1=Configure in a folder: -ConfigVisualPanel1.descriptionTextArea.text=Select a location for the Logical Imager. This location will contain the imaging program and a configuration file. If that location already contains a configuration file, it will be loaded to edit. Imaging results will be saved to this location, so ensure it has enough free space. +ConfigVisualPanel1.descriptionTextArea.text=Select a location for the Logical Imager. This location will contain the imaging program and a configuration file. If that location already contains a configuration file, it will be loaded to edit. Imaging results will be saved to this location, so ensure it has enough free space. Drives with FAT format are not supported. ConfigVisualPanel1.refreshButton.text=Refresh ConfigVisualPanel3.saveButton.text=Save ConfigVisualPanel3.configLabel.text=Logical Imager config file save status: diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/ConfigVisualPanel1.java b/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/ConfigVisualPanel1.java index 2765bb1856..042906c0b3 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/ConfigVisualPanel1.java +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/ConfigVisualPanel1.java @@ -22,6 +22,8 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.JsonIOException; import com.google.gson.JsonSyntaxException; +import com.sun.jna.platform.win32.WinDef.DWORD; +import com.sun.jna.ptr.IntByReference; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; @@ -241,10 +243,51 @@ final class ConfigVisualPanel1 extends JPanel { firePropertyChange(UPDATE_UI_EVENT_NAME, false, true); // NON-NLS }//GEN-LAST:event_driveListMouseReleasedSelection + /* + * Return the Windows file system name of the drive + * @param drive File system drive, should be of the form "C:\" + * + */ + @Messages({"ConfigVisualPanel1.unknown=Unknown"}) + private String getFileSystemName(String drive){ + char[] lpVolumeNameBuffer = new char[256]; + DWORD nVolumeNameSize = new DWORD(256); + IntByReference lpVolumeSerialNumber = new IntByReference(); + IntByReference lpMaximumComponentLength = new IntByReference(); + IntByReference lpFileSystemFlags = new IntByReference(); + + char[] lpFileSystemNameBuffer = new char[256]; + DWORD nFileSystemNameSize = new DWORD(256); + + lpVolumeSerialNumber.setValue(0); + lpMaximumComponentLength.setValue(256); + lpFileSystemFlags.setValue(0); + + Kernel32.INSTANCE.GetVolumeInformation( + drive, + lpVolumeNameBuffer, + nVolumeNameSize, + lpVolumeSerialNumber, + lpMaximumComponentLength, + lpFileSystemFlags, + lpFileSystemNameBuffer, + nFileSystemNameSize); + if (Kernel32.INSTANCE.GetLastError() != 0) { + logger.log(Level.INFO, String.format("Last error: %d", Kernel32.INSTANCE.GetLastError())); // NON-NLS + return Bundle.ConfigVisualPanel1_unknown(); + } + + String fs = new String(lpFileSystemNameBuffer); + return fs.trim(); + } + /** * Refresh the list of local drives on the current machine */ - @Messages({"ConfigVisualPanel1.messageLabel.noExternalDriveFound=No drive found"}) + @NbBundle.Messages({ + "ConfigVisualPanel1.messageLabel.noExternalDriveFound=No drive found", + "ConfigVisualPanel1.fileSystem=File system" + }) private void refreshDriveList() { List listData = new ArrayList<>(); File[] roots = File.listRoots(); @@ -257,7 +300,8 @@ final class ConfigVisualPanel1 extends JPanel { String description = FileSystemView.getFileSystemView().getSystemTypeDescription(root); long spaceInBytes = root.getTotalSpace(); String sizeWithUnit = DriveListUtils.humanReadableByteCount(spaceInBytes, false); - listData.add(root + " (" + description + ") (" + sizeWithUnit + ")"); + String fileSystem = getFileSystemName(root.toString()); + listData.add(root + " (" + description + ") (" + sizeWithUnit + ") - " + Bundle.ConfigVisualPanel1_fileSystem() + ": " + fileSystem); if (firstRemovableDrive == -1) { try { FileStore fileStore = Files.getFileStore(root.toPath()); @@ -266,7 +310,7 @@ final class ConfigVisualPanel1 extends JPanel { } } catch (IOException ignored) { //unable to get this removable drive for default selection will try and select next removable drive by default - logger.log(Level.INFO, "Unable to select first removable drive found", ignored); + logger.log(Level.INFO, String.format("Unable to select first removable drive found %s", root.toString())); // NON-NLS } } i++; @@ -431,8 +475,7 @@ final class ConfigVisualPanel1 extends JPanel { return UPDATE_UI_EVENT_NAME; } - void setConfigFilename(String filename - ) { + void setConfigFilename(String filename) { configFileTextField.setText(filename); } @@ -442,9 +485,11 @@ final class ConfigVisualPanel1 extends JPanel { * @return true if panel has valid settings selected, false otherwise */ boolean isPanelValid() { - return !StringUtils.isBlank(getConfigPath()) && ((configureDriveRadioButton.isSelected() && !StringUtils.isBlank(driveList.getSelectedValue())) - || (configureFolderRadioButton.isSelected() && (!configFileTextField.getText().isEmpty()))); - + return !StringUtils.isBlank(getConfigPath()) + && (getFileSystemName(getConfigPath().substring(0, 3)).equals("NTFS") // NON-NLS + || getFileSystemName(getConfigPath().substring(0, 3)).equals("exFAT")) // NON-NLS + && ((configureDriveRadioButton.isSelected() && !StringUtils.isBlank(driveList.getSelectedValue())) + || (configureFolderRadioButton.isSelected() && (!configFileTextField.getText().isEmpty()))); } /** diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/Kernel32.java b/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/Kernel32.java new file mode 100644 index 0000000000..a3abed46f2 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/Kernel32.java @@ -0,0 +1,56 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ +package org.sleuthkit.autopsy.logicalimager.configuration; + +import com.sun.jna.Library; +import com.sun.jna.Native; +import com.sun.jna.platform.win32.WinDef.DWORD; +import com.sun.jna.ptr.IntByReference; +import com.sun.jna.win32.StdCallLibrary; +import com.sun.jna.win32.W32APIFunctionMapper; +import com.sun.jna.win32.W32APITypeMapper; +import java.util.HashMap; +import java.util.Map; + +public interface Kernel32 extends StdCallLibrary { + + final static Map WIN32API_OPTIONS = new HashMap() { + + private static final long serialVersionUID = 1L; + + { + put(Library.OPTION_FUNCTION_MAPPER, W32APIFunctionMapper.UNICODE); + put(Library.OPTION_TYPE_MAPPER, W32APITypeMapper.UNICODE); + } + }; + + public Kernel32 INSTANCE = (Kernel32) Native.loadLibrary("Kernel32", Kernel32.class, WIN32API_OPTIONS); + + /* + BOOL WINAPI GetVolumeInformation( + __in_opt LPCTSTR lpRootPathName, + __out LPTSTR lpVolumeNameBuffer, + __in DWORD nVolumeNameSize, + __out_opt LPDWORD lpVolumeSerialNumber, + __out_opt LPDWORD lpMaximumComponentLength, + __out_opt LPDWORD lpFileSystemFlags, + __out LPTSTR lpFileSystemNameBuffer, + __in DWORD nFileSystemNameSize + ); + */ + public boolean GetVolumeInformation( + String lpRootPathName, + char[] lpVolumeNameBuffer, + DWORD nVolumeNameSize, + IntByReference lpVolumeSerialNumber, + IntByReference lpMaximumComponentLength, + IntByReference lpFileSystemFlags, + char[] lpFileSystemNameBuffer, + DWORD nFileSystemNameSize + ); + + public int GetLastError(); +} From f680dddd345297d9b117336744a39a04e4f1fcf4 Mon Sep 17 00:00:00 2001 From: Joe Ho Date: Tue, 3 Sep 2019 10:17:05 -0400 Subject: [PATCH 040/102] Fix codacy errors --- .../configuration/Bundle.properties-MERGED | 2 ++ .../logicalimager/configuration/Kernel32.java | 32 +++++++++++++------ 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/Bundle.properties-MERGED index 2216c2e69a..55d0634c72 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/Bundle.properties-MERGED @@ -27,9 +27,11 @@ ConfigVisualPanel1.chooseFileTitle=Select a Logical Imager configuration ConfigVisualPanel1.configFileIsEmpty=Configuration file {0} is empty ConfigVisualPanel1.configurationError=Configuration error ConfigVisualPanel1.fileNameExtensionFilter=Configuration JSON File +ConfigVisualPanel1.fileSystem=File system ConfigVisualPanel1.invalidConfigJson=Invalid config JSON: ConfigVisualPanel1.messageLabel.noExternalDriveFound=No drive found ConfigVisualPanel1.selectConfigurationFile=Select location +ConfigVisualPanel1.unknown=Unknown ConfigVisualPanel2.cancel=Cancel ConfigVisualPanel2.deleteRuleSet=Delete rule ConfigVisualPanel2.deleteRuleSetConfirmation=Delete rule confirmation diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/Kernel32.java b/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/Kernel32.java index a3abed46f2..dcc7c44a19 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/Kernel32.java +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/Kernel32.java @@ -1,7 +1,20 @@ /* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. + * Autopsy + * + * Copyright 2019 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.sleuthkit.autopsy.logicalimager.configuration; @@ -15,19 +28,20 @@ import com.sun.jna.win32.W32APITypeMapper; import java.util.HashMap; import java.util.Map; +/* + * Windows Kernel32 interface + */ public interface Kernel32 extends StdCallLibrary { - final static Map WIN32API_OPTIONS = new HashMap() { - + Map WIN32API_OPTIONS = new HashMap() { private static final long serialVersionUID = 1L; - { put(Library.OPTION_FUNCTION_MAPPER, W32APIFunctionMapper.UNICODE); put(Library.OPTION_TYPE_MAPPER, W32APITypeMapper.UNICODE); } }; - public Kernel32 INSTANCE = (Kernel32) Native.loadLibrary("Kernel32", Kernel32.class, WIN32API_OPTIONS); + Kernel32 INSTANCE = (Kernel32) Native.loadLibrary("Kernel32", Kernel32.class, WIN32API_OPTIONS); /* BOOL WINAPI GetVolumeInformation( @@ -41,7 +55,7 @@ public interface Kernel32 extends StdCallLibrary { __in DWORD nFileSystemNameSize ); */ - public boolean GetVolumeInformation( + boolean GetVolumeInformation( String lpRootPathName, char[] lpVolumeNameBuffer, DWORD nVolumeNameSize, @@ -52,5 +66,5 @@ public interface Kernel32 extends StdCallLibrary { DWORD nFileSystemNameSize ); - public int GetLastError(); + int GetLastError(); } From 0480751945b7cbd8e6b2a4773f1ed33f5b1cd50c Mon Sep 17 00:00:00 2001 From: Joe Ho Date: Tue, 3 Sep 2019 13:23:41 -0400 Subject: [PATCH 041/102] Fix codacy error --- .../autopsy/logicalimager/configuration/Kernel32.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/Kernel32.java b/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/Kernel32.java index dcc7c44a19..59f62e9909 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/Kernel32.java +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/Kernel32.java @@ -28,12 +28,12 @@ import com.sun.jna.win32.W32APITypeMapper; import java.util.HashMap; import java.util.Map; -/* +/** * Windows Kernel32 interface */ public interface Kernel32 extends StdCallLibrary { - Map WIN32API_OPTIONS = new HashMap() { + static Map WIN32API_OPTIONS = new HashMap() { private static final long serialVersionUID = 1L; { put(Library.OPTION_FUNCTION_MAPPER, W32APIFunctionMapper.UNICODE); From e7ebc3ce4b54666efae7c02a44af393390eeadba Mon Sep 17 00:00:00 2001 From: Joe Ho Date: Tue, 3 Sep 2019 14:38:09 -0400 Subject: [PATCH 042/102] Fix codacy error --- .../autopsy/logicalimager/dsp/AddLogicalImageTask.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java index 48488dd82a..8b8c2b3012 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java @@ -192,7 +192,7 @@ final class AddLogicalImageTask implements Runnable { progressMonitor.setProgressText(Bundle.AddLogicalImageTask_addingExtractedFiles()); addExtractedFiles(dest, Paths.get(dest.toString(), resultsFilename), newDataSources); progressMonitor.setProgressText(Bundle.AddLogicalImageTask_doneAddingExtractedFiles()); - } catch (TskCoreException ex) { + } catch (IOException | TskCoreException ex) { errorList.add(ex.getMessage()); LOGGER.log(Level.SEVERE, String.format("Failed to add datasource: %s", ex.getMessage()), ex); // NON-NLS callback.done(DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS, errorList, emptyDataSources); @@ -357,7 +357,7 @@ final class AddLogicalImageTask implements Runnable { } } - private void addExtractedFiles(File src, Path resultsPath, List newDataSources) throws TskCoreException { + private void addExtractedFiles(File src, Path resultsPath, List newDataSources) throws TskCoreException, IOException { SleuthkitCase skCase = Case.getCurrentCase().getSleuthkitCase(); SleuthkitCase.CaseDbTransaction trans = null; try { @@ -377,6 +377,7 @@ final class AddLogicalImageTask implements Runnable { } String[] fields = line.split("\t", -1); // NON-NLS if (fields.length != 14) { + rollbackTransaction(trans); throw new IOException(Bundle.AddLogicalImageTask_notEnoughFields(lineNumber, fields.length, 14)); } String vhdFilename = fields[0]; @@ -412,7 +413,7 @@ final class AddLogicalImageTask implements Runnable { trans.commit(); newDataSources.add(localFilesDataSource); - } catch (IOException | NumberFormatException | TskCoreException ex) { + } catch (NumberFormatException | TskCoreException ex) { LOGGER.log(Level.SEVERE, "Error adding extracted files", ex); // NON-NLS rollbackTransaction(trans); throw new TskCoreException("Error adding extracted files", ex); From c0affed0735f04a3e36702099ac4da43447d4e0d Mon Sep 17 00:00:00 2001 From: Kelly Kelly Date: Tue, 3 Sep 2019 15:14:53 -0400 Subject: [PATCH 043/102] Added the account user groups --- .../recentactivity/Bundle.properties-MERGED | 8 +- .../autopsy/recentactivity/Extract.java | 4 + .../recentactivity/ExtractRegistry.java | 421 +++++++++++------- 3 files changed, 272 insertions(+), 161 deletions(-) diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Bundle.properties-MERGED b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Bundle.properties-MERGED index 5e11018086..f27b253e16 100755 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Bundle.properties-MERGED +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Bundle.properties-MERGED @@ -2,14 +2,9 @@ cannotBuildXmlParser=Unable to build XML parser: cannotLoadSEUQA=Unable to load Search Engine URL Query Analyzer settings file, SEUQAMappings.xml: cannotParseXml=Unable to parse XML file: ChromeCacheExtractor.moduleName=ChromeCacheExtractor -# {0} - module name -# {1} - row number -# {2} - table length -# {3} - cache path ChromeCacheExtractor.progressMsg={0}: Extracting cache entry {1} of {2} entries from {3} DataSourceUsage_AndroidMedia=Android Media Card DataSourceUsage_FlashDrive=Flash Drive -# {0} - OS name DataSourceUsageAnalyzer.customVolume.label=OS Drive ({0}) DataSourceUsageAnalyzer.parentModuleName=Recent Activity Extract.indexError.message=Failed to index artifact for keyword search. @@ -64,7 +59,7 @@ ExtractZone_progress_Msg=Extracting :Zone.Identifer files ExtractZone_Restricted=Restricted Sites Zone ExtractZone_Trusted=Trusted Sites Zone OpenIDE-Module-Display-Category=Ingest Module -OpenIDE-Module-Long-Description=Recent Activity ingest module.\n\n\The module extracts useful information about the recent user activity on the disk image being ingested, such as:\n\n- Recently open documents,\n- Web activity (sites visited, stored cookies, book marked sites, search engine queries, file downloads),\n- Recently attached devices,\n- Installed programs.\n\nThe module currently supports Windows only disk images.\nThe plugin is also fully functional when deployed on Windows version of Autopsy. +OpenIDE-Module-Long-Description=Recent Activity ingest module.\n\nThe module extracts useful information about the recent user activity on the disk image being ingested, such as:\n\n- Recently open documents,\n- Web activity (sites visited, stored cookies, book marked sites, search engine queries, file downloads),\n- Recently attached devices,\n- Installed programs.\n\nThe module currently supports Windows only disk images.\nThe plugin is also fully functional when deployed on Windows version of Autopsy. OpenIDE-Module-Name=RecentActivity OpenIDE-Module-Short-Description=Recent Activity finder ingest module Chrome.moduleName=Chrome @@ -187,7 +182,6 @@ RecentDocumentsByLnk.parentModuleName.noSpace=RecentActivity RecentDocumentsByLnk.parentModuleName=Recent Activity RegRipperFullNotFound=Full version RegRipper executable not found. RegRipperNotFound=Autopsy RegRipper executable not found. -# {0} - file name SearchEngineURLQueryAnalyzer.init.exception.msg=Unable to find {0}. SearchEngineURLQueryAnalyzer.moduleName.text=Search Engine SearchEngineURLQueryAnalyzer.engineName.none=NONE diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Extract.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Extract.java index bef51a8a95..ebf483b5d0 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Extract.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Extract.java @@ -235,6 +235,10 @@ abstract class Extract { protected String getName() { return moduleName; } + + protected String getRAModuleName() { + return RecentActivityExtracterModuleFactory.getModuleName(); + } /** * Returns the state of foundData diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java index 63ac3b6ee6..a129bea794 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java @@ -54,6 +54,7 @@ import org.w3c.dom.NodeList; import org.xml.sax.InputSource; import org.xml.sax.SAXException; import java.nio.file.Path; +import java.util.AbstractMap; import java.util.ArrayList; import java.util.List; import java.util.Collection; @@ -89,6 +90,36 @@ import org.sleuthkit.datamodel.TskCoreException; "Progress_Message_Analyze_Registry=Analyzing Registry Files" }) class ExtractRegistry extends Extract { + + private static final String USERNAME_KEY = "Username"; //NON-NLS + private static final String SID_KEY = "SID"; //NON-NLS + private static final String RID_KEY = "RID"; //NON-NLS + private static final String ACCOUNT_CREATED_KEY = "Account Created"; //NON-NLS + private static final String LAST_LOGIN_KEY = "Last Login Date"; //NON-NLS + private static final String LOGIN_COUNT_KEY = "Login Count"; //NON-NLS + private static final String FULL_NAME_KEY = "Full Name"; //NON-NLS + private static final String USER_COMMENT_KEY = "User Comment"; //NON-NLS + private static final String ACCOUNT_TYPE_KEY = "Account Type"; //NON-NLS + private static final String NAME_KEY = "Name"; //NON-NLS + private static final String PWD_RESET_KEY = "Pwd Rest Date"; //NON-NLS + private static final String PWD_FAILE_KEY = "Pwd Fail Date"; //NON-NLS + private static final String INTERNET_NAME_KEY = "InternetName"; //NON-NLS + private static final String PWD_DOES_NOT_EXPIRE_KEY = "Password does not expire"; //NON-NLS + private static final String ACCOUNT_DISABLED_KEY = "Account Disabled"; //NON-NLS + private static final String PWD_NOT_REQUIRED_KEY = "Password not required"; //NON-NLS + private static final String NORMAL_ACCOUNT_KEY = "Normal user account"; //NON-NLS + private static final String HOME_DIRECTORY_REQUIRED_KEY = "Home directory required"; + private static final String TEMPORARY_DUPLICATE_ACCOUNT = "Temporary duplicate account"; + private static final String MNS_LOGON_ACCOUNT_KEY = "MNS logon user account"; + private static final String INTERDOMAIN_TRUST_ACCOUNT_KEY = "Interdomain trust account"; + private static final String WORKSTATION_TRUST_ACCOUNT = "Workstation trust account"; + private static final String SERVER_TRUST_ACCOUNT = "Server trust account"; + private static final String ACCOUNT_AUTO_LOCKED = "Account auto locked"; + private static final String PASSWORD_HINT = "Password Hint"; + + private static final String[] PASSWORD_SETTINGS_FLAGS = {PWD_DOES_NOT_EXPIRE_KEY, PWD_NOT_REQUIRED_KEY}; + private static final String[] ACCOUNT_SETTINGS_FLAGS = {ACCOUNT_AUTO_LOCKED, HOME_DIRECTORY_REQUIRED_KEY, ACCOUNT_DISABLED_KEY}; + private static final String[] ACCOUNT_TYPE_FLAGS = {NORMAL_ACCOUNT_KEY, SERVER_TRUST_ACCOUNT, WORKSTATION_TRUST_ACCOUNT, INTERDOMAIN_TRUST_ACCOUNT_KEY, MNS_LOGON_ACCOUNT_KEY, TEMPORARY_DUPLICATE_ACCOUNT}; final private static UsbDeviceIdMapper USB_MAPPER = new UsbDeviceIdMapper(); final private static String RIP_EXE = "rip.exe"; @@ -852,27 +883,29 @@ class ExtractRegistry extends Extract { */ private boolean parseSamPluginOutput(String regFilePath, AbstractFile regAbstractFile) { File regfile = new File(regFilePath); - String parentModuleName = RecentActivityExtracterModuleFactory.getModuleName(); - SimpleDateFormat regRipperTimeFormat = new SimpleDateFormat("EEE MMM dd HH:mm:ss yyyy 'Z'"); - regRipperTimeFormat.setTimeZone(getTimeZone("GMT")); try (BufferedReader bufferedReader = new BufferedReader(new FileReader(regfile))) { // Read the file in and create a Document and elements String userInfoSection = "User Information"; String previousLine = null; String line = bufferedReader.readLine(); - Set userSet = new HashSet<>(); + Set> userSet = new HashSet<>(); + Map> groupMap = null; while (line != null) { if (line.contains(SECTION_DIVIDER) && previousLine != null && previousLine.contains(userInfoSection)) { readUsers(bufferedReader, userSet); - } + + if(line.contains(SECTION_DIVIDER) && previousLine != null && previousLine.contains("Group Membership Information")) { + groupMap = readGroups(bufferedReader); + } + previousLine = line; line = bufferedReader.readLine(); } - Map userInfoMap = new HashMap<>(); + Map> userInfoMap = new HashMap<>(); //load all the user info which was read into a map - for (UserInfo userInfo : userSet) { - userInfoMap.put(userInfo.getUserSid(), userInfo); + for (HashMap userInfo : userSet) { + userInfoMap.put(userInfo.get(SID_KEY), userInfo); } //get all existing OS account artifacts List existingOsAccounts = tskCase.getBlackboardArtifacts(ARTIFACT_TYPE.TSK_OS_ACCOUNT); @@ -881,45 +914,19 @@ class ExtractRegistry extends Extract { if (osAccount.getDataSource().getId() == regAbstractFile.getDataSourceObjectId()) { BlackboardAttribute existingUserId = osAccount.getAttribute(new BlackboardAttribute.Type(ATTRIBUTE_TYPE.TSK_USER_ID)); if (existingUserId != null) { - UserInfo userInfo = userInfoMap.remove(existingUserId.getValueString().trim()); + String userID = existingUserId.getValueString().trim(); + HashMap userInfo = userInfoMap.remove(userID); //if the existing user id matches a user id which we parsed information for check if that information exists and if it doesn't add it if (userInfo != null) { - Collection bbattributes = new ArrayList<>(); - if (userInfo.getAccountCreatedDate() != null && !userInfo.getAccountCreatedDate().equals(NEVER_DATE)) { - bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_CREATED, - parentModuleName, regRipperTimeFormat.parse(userInfo.getAccountCreatedDate()).getTime() / MS_IN_SEC)); - } - if (userInfo.getLastLoginDate() != null && !userInfo.getLastLoginDate().equals(NEVER_DATE)) { - bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_ACCESSED, - parentModuleName, regRipperTimeFormat.parse(userInfo.getLastLoginDate()).getTime() / MS_IN_SEC)); - } - bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_COUNT, - parentModuleName, userInfo.getLoginCount())); - osAccount.addAttributes(bbattributes); + osAccount.addAttributes(getAttributesForAccount(userInfo, groupMap.get(userID), true)); } } } } //add remaining userinfos as accounts; - for (String userId : userInfoMap.keySet()) { - UserInfo userInfo = userInfoMap.get(userId); - Collection bbattributes = new ArrayList<>(); + for (HashMap userInfo: userInfoMap.values()) { BlackboardArtifact bbart = regAbstractFile.newArtifact(ARTIFACT_TYPE.TSK_OS_ACCOUNT); - bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_USER_NAME, - parentModuleName, userInfo.getUserName())); - bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_USER_ID, - parentModuleName, userId)); - if (userInfo.getAccountCreatedDate() != null && !userInfo.getAccountCreatedDate().equals(NEVER_DATE)) { - bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_CREATED, - parentModuleName, regRipperTimeFormat.parse(userInfo.getAccountCreatedDate()).getTime() / MS_IN_SEC)); - } - if (userInfo.getLastLoginDate() != null && !userInfo.getLastLoginDate().equals(NEVER_DATE)) { - bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_ACCESSED, - parentModuleName, regRipperTimeFormat.parse(userInfo.getLastLoginDate()).getTime() / MS_IN_SEC)); - } - bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_COUNT, - parentModuleName, userInfo.getLoginCount())); - bbart.addAttributes(bbattributes); + bbart.addAttributes(getAttributesForAccount(userInfo, groupMap.get(userInfo.get(SID_KEY)), false)); // index the artifact for keyword search postArtifact(bbart); } @@ -936,6 +943,182 @@ class ExtractRegistry extends Extract { } return false; } + + Collection getAttributesForAccount(HashMap userInfo, List groupList, boolean existingUser) throws ParseException { + Collection bbattributes = new ArrayList<>(); + + SimpleDateFormat regRipperTimeFormat = new SimpleDateFormat("EEE MMM dd HH:mm:ss yyyy 'Z'"); + regRipperTimeFormat.setTimeZone(getTimeZone("GMT")); + + if (! existingUser) { + bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_USER_ID, + getRAModuleName(), userInfo.get(SID_KEY))); + + bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_USER_NAME, + this.moduleName, userInfo.get(USERNAME_KEY))); + } + + String value = userInfo.get(ACCOUNT_CREATED_KEY); + if (value != null && !value.isEmpty() && !value.equals(NEVER_DATE)) { + bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_CREATED, + getRAModuleName(), regRipperTimeFormat.parse(value).getTime() / MS_IN_SEC)); + } + + value = userInfo.get(LAST_LOGIN_KEY); + if (value != null && !value.isEmpty() && !value.equals(NEVER_DATE) ) { + bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_ACCESSED, + getRAModuleName(), regRipperTimeFormat.parse(value).getTime() / MS_IN_SEC)); + } + + value = userInfo.get(LOGIN_COUNT_KEY); + if(value != null && !value.isEmpty()) { + bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_COUNT, + getRAModuleName(), Integer.parseInt(value))); + } + + value = userInfo.get(ACCOUNT_TYPE_KEY); + if(value != null && !value.isEmpty()) { + bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_ACCOUNT_TYPE, + getRAModuleName(), value)); + } + + value = userInfo.get(USER_COMMENT_KEY); + if(value != null && !value.isEmpty()) { + bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DESCRIPTION, + getRAModuleName(), value)); + } + + value = userInfo.get(NAME_KEY); + if(value != null && !value.isEmpty()) { + bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_NAME, + getRAModuleName(), value)); + } + + value = userInfo.get(INTERNET_NAME_KEY); + if(value != null && !value.isEmpty()) { + bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_EMAIL, + getRAModuleName(), value)); + } + + value = userInfo.get(FULL_NAME_KEY); + if(value != null && !value.isEmpty()) { + bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DISPLAY_NAME, + getRAModuleName(), value)); + } + + value = userInfo.get(PWD_RESET_KEY); + if(value != null && !value.isEmpty() && !value.equals(NEVER_DATE)) { + bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_PASSWORD_RESET, + getRAModuleName(), regRipperTimeFormat.parse(value).getTime() / MS_IN_SEC)); + } + + value = userInfo.get(PASSWORD_HINT); + if(value != null && !value.isEmpty()) { + bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PASSWORD_HINT, + getRAModuleName(), value)); + } + + value = userInfo.get(PWD_FAILE_KEY); + if (value != null && !value.isEmpty() && !value.equals(NEVER_DATE)) { + bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_PASSWORD_FAIL, + getRAModuleName(), regRipperTimeFormat.parse(value).getTime() / MS_IN_SEC)); + } + + String settingString = ""; + for (String setting: PASSWORD_SETTINGS_FLAGS) { + if (userInfo.containsKey(setting)) { + settingString += setting + ", "; + } + } + + if (!settingString.isEmpty()) { + settingString = settingString.substring(0, settingString.length() - 2); + bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PASSWORD_SETTINGS, + getRAModuleName(), settingString)); + } + + settingString = ""; + for (String setting: ACCOUNT_SETTINGS_FLAGS) { + if (userInfo.containsKey(setting)) { + settingString += setting + ", "; + } + } + + if (!settingString.isEmpty()) { + settingString = settingString.substring(0, settingString.length() - 2); + bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_ACCOUNT_SETTINGS, + getRAModuleName(), settingString)); + } + + settingString = ""; + for (String setting: ACCOUNT_TYPE_FLAGS) { + if (userInfo.containsKey(setting)) { + settingString += setting + ", "; + } + } + + if (!settingString.isEmpty()) { + settingString = settingString.substring(0, settingString.length() - 2); + bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_FLAGS, + getRAModuleName(), settingString)); + } + + if (groupList != null && groupList.size() > 0) { + String groups = new String(); + for (String group: groupList) { + groups += group + ", "; + } + groups = groups.substring(0, groups.length() - 2); + bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_GROUPS, + getRAModuleName(), groups)); + } + + return bbattributes; + } + + Map> readGroups(BufferedReader bufferedReader) throws IOException { + HashMap> groupMap = new HashMap<>(); + + String line = bufferedReader.readLine(); + + int userCount = 0; + String groupName = null; + + while (line != null && !line.contains(SECTION_DIVIDER)) { + + if(line.contains("Group Name")) { + String value = line.replaceAll("Group Name\\s*?:", "").trim(); + groupName = (value.replaceAll("\\[\\d*?\\]", "")).trim(); + int startIndex = value.indexOf('['); + int endIndex = value.indexOf(']'); + + if(startIndex != -1 && endIndex != -1) { + String countStr = value.substring(startIndex+1, endIndex); + userCount = Integer.parseInt(countStr); + } + } else if(line.matches("Users\\s*?:")) { + for(int i = 0; i < userCount; i++) { + line = bufferedReader.readLine(); + if(line != null) { + String sid = line.trim(); + List groupList = groupMap.get(sid); + if(groupList == null) { + groupList = new ArrayList<>(); + groupMap.put(sid, groupList); + } + + groupList.add(groupName); + } + } + + groupName = null; + } + + line = bufferedReader.readLine(); + } + + return groupMap; + } /** * Read the User Information section of the SAM regripper plugin's output @@ -948,41 +1131,68 @@ class ExtractRegistry extends Extract { * * @throws IOException */ - private void readUsers(BufferedReader bufferedReader, Set users) throws IOException { - String userNameLabel = "Username :"; - String sidLabel = "SID :"; - String accountCreatedLabel = "Account Created :"; - String loginCountLabel = "Login Count :"; - String lastLoginLabel = "Last Login Date :"; + private void readUsers(BufferedReader bufferedReader, Set> users) throws IOException { String line = bufferedReader.readLine(); //read until end of file or next section divider String userName = ""; + String user_rid = ""; while (line != null && !line.contains(SECTION_DIVIDER)) { //when a user name field exists read the name and id number - if (line.contains(userNameLabel)) { - String userNameAndIdString = line.replace(userNameLabel, ""); + if (line.contains(USERNAME_KEY)) { + String regx = USERNAME_KEY + "\\s*?:"; + String userNameAndIdString = line.replaceAll(regx, ""); userName = userNameAndIdString.substring(0, userNameAndIdString.lastIndexOf('[')).trim(); - } else if (line.contains(sidLabel) && !userName.isEmpty()) { - String sid = line.replace(sidLabel, "").trim(); - UserInfo userInfo = new UserInfo(userName, sid); + user_rid = userNameAndIdString.substring(userNameAndIdString.lastIndexOf('['), userNameAndIdString.lastIndexOf(']')); + } else if (line.contains(SID_KEY) && !userName.isEmpty()) { + Map.Entry entry = getSAMKeyValue(line); + + HashMap userInfo = new HashMap<>(); + userInfo.put(USERNAME_KEY, userName); + userInfo.put(RID_KEY, user_rid); + userInfo.put(entry.getKey(), entry.getValue()); + //continue reading this users information until end of file or a blank line between users line = bufferedReader.readLine(); while (line != null && !line.isEmpty()) { - if (line.contains(accountCreatedLabel)) { - userInfo.setAccountCreatedDate(line.replace(accountCreatedLabel, "").trim()); - } else if (line.contains(loginCountLabel)) { - userInfo.setLoginCount(Integer.parseInt(line.replace(loginCountLabel, "").trim())); - } else if (line.contains(lastLoginLabel)) { - userInfo.setLastLoginDate(line.replace(lastLoginLabel, "").trim()); - } + entry = getSAMKeyValue(line); + userInfo.put(entry.getKey(), entry.getValue()); line = bufferedReader.readLine(); } users.add(userInfo); + userName = ""; } line = bufferedReader.readLine(); } } + + private Map.Entry getSAMKeyValue(String line) { + int index = line.indexOf(':'); + Map.Entry returnValue = null; + String key = null; + String value = null; + + if (index != -1) { + key = line.substring(0, index).trim(); + if (index + 1 < line.length()) { + value = line.substring(index+1).trim(); + } else { + value = ""; + } + + return new AbstractMap.SimpleEntry<>(key, value); + + } else if (line.contains("-->")) { + key = line.replace("-->", "").trim(); + value = "true"; + } + + if (key != null) { + returnValue = new AbstractMap.SimpleEntry<>(key, value); + } + + return returnValue; + } @Override public void process(Content dataSource, IngestJobContext context, DataSourceIngestModuleProgress progressBar) { @@ -1002,101 +1212,4 @@ class ExtractRegistry extends Extract { public String autopsyPlugins = ""; public String fullPlugins = ""; } - - /** - * Class for organizing information associated with a TSK_OS_ACCOUNT before - * the artifact is created. - */ - private class UserInfo { - - private final String userName; - private final String userSid; - private String lastLoginDate; - private String accountCreatedDate; - private int loginCount = 0; - - /** - * Create a UserInfo object - * - * @param name - the os user account name - * @param userSidString - the SID for the user account - */ - private UserInfo(String name, String userSidString) { - userName = name; - userSid = userSidString; - } - - /** - * Get the user name. - * - * @return the userName - */ - String getUserName() { - return userName; - } - - /** - * Get the user SID. - * - * @return the user SID - */ - String getUserSid() { - return userSid; - } - - /** - * Get the last login date for the user - * - * @return the lastLoginDate - */ - String getLastLoginDate() { - return lastLoginDate; - } - - /** - * Set the last login date for the users - * - * @param lastLoginDate the lastLoginDate to set - */ - void setLastLoginDate(String lastLoginDate) { - this.lastLoginDate = lastLoginDate; - } - - /** - * Get the account creation date. - * - * @return the accountCreatedDate - */ - String getAccountCreatedDate() { - return accountCreatedDate; - } - - /** - * Set the account creation date. - * - * @param accountCreatedDate the accountCreatedDate to set - */ - void setAccountCreatedDate(String accountCreatedDate) { - this.accountCreatedDate = accountCreatedDate; - } - - /** - * Get the number of times the user logged in. - * - * @return the loginCount - */ - int getLoginCount() { - return loginCount; - } - - /** - * Set the number of times the user logged in. - * - * @param loginCount the loginCount to set - */ - void setLoginCount(int loginCount) { - this.loginCount = loginCount; - } - - } } From 932bc3f6bf1181d0a329ae9213c27375c64cefc7 Mon Sep 17 00:00:00 2001 From: Joe Ho Date: Tue, 3 Sep 2019 17:02:00 -0400 Subject: [PATCH 044/102] Use FileSystem class to get at the file system type --- .../configuration/ConfigVisualPanel1.java | 41 +++-------- .../logicalimager/configuration/Kernel32.java | 70 ------------------- 2 files changed, 11 insertions(+), 100 deletions(-) delete mode 100644 Core/src/org/sleuthkit/autopsy/logicalimager/configuration/Kernel32.java diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/ConfigVisualPanel1.java b/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/ConfigVisualPanel1.java index 042906c0b3..fc683be97a 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/ConfigVisualPanel1.java +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/ConfigVisualPanel1.java @@ -22,8 +22,6 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.JsonIOException; import com.google.gson.JsonSyntaxException; -import com.sun.jna.platform.win32.WinDef.DWORD; -import com.sun.jna.ptr.IntByReference; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; @@ -31,7 +29,11 @@ import java.io.IOException; import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; import java.nio.file.FileStore; +import java.nio.file.FileSystem; +import java.nio.file.FileSystems; import java.nio.file.Files; +import java.nio.file.Paths; +import java.nio.file.spi.FileSystemProvider; import java.util.ArrayList; import java.util.List; import java.util.logging.Level; @@ -249,36 +251,15 @@ final class ConfigVisualPanel1 extends JPanel { * */ @Messages({"ConfigVisualPanel1.unknown=Unknown"}) - private String getFileSystemName(String drive){ - char[] lpVolumeNameBuffer = new char[256]; - DWORD nVolumeNameSize = new DWORD(256); - IntByReference lpVolumeSerialNumber = new IntByReference(); - IntByReference lpMaximumComponentLength = new IntByReference(); - IntByReference lpFileSystemFlags = new IntByReference(); - - char[] lpFileSystemNameBuffer = new char[256]; - DWORD nFileSystemNameSize = new DWORD(256); - - lpVolumeSerialNumber.setValue(0); - lpMaximumComponentLength.setValue(256); - lpFileSystemFlags.setValue(0); - - Kernel32.INSTANCE.GetVolumeInformation( - drive, - lpVolumeNameBuffer, - nVolumeNameSize, - lpVolumeSerialNumber, - lpMaximumComponentLength, - lpFileSystemFlags, - lpFileSystemNameBuffer, - nFileSystemNameSize); - if (Kernel32.INSTANCE.GetLastError() != 0) { - logger.log(Level.INFO, String.format("Last error: %d", Kernel32.INSTANCE.GetLastError())); // NON-NLS + private String getFileSystemName(String drive) { + FileSystem fileSystem = FileSystems.getDefault(); + FileSystemProvider provider = fileSystem.provider(); + try { + FileStore fileStore = provider.getFileStore(Paths.get(drive)); + return fileStore.type(); + } catch (IOException ex) { return Bundle.ConfigVisualPanel1_unknown(); } - - String fs = new String(lpFileSystemNameBuffer); - return fs.trim(); } /** diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/Kernel32.java b/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/Kernel32.java deleted file mode 100644 index 59f62e9909..0000000000 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/Kernel32.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Autopsy - * - * Copyright 2019 Basis Technology Corp. - * Contact: carrier sleuthkit org - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.sleuthkit.autopsy.logicalimager.configuration; - -import com.sun.jna.Library; -import com.sun.jna.Native; -import com.sun.jna.platform.win32.WinDef.DWORD; -import com.sun.jna.ptr.IntByReference; -import com.sun.jna.win32.StdCallLibrary; -import com.sun.jna.win32.W32APIFunctionMapper; -import com.sun.jna.win32.W32APITypeMapper; -import java.util.HashMap; -import java.util.Map; - -/** - * Windows Kernel32 interface - */ -public interface Kernel32 extends StdCallLibrary { - - static Map WIN32API_OPTIONS = new HashMap() { - private static final long serialVersionUID = 1L; - { - put(Library.OPTION_FUNCTION_MAPPER, W32APIFunctionMapper.UNICODE); - put(Library.OPTION_TYPE_MAPPER, W32APITypeMapper.UNICODE); - } - }; - - Kernel32 INSTANCE = (Kernel32) Native.loadLibrary("Kernel32", Kernel32.class, WIN32API_OPTIONS); - - /* - BOOL WINAPI GetVolumeInformation( - __in_opt LPCTSTR lpRootPathName, - __out LPTSTR lpVolumeNameBuffer, - __in DWORD nVolumeNameSize, - __out_opt LPDWORD lpVolumeSerialNumber, - __out_opt LPDWORD lpMaximumComponentLength, - __out_opt LPDWORD lpFileSystemFlags, - __out LPTSTR lpFileSystemNameBuffer, - __in DWORD nFileSystemNameSize - ); - */ - boolean GetVolumeInformation( - String lpRootPathName, - char[] lpVolumeNameBuffer, - DWORD nVolumeNameSize, - IntByReference lpVolumeSerialNumber, - IntByReference lpMaximumComponentLength, - IntByReference lpFileSystemFlags, - char[] lpFileSystemNameBuffer, - DWORD nFileSystemNameSize - ); - - int GetLastError(); -} From 0c848f8fdd5bb35c77dbfc1d3f353c7e81de7d20 Mon Sep 17 00:00:00 2001 From: Joe Ho Date: Tue, 3 Sep 2019 17:25:59 -0400 Subject: [PATCH 045/102] Fix codacy error --- .../dsp/AddLogicalImageTask.java | 40 ++++++++++--------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java index 8b8c2b3012..fc72965bf6 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java @@ -284,8 +284,6 @@ final class AddLogicalImageTask implements Runnable { "# {0} - target image path", "AddLogicalImageTask.cannotFindDataSourceObjId=Cannot find obj_id in tsk_image_names for {0}" }) private void addInterestingFiles(File src, Path resultsPath, boolean createVHD) throws IOException, TskCoreException { - Map> imagePaths = currentCase.getSleuthkitCase().getImagePaths(); - Map imagePathToObjIdMap = imagePathsToDataSourceObjId(imagePaths); try (BufferedReader br = new BufferedReader(new InputStreamReader( new FileInputStream(resultsPath.toFile()), "UTF8"))) { // NON-NLS @@ -309,23 +307,9 @@ final class AddLogicalImageTask implements Runnable { String ruleName = fields[5]; // String description = fields[6]; String filename = fields[7]; + String parentPath = fields[8]; - String query; - String targetImagePath; - if (createVHD) { - targetImagePath = Paths.get(src.toString(), vhdFilename).toString(); - Long dataSourceObjId = imagePathToObjIdMap.get(targetImagePath); - if (dataSourceObjId == null) { - throw new TskCoreException(Bundle.AddLogicalImageTask_cannotFindDataSourceObjId(targetImagePath)); - } - query = String.format("data_source_obj_id = '%s' AND meta_addr = '%s' AND name = '%s'", // NON-NLS - dataSourceObjId.toString(), fileMetaAddressStr, filename.replace("'", "''")); - } else { - String parentPath = fields[8]; - parentPath = "/" + ROOT_STR + "/" + vhdFilename + "/" + parentPath; - query = String.format("name = '%s' AND parent_path = '%s'", // NON-NLS - filename.replace("'", "''"), parentPath.replace("'", "''")); - } + String query = makeQuery(createVHD, vhdFilename, fileMetaAddressStr, parentPath, filename); // TODO - findAllFilesWhere should SQL-escape the query List matchedFiles = Case.getCurrentCase().getSleuthkitCase().findAllFilesWhere(query); @@ -430,4 +414,24 @@ final class AddLogicalImageTask implements Runnable { } } + String makeQuery(boolean createVHD, String vhdFilename, String fileMetaAddressStr, String parentPath, String filename) throws TskCoreException { + String query; + if (createVHD) { + Map> imagePaths = currentCase.getSleuthkitCase().getImagePaths(); + Map imagePathToObjIdMap = imagePathsToDataSourceObjId(imagePaths); + String targetImagePath = Paths.get(src.toString(), vhdFilename).toString(); + Long dataSourceObjId = imagePathToObjIdMap.get(targetImagePath); + if (dataSourceObjId == null) { + throw new TskCoreException(Bundle.AddLogicalImageTask_cannotFindDataSourceObjId(targetImagePath)); + } + query = String.format("data_source_obj_id = '%s' AND meta_addr = '%s' AND name = '%s'", // NON-NLS + dataSourceObjId.toString(), fileMetaAddressStr, filename.replace("'", "''")); + } else { + parentPath = "/" + ROOT_STR + "/" + vhdFilename + "/" + parentPath; + query = String.format("name = '%s' AND parent_path = '%s'", // NON-NLS + filename.replace("'", "''"), parentPath.replace("'", "''")); + } + return query; + } + } From 536550403bedb90416aa7052178ef03856d914ed Mon Sep 17 00:00:00 2001 From: Raman Date: Wed, 4 Sep 2019 08:39:27 -0400 Subject: [PATCH 046/102] 5394: Implement utility class supporting mobile app parsers and first parser --- .../autopsy/coreutils/AppDBParserHelper.java | 1057 +++++++++++++++++ .../autopsy/coreutils/AppSQLiteDB.java | 268 +++++ InternalPythonModules/android/imo.py | 127 ++ InternalPythonModules/android/module.py | 6 +- 4 files changed, 1457 insertions(+), 1 deletion(-) create mode 100644 Core/src/org/sleuthkit/autopsy/coreutils/AppDBParserHelper.java create mode 100644 Core/src/org/sleuthkit/autopsy/coreutils/AppSQLiteDB.java create mode 100644 InternalPythonModules/android/imo.py diff --git a/Core/src/org/sleuthkit/autopsy/coreutils/AppDBParserHelper.java b/Core/src/org/sleuthkit/autopsy/coreutils/AppDBParserHelper.java new file mode 100644 index 0000000000..34c0fefc48 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/coreutils/AppDBParserHelper.java @@ -0,0 +1,1057 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2019 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.coreutils; + +import java.util.Collection; +import java.util.Collections; +import java.util.logging.Level; +import org.apache.commons.lang3.StringUtils; +import org.sleuthkit.autopsy.casemodule.Case; +import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.datamodel.AccountFileInstance; +import org.sleuthkit.datamodel.BlackboardArtifact; +import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE; +import org.sleuthkit.datamodel.BlackboardAttribute; +import org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE; +import org.sleuthkit.datamodel.Account; +import org.sleuthkit.datamodel.Blackboard; +import org.sleuthkit.datamodel.DataSource; +import org.sleuthkit.datamodel.Relationship; +import org.sleuthkit.datamodel.TskCoreException; +import org.sleuthkit.datamodel.TskDataException; + + +/** + * A helper class to support modules that parse SQLite databases from mobile + * apps and create artifacts. + */ +public final class AppDBParserHelper { + + private static final Logger logger = Logger.getLogger(AppDBParserHelper.class.getName()); + + private final AbstractFile dbAbstractFile; + private final String moduleName; + + // 'self' account for the application. + private final AccountFileInstance selfAccountInstance; + + // type of accounts to be created for the Application using this helper + private final Account.Type accountsType; + + /** + * Constructs a AppDB parser helper for the given DB file. + * + * This is a constructor for Apps that that do not have any app specific account information + * for device owner and will use a 'Device' account in lieu. + * + * It creates a DeviceAccount instance to use as a self account. + * + * @param moduleName name module using the helper + * @param dbFile database file being parsed by the module + * @param accountsType account types created by this module + * + * @throws TskCoreException + */ + public AppDBParserHelper(String moduleName, AbstractFile dbFile, Account.Type accountsType) throws TskCoreException { + + this.moduleName = moduleName; + this.dbAbstractFile = dbFile; + this.accountsType = accountsType; + this.selfAccountInstance = Case.getCurrentCase().getSleuthkitCase().getCommunicationsManager().createAccountFileInstance(Account.Type.DEVICE, ((DataSource)dbFile.getDataSource()).getDeviceId(), moduleName, dbFile); + } + + /** + * Constructs a AppDB parser helper for the given DB file. + * + * This constructor is for Apps that do have app specific account information + * for the device owner to create a 'self' account. + * + * It creates a an account instance with specified type & id and uses it as + * a self account. + * + * @param moduleName name module using the helper + * @param dbFile database file being parsed by the module + * @param accountsType account types created by this module + * @param selfAccountType self account type to be created for this module + * @param selfAccountId account unique id for the self account + * + * @throws TskCoreException + */ + public AppDBParserHelper(String moduleName, AbstractFile dbFile, Account.Type accountsType, Account.Type selfAccountType, String selfAccountId) throws TskCoreException { + + this.moduleName = moduleName; + this.dbAbstractFile = dbFile; + this.accountsType = accountsType; + + this.selfAccountInstance = Case.getCurrentCase().getSleuthkitCase().getCommunicationsManager().createAccountFileInstance(selfAccountType, selfAccountId, moduleName, dbFile); + } + + /** + * Constructs a AppDB parser helper for the given DB file. + * + * This is a constructor for Apps that do not need to create any + * accounts/relationships. + * + * @param moduleName name of module parsing the DB + * @param dbFile db file + * + */ + public AppDBParserHelper(String moduleName, AbstractFile dbFile) { + this.moduleName = moduleName; + this.dbAbstractFile = dbFile; + this.selfAccountInstance = null; + this.accountsType = null; + } + + + /** + * Creates and adds a TSK_CONTACT artifact to the case, with specified + * attributes. + * Also creates an account instance of specified type for the contact with the + * specified ID. + * + * @param contactAccountUniqueID unique id for the contact's account + * @param contactName Name of contact + * @param phoneNumber primary phone number for contact + * @param homePhoneNumber home phone number + * @param mobilePhoneNumber mobile phone number, + * @param emailAddr Email address for contact + * + * @return artifact created + * + */ + public BlackboardArtifact addContact(String contactAccountUniqueID, String contactName, + String phoneNumber, String homePhoneNumber, + String mobilePhoneNumber, String emailAddr) { + return addContact(contactAccountUniqueID, contactName,phoneNumber, + homePhoneNumber,mobilePhoneNumber, emailAddr, + Collections.emptyList() ); + } + + + /** + * Creates and adds a TSK_CONTACT artifact to the case, with specified + * attributes. + * Also creates an account instance for the contact with the + * specified ID. + * + * @param contactAccountUniqueID unique id for contact account + * @param contactName Name of contact + * @param phoneNumber primary phone number for contact + * @param homePhoneNumber home phone number + * @param mobilePhoneNumber mobile phone number, + * @param emailAddr Email address for contact + * + * @param additionalAttributes additional attributes for contact + * + * @return contact artifact created + * + */ + public BlackboardArtifact addContact(String contactAccountUniqueID, String contactName, + String phoneNumber, String homePhoneNumber, + String mobilePhoneNumber, String emailAddr, + Collection additionalAttributes) { + + BlackboardArtifact contactArtifact = null; + try { + // Create TSK_CONTACT artifact + contactArtifact = this.dbAbstractFile.newArtifact(ARTIFACT_TYPE.TSK_CONTACT); + + // Add basic attributes for name phonenumber email, if specified + contactArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_NAME, moduleName, contactName)); + + if (!StringUtils.isEmpty(phoneNumber)) { + contactArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PHONE_NUMBER, moduleName, phoneNumber)); + } + if (!StringUtils.isEmpty(homePhoneNumber)) { + contactArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_HOME, moduleName, homePhoneNumber)); + } + if (!StringUtils.isEmpty(mobilePhoneNumber)) { + contactArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_MOBILE, moduleName, mobilePhoneNumber)); + } + if (!StringUtils.isEmpty(emailAddr)) { + contactArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_EMAIL, moduleName, emailAddr)); + } + + // Add additional specified attributes + for (BlackboardAttribute additionalAttribute: additionalAttributes) { + contactArtifact.addAttribute(additionalAttribute); + } + + // Find/Create an account instance for the contact + // Create a relationship between selfAccount and contactAccount + AccountFileInstance contactAccountInstance = createAccountInstance(accountsType, contactAccountUniqueID); + if (selfAccountInstance != null) { + addRelationship (selfAccountInstance, contactAccountInstance, contactArtifact, Relationship.Type.CONTACT, 0 ); + } + + // post artifact + Case.getCurrentCase().getSleuthkitCase().getBlackboard().postArtifact(contactArtifact, this.moduleName); + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, "Unable to add contact artifact", ex); //NON-NLS + return null; + } + catch (Blackboard.BlackboardException ex) { + logger.log(Level.SEVERE, String.format("Unable to post artifact %s", ((contactArtifact != null)? contactArtifact.getArtifactID() : "")), ex); //NON-NLS + } + + return contactArtifact; + } + + + /** + * Creates an account file instance associated with the DB file. + * @param accountType + * @param accountUniqueID + * @return + * @throws TskCoreException + */ + private AccountFileInstance createAccountInstance(Account.Type accountType, String accountUniqueID ) throws TskCoreException { + return Case.getCurrentCase().getSleuthkitCase().getCommunicationsManager().createAccountFileInstance(accountType, accountUniqueID, moduleName, this.dbAbstractFile); + } + + + /** + * Adds a relations between the two specified account instances. + * + * @param selfAccount device owner account + * @param otherAccount other account + * @param sourceArtifact artifact from which relationship is derived. + * @param relationshipType type of relationship + * @param dateTime date/time of relationship + */ + private void addRelationship(AccountFileInstance selfAccount, AccountFileInstance otherAccount, + BlackboardArtifact sourceArtifact, Relationship.Type relationshipType, long dateTime) { + try { + Case.getCurrentCase().getSleuthkitCase().getCommunicationsManager().addRelationships(selfAccount, + Collections.singletonList(otherAccount), sourceArtifact, relationshipType, dateTime); + } catch (TskCoreException | TskDataException ex) { + logger.log(Level.SEVERE, String.format("Unable to add relationship between account %s and account %s", selfAccount.toString(), otherAccount.toString()), ex); //NON-NLS + } + } + + + /** + * Adds a TSK_MESSAGE artifact. + * + * Also creates an account instance for the sender/receiver, and creates a + * relationship between the self account and the sender/receiver account. + * + * @param otherAccountUniqueID unique id for the sender/receiver account + * @param messageType message type + * @param direction message direction + * @param fromAddress sender address, may be empty + * @param toAddress recipient address, may be empty + * @param dateTime date/time of message, + * @param readStatus message read or not + * @param subject message subject, may be empty + * @param messageText message body, may be empty + * @param threadId, message thread id + * + * @return message artifact + */ + public BlackboardArtifact addMessage(String otherAccountUniqueID, + String messageType, String direction, String fromAddress, String toAddress, + long dateTime, int readStatus, String subject, String messageText, String threadId) { + return addMessage(otherAccountUniqueID, messageType, direction, + fromAddress, toAddress, dateTime, readStatus, + subject, messageText, threadId, + Collections.emptyList()); + } + + /** + * Adds a TSK_MESSAGE artifact. + * + * Also creates an account instance for the sender/receiver, and creates a + * relationship between the self account and the sender/receiver account. + * + * @param otherAccountUniqueID unique id for the sender/receiver account + * @param messageType message type + * @param direction message direction + * @param fromAddress sender address, may be empty + * @param toAddress recipient address, may be empty + * @param dateTime date/time of message, + * @param readStatus message read or not + * @param subject message subject, may be empty + * @param messageText message body, may be empty + * @param threadId, message thread id + * + * @param otherAttributesList additional attributes + * + * @return message artifact + */ + public BlackboardArtifact addMessage(String otherAccountUniqueID, + String messageType, String direction, String fromAddress, + String toAddress, long dateTime, int readStatus, String subject, + String messageText, String threadId, + Collection otherAttributesList) { + + BlackboardArtifact msgArtifact = null; + try { + // Create TSK_MESSAGE artifact + msgArtifact = dbAbstractFile.newArtifact(ARTIFACT_TYPE.TSK_MESSAGE); + if (dateTime > 0) { + msgArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME, moduleName, dateTime)); + } + if (readStatus > 0) { + msgArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_READ_STATUS, moduleName, readStatus)); + } + + // Add basic attribute, if the correspond value is specified + if (!StringUtils.isEmpty(messageType)) { + msgArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_MESSAGE_TYPE, moduleName, messageType)); + } + if (!StringUtils.isEmpty(direction)) { + msgArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DIRECTION, moduleName, direction)); + } + if (!StringUtils.isEmpty(fromAddress)) { + msgArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_FROM, moduleName, fromAddress)); + } + if (!StringUtils.isEmpty(toAddress)) { + msgArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_TO, moduleName, toAddress)); + } + + if (!StringUtils.isEmpty(subject)) { + msgArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_SUBJECT, moduleName, subject)); + } + if (!StringUtils.isEmpty(messageText)) { + msgArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_TEXT, moduleName, messageText)); + } + if (!StringUtils.isEmpty(threadId)) { + msgArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_THREAD_ID, moduleName, threadId)); + } + + + // Add other specified attributes + for (BlackboardAttribute otherAttribute: otherAttributesList) { + msgArtifact.addAttribute(otherAttribute); + } + + // Find/Create an account instance for the sender/recipient + AccountFileInstance contactAccountInstance = createAccountInstance(accountsType, otherAccountUniqueID); + + // Create a relationship between selfAccount and contactAccount + if (selfAccountInstance != null) { + addRelationship (selfAccountInstance, contactAccountInstance, msgArtifact, Relationship.Type.MESSAGE, 0 ); + } + + // post artifact + Case.getCurrentCase().getSleuthkitCase().getBlackboard().postArtifact(msgArtifact, this.moduleName); + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, "Unable to add message artifact", ex); //NON-NLS + return null; + } + catch (Blackboard.BlackboardException ex) { + logger.log(Level.SEVERE, String.format("Unable to post artifact %s", ((msgArtifact != null)? msgArtifact.getArtifactID() : "")), ex); //NON-NLS + } + + // return the artifact + return msgArtifact; + } + + /** + * Adds a TSK_CALLLOG artifact. + * + * Also creates an account instance for the caller/receiver, and creates a + * relationship between the self account and the caller/receiver account. + * + * @param otherAccountUniqueID unique id for the caller/receiver account + * @param direction call direction + * @param fromPhoneNumber originating phone number, may be empty + * @param toPhoneNumber recipient phone number, may be empty + * @param startDateTime start date/time + * @param endDateTime end date/time + * @param contactName contact name, may be empty + * + * @return call log artifact + */ + public BlackboardArtifact addCalllog( String otherAccountUniqueID, + String direction, String fromPhoneNumber, String toPhoneNumber, + long startDateTime, long endDateTime, String contactName) { + return addCalllog(otherAccountUniqueID, direction, fromPhoneNumber, toPhoneNumber, + startDateTime, endDateTime, contactName, + Collections.emptyList()); + } + + /** + * Adds a TSK_CALLLOG artifact. + * + * Also creates an account instance for the caller/receiver, and creates a + * relationship between the device owner account and the caller/receiver account. + * + * @param otherAccountUniqueID + * @param direction + * @param fromPhoneNumber + * @param toPhoneNumber + * @param startDateTime + * @param endDateTime + * @param contactName + * @param otherAttributesList + * + * @return calllog artifact + */ + public BlackboardArtifact addCalllog(String otherAccountUniqueID, + String direction, String fromPhoneNumber, String toPhoneNumber, + long startDateTime, long endDateTime, String contactName, + Collection otherAttributesList) { + BlackboardArtifact callLogArtifact = null; + try { + // Create TSK_CALLLOG artifact + callLogArtifact = dbAbstractFile.newArtifact(ARTIFACT_TYPE.TSK_CALLLOG); + + // Add basic attributes + if (startDateTime > 0) { + callLogArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_START, moduleName, startDateTime)); + } + if (endDateTime > 0) { + callLogArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_END, moduleName, endDateTime)); + } + + if (!StringUtils.isEmpty(direction)) { + callLogArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DIRECTION, moduleName, direction)); + } + if (!StringUtils.isEmpty(fromPhoneNumber)) { + callLogArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_FROM, moduleName, fromPhoneNumber)); + } + if (!StringUtils.isEmpty(toPhoneNumber)) { + callLogArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_TO, moduleName, toPhoneNumber)); + } + if (!StringUtils.isEmpty(contactName)) { + callLogArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_NAME, moduleName, contactName)); + } + + // Add other specified attributes + for (BlackboardAttribute otherAttribute: otherAttributesList) { + callLogArtifact.addAttribute(otherAttribute); + } + + // Find/Create an account instance for the sender/recipient + // Create a relationship between selfAccount and contactAccount + AccountFileInstance contactAccountInstance = createAccountInstance(accountsType, otherAccountUniqueID); + if (selfAccountInstance != null) { + addRelationship (selfAccountInstance, contactAccountInstance, callLogArtifact, Relationship.Type.CALL_LOG, 0 ); + } + + // post artifact + Case.getCurrentCase().getSleuthkitCase().getBlackboard().postArtifact(callLogArtifact, this.moduleName); + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, "Unable to add calllog artifact", ex); //NON-NLS + return null; + } + catch (Blackboard.BlackboardException ex) { + logger.log(Level.SEVERE, String.format("Unable to post artifact %s", ((callLogArtifact != null)? callLogArtifact.getArtifactID() : "")), ex); //NON-NLS + } + + // return the artifact + return callLogArtifact; + } + + + // TBD + public void addAttachment(BlackboardArtifact parentArtifact) { + // RAMAN TBD + } + + /** + * Adds a TSK_WEB_BOOKMARK artifact. + * + * @param url bookmark URL + * @param title bookmark title, may be empty + * @param creationTime date/time created + * @param progName application/program that created bookmark + * + * @return bookmark artifact + */ + public BlackboardArtifact addWebBookmark(String url, String title, long creationTime, String progName) { + return addWebBookmark(url, title, creationTime, progName, + Collections.emptyList()); + } + + /** + * Adds a TSK_WEB_BOOKMARK artifact. + * + * @param url bookmark URL + * @param title bookmark title, may be empty + * @param creationTime date/time created + * @param progName application/program that created bookmark + * @param otherAttributesList other attributes + + * @return bookmark artifact + */ + public BlackboardArtifact addWebBookmark(String url, String title, long creationTime, String progName, + Collection otherAttributesList) { + + BlackboardArtifact bookMarkArtifact = null; + try { + // Create artifact + bookMarkArtifact = dbAbstractFile.newArtifact(ARTIFACT_TYPE.TSK_WEB_BOOKMARK); + + // Add basic attributes + bookMarkArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL, moduleName, url)); + if (creationTime > 0) { + bookMarkArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_CREATED, moduleName, creationTime)); + } + + if (!StringUtils.isEmpty(title)) { + bookMarkArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_TITLE, moduleName, title)); + } + if (!StringUtils.isEmpty(url)) { + bookMarkArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DOMAIN, moduleName, NetworkUtils.extractDomain(url))); + } + if (!StringUtils.isEmpty(progName)) { + bookMarkArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PROG_NAME, moduleName, progName)); + } + + // Add other specified attributes + for (BlackboardAttribute otherAttribute: otherAttributesList) { + bookMarkArtifact.addAttribute(otherAttribute); + } + + // post artifact + Case.getCurrentCase().getSleuthkitCase().getBlackboard().postArtifact(bookMarkArtifact, this.moduleName); + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, "Unable to add bookmark artifact", ex); //NON-NLS + return null; + } + catch (Blackboard.BlackboardException ex) { + logger.log(Level.SEVERE, String.format("Unable to post artifact %s", ((bookMarkArtifact != null)? bookMarkArtifact.getArtifactID() : "")), ex); //NON-NLS + } + + // return the artifact + return bookMarkArtifact; + } + + + /** + * Adds a TSK_WEB_COOKIE artifact + * + * @param url url of the site that created the cookie + * @param creationTime create time of cookie + * @param name cookie name + * @param value cookie value + * @param programName name of the application that created the cookie + * + * @return WebCookie artifact + */ + public BlackboardArtifact addWebCookie(String url, long creationTime, + String name, String value, String programName) { + + return addWebCookie(url, creationTime, name, value, programName, + Collections.emptyList()); + } + + /** + * Adds a TSK_WEB_COOKIE artifact + * + * @param url url of the site that created the cookie + * @param creationTime create time of cookie + * @param name cookie name + * @param value cookie value + * @param programName name of the application that created the cookie + * + * @param otherAttributesList other attributes + * + * @return WebCookie artifact + */ + public BlackboardArtifact addWebCookie(String url, + long creationTime, String name, String value, String programName, + Collection otherAttributesList) { + + + BlackboardArtifact cookieArtifact = null; + try { + // Create artifact + cookieArtifact = dbAbstractFile.newArtifact(ARTIFACT_TYPE.TSK_WEB_COOKIE); + + // Add basic attributes + cookieArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL, moduleName, url)); + if (creationTime > 0) { + cookieArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME, moduleName, creationTime)); + } + + if (!StringUtils.isEmpty(name)) { + cookieArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_NAME, moduleName, name)); + } + if (!StringUtils.isEmpty(value)) { + cookieArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_VALUE, moduleName, value)); + } + if (!StringUtils.isEmpty(url)) { + cookieArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DOMAIN, moduleName, NetworkUtils.extractDomain(url))); + } + if (!StringUtils.isEmpty(programName)) { + cookieArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PROG_NAME, moduleName, programName)); + } + + // Add other specified attributes + for (BlackboardAttribute otherAttribute: otherAttributesList) { + cookieArtifact.addAttribute(otherAttribute); + } + + // post artifact + Case.getCurrentCase().getSleuthkitCase().getBlackboard().postArtifact(cookieArtifact, this.moduleName); + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, "Unable to add bookmark artifact", ex); //NON-NLS + return null; + } + catch (Blackboard.BlackboardException ex) { + logger.log(Level.SEVERE, String.format("Unable to post artifact %s", ((cookieArtifact != null)? cookieArtifact.getArtifactID() : "")), ex); //NON-NLS + } + + // return the artifact + return cookieArtifact; + } + + /** + * Adds a Web History artifact + * + * @param url url visited + * @param accessTime last access time + * @param referrer referrer, may be empty + * @param title website title, may be empty + * @param programName, application recording the history + * + * @return artifact created + */ + public BlackboardArtifact addWebHistory(String url, long accessTime, + String referrer, String title, String programName) { + return addWebHistory(url, accessTime, referrer, title, programName, + Collections.emptyList()); + } + + /** + * Adds a Web History artifact + * + * @param url url visited + * @param accessTime last access time + * @param referrer referrer, may be empty + * @param title website title, may be empty + * @param programName, application recording the history + * @param otherAttributesList other attributes + * + * + * + * @return artifact created + */ + public BlackboardArtifact addWebHistory(String url, long accessTime, + String referrer, String title, String programName, + Collection otherAttributesList) { + + BlackboardArtifact webHistoryArtifact = null; + try { + // Create artifact + webHistoryArtifact = dbAbstractFile.newArtifact(ARTIFACT_TYPE.TSK_WEB_HISTORY); + + // Add basic attributes + webHistoryArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL, moduleName, url)); + if (accessTime > 0) { + webHistoryArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_ACCESSED, moduleName, accessTime)); + } + + if (!StringUtils.isEmpty(title)) { + webHistoryArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_TITLE, moduleName, title)); + } + if (!StringUtils.isEmpty(referrer)) { + webHistoryArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_REFERRER, moduleName, referrer)); + } + + if (!StringUtils.isEmpty(programName)) { + webHistoryArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PROG_NAME, moduleName, programName)); + } + if (!StringUtils.isEmpty(url)) { + webHistoryArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DOMAIN, moduleName, NetworkUtils.extractDomain(url))); + } + + // Add other specified attributes + for (BlackboardAttribute otherAttribute: otherAttributesList) { + webHistoryArtifact.addAttribute(otherAttribute); + } + + // post artifact + Case.getCurrentCase().getSleuthkitCase().getBlackboard().postArtifact(webHistoryArtifact, this.moduleName); + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, "Unable to add bookmark artifact", ex); //NON-NLS + return null; + } + catch (Blackboard.BlackboardException ex) { + logger.log(Level.SEVERE, String.format("Unable to post artifact %s", ((webHistoryArtifact != null)? webHistoryArtifact.getArtifactID() : "")), ex); //NON-NLS + } + + // return the artifact + return webHistoryArtifact; + } + + /** + * Created a TSK_WEB_DOWNNLOAD artifact + * + * @param path path of downloaded file + * @param startTime date/time downloaded + * @param url URL downloaded from + * @param progName program that initiated download + * + * @return artifact created + */ + public BlackboardArtifact addWebDownload(String path, long startTime, String url, String progName) { + return addWebDownload(path, startTime, url, progName, Collections.emptyList() ); + } + + /** + * Created a TSK_WEB_DOWNNLOAD artifact + * + * @param path path of downloaded file + * @param startTime date/time downloaded + * @param url URL downloaded from + * @param programName program that initiated download + * @param otherAttributesList other attributes + * + * + * @return artifact created + */ + public BlackboardArtifact addWebDownload(String path, long startTime, String url, String programName, + Collection otherAttributesList ) { + + BlackboardArtifact webDownloadArtifact = null; + try { + // Create artifact + webDownloadArtifact = dbAbstractFile.newArtifact(ARTIFACT_TYPE.TSK_WEB_DOWNLOAD); + + // Add basic attributes + webDownloadArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL, moduleName, url)); + if (startTime > 0) { + webDownloadArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_ACCESSED, moduleName, startTime)); + } + webDownloadArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PATH, moduleName, path)); + + /** Convert path to pathID ****/ +// long pathID = Util.findID(dataSource, downloadedFilePath); +// if (pathID != -1) { +// bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PATH_ID, moduleName, pathID)); +// } + + if (!StringUtils.isEmpty(programName)) { + webDownloadArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PROG_NAME, moduleName, programName)); + } + if (!StringUtils.isEmpty(url)) { + webDownloadArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DOMAIN, moduleName, NetworkUtils.extractDomain(url))); + } + + // Add other specified attributes + for (BlackboardAttribute otherAttribute: otherAttributesList) { + webDownloadArtifact.addAttribute(otherAttribute); + } + + // post artifact + Case.getCurrentCase().getSleuthkitCase().getBlackboard().postArtifact(webDownloadArtifact, this.moduleName); + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, "Unable to add web download artifact", ex); //NON-NLS + return null; + } + catch (Blackboard.BlackboardException ex) { + logger.log(Level.SEVERE, String.format("Unable to post artifact %s", ((webDownloadArtifact != null)? webDownloadArtifact.getArtifactID() : "")), ex); //NON-NLS + } + + // return the artifact + return webDownloadArtifact; + } + + + /** + * Adds a TSK_WEB_FORM_AUTOFILL artifact + * + * @param name name of autofill field + * @param value value of autofill field + * @param creationTime create date/time + * @param accessTime last access date/time + * @param count count of times used + * + * @return artifact created + */ + public BlackboardArtifact addWebFormAutofill(String name, String value, + long creationTime, long accessTime, int count) { + return addWebFormAutofill(name, value, creationTime, accessTime, count, + Collections.emptyList() ); + } + + /** + * Adds a TSK_WEB_FORM_AUTOFILL artifact + * + * @param name name of autofill field + * @param value value of autofill field + * @param creationTime create date/time + * @param accessTime last access date/time + * @param count count of times used + * @param otherAttributesList additional attributes + * + * @return artifact created + */ + public BlackboardArtifact addWebFormAutofill(String name, String value, + long creationTime, long accessTime, int count, + Collection otherAttributesList ) { + BlackboardArtifact webFormAutofillArtifact = null; + try { + // Create artifact + webFormAutofillArtifact = dbAbstractFile.newArtifact(ARTIFACT_TYPE.TSK_WEB_FORM_AUTOFILL); + + // Add basic attributes + webFormAutofillArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_NAME, moduleName, name)); + webFormAutofillArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_VALUE, moduleName, value)); + if (creationTime > 0) { + webFormAutofillArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_CREATED, moduleName, creationTime)); + } + if (accessTime > 0) { + webFormAutofillArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_ACCESSED, moduleName, accessTime)); + } + if (count > 0) { + webFormAutofillArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_COUNT, moduleName, count)); + } + + // Add other specified attributes + for (BlackboardAttribute otherAttribute: otherAttributesList) { + webFormAutofillArtifact.addAttribute(otherAttribute); + } + + // post artifact + Case.getCurrentCase().getSleuthkitCase().getBlackboard().postArtifact(webFormAutofillArtifact, this.moduleName); + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, "Unable to add web form autofill artifact", ex); //NON-NLS + return null; + } + catch (Blackboard.BlackboardException ex) { + logger.log(Level.SEVERE, String.format("Unable to post artifact %s", ((webFormAutofillArtifact != null)? webFormAutofillArtifact.getArtifactID() : "")), ex); //NON-NLS + } + + // return the artifact + return webFormAutofillArtifact; + } + + + /** + * Adds a TSK_WEB_FORM_AUTOFILL artifact. + * + * @param personName person name + * @param email email address + * @param phoneNumber phone number + * @param mailingAddress mailing address + * @param creationTime creation time + * @param accessTime last access time + * @param count use count + * + * @return artifact created + */ + public BlackboardArtifact addWebFormAddress(String personName, String email, + String phoneNumber, String mailingAddress, + long creationTime, long accessTime, int count ) { + return addWebFormAddress(personName, email, phoneNumber, + mailingAddress, creationTime, accessTime, count, + Collections.emptyList() ); + } + + /** + * Adds a TSK_WEB_FORM_AUTOFILL artifact. + * + * @param personName person name + * @param email email address + * @param phoneNumber phone number + * @param mailingAddress mailing address + * @param creationTime creation time + * @param accessTime last access time + * @param count use count + * @param otherAttributesList other attributes + * + * @return artifact created + */ + public BlackboardArtifact addWebFormAddress(String personName, String email, + String phoneNumber, String mailingAddress, + long creationTime, long accessTime, int count, + Collection otherAttributesList ) { + + BlackboardArtifact webFormAddressArtifact = null; + try { + // Create artifact + webFormAddressArtifact = dbAbstractFile.newArtifact(ARTIFACT_TYPE.TSK_WEB_FORM_AUTOFILL); + + // Add basic attributes + webFormAddressArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_NAME, moduleName, personName)); + if (creationTime > 0) { + webFormAddressArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_CREATED, moduleName, creationTime)); + } + if (accessTime > 0) { + webFormAddressArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_ACCESSED, moduleName, accessTime)); + } + if (count > 0) { + webFormAddressArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_COUNT, moduleName, count)); + } + + if (!StringUtils.isEmpty(email)) { + webFormAddressArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_EMAIL, moduleName, email)); + } + if (!StringUtils.isEmpty(phoneNumber)) { + webFormAddressArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PHONE_NUMBER, moduleName, phoneNumber)); + } + if (!StringUtils.isEmpty(mailingAddress)) { + webFormAddressArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_LOCATION, moduleName, mailingAddress)); + } + + // Add other specified attributes + for (BlackboardAttribute otherAttribute: otherAttributesList) { + webFormAddressArtifact.addAttribute(otherAttribute); + } + + // post artifact + Case.getCurrentCase().getSleuthkitCase().getBlackboard().postArtifact(webFormAddressArtifact, this.moduleName); + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, "Unable to add web form address artifact", ex); //NON-NLS + return null; + } + catch (Blackboard.BlackboardException ex) { + logger.log(Level.SEVERE, String.format("Unable to post artifact %s", ((webFormAddressArtifact != null)? webFormAddressArtifact.getArtifactID() : "")), ex); //NON-NLS + } + + // return the artifact + return webFormAddressArtifact; + } + + /** + * Adds a TSK_INSTALLED_PROGRAM artifact + * + * @param programName name of program + * @param dateInstalled date of install + * + * @return artifact added + */ + public BlackboardArtifact addInstalledProgram(String programName, long dateInstalled) { + return addInstalledProgram(programName, dateInstalled, + Collections.emptyList() ); + } + + /** + * Adds a TSK_INSTALLED_PROGRAM artifact + * + * @param programName name of program + * @param dateInstalled date of install + * @param otherAttributesList additional attributes + * + * @return artifact added + */ + public BlackboardArtifact addInstalledProgram(String programName, long dateInstalled, + Collection otherAttributesList ) { + + BlackboardArtifact installedProgramArtifact = null; + try { + // Create artifact + installedProgramArtifact = dbAbstractFile.newArtifact(ARTIFACT_TYPE.TSK_INSTALLED_PROG); + + // Add basic attributes + installedProgramArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PROG_NAME, moduleName, programName)); + if (dateInstalled > 0) { + installedProgramArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME, moduleName, dateInstalled)); + } + + // Add other specified attributes + for (BlackboardAttribute otherAttribute: otherAttributesList) { + installedProgramArtifact.addAttribute(otherAttribute); + } + + // post artifact + Case.getCurrentCase().getSleuthkitCase().getBlackboard().postArtifact(installedProgramArtifact, this.moduleName); + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, "Unable to add installed program artifact", ex); //NON-NLS + return null; + } + catch (Blackboard.BlackboardException ex) { + logger.log(Level.SEVERE, String.format("Unable to post artifact %s", ((installedProgramArtifact != null)? installedProgramArtifact.getArtifactID() : "")), ex); //NON-NLS + } + + // return the artifact + return installedProgramArtifact; + } + + + /** + * Adds a TSK_GPS_TRACKPOINT artifact + * + * @param latitude location latitude + * @param longitude location longitude + * @param timeStamp date/time trackpoint recoded + * @param poiName trackpoint name + * @param programName name of program that recorded trackpoint + * + * @return artifact added + */ + public BlackboardArtifact addGPSLocation(double latitude, double longitude, + long timeStamp, String poiName, String programName) { + + return addGPSLocation(latitude, longitude, timeStamp, poiName, programName, + Collections.emptyList()); + } + + /** + * Adds a TSK_GPS_TRACKPOINT artifact + * + * @param latitude location latitude + * @param longitude location longitude + * @param timeStamp date/time trackpoint recorded + * @param name trackpoint name + * @param programName name of program that recorded trackpoint + * @param otherAttributesList other attributes + * + * @return artifact added + */ + public BlackboardArtifact addGPSLocation(double latitude, double longitude, long timeStamp, String name, String programName, + Collection otherAttributesList) { + + BlackboardArtifact gpsTrackpointArtifact = null; + try { + // Create artifact + gpsTrackpointArtifact = dbAbstractFile.newArtifact(ARTIFACT_TYPE.TSK_GPS_TRACKPOINT); + + // Add basic attributes + gpsTrackpointArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_GEO_LATITUDE, moduleName, latitude)); + gpsTrackpointArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_GEO_LONGITUDE, moduleName, longitude)); + if (timeStamp > 0) { + gpsTrackpointArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME, moduleName, timeStamp)); + } + + if (!StringUtils.isEmpty(name)) { + gpsTrackpointArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_NAME, moduleName, name)); + } + + if (!StringUtils.isEmpty(programName)) { + gpsTrackpointArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PROG_NAME, moduleName, programName)); + } + + // Add other specified attributes + for (BlackboardAttribute otherAttribute: otherAttributesList) { + gpsTrackpointArtifact.addAttribute(otherAttribute); + } + + // post artifact + Case.getCurrentCase().getSleuthkitCase().getBlackboard().postArtifact(gpsTrackpointArtifact, this.moduleName); + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, "Unable to add GPS trackpoint artifact", ex); //NON-NLS + return null; + } + catch (Blackboard.BlackboardException ex) { + logger.log(Level.SEVERE, String.format("Unable to post artifact %s", ((gpsTrackpointArtifact != null)? gpsTrackpointArtifact.getArtifactID() : "")), ex); //NON-NLS + } + + // return the artifact + return gpsTrackpointArtifact; + } + +} diff --git a/Core/src/org/sleuthkit/autopsy/coreutils/AppSQLiteDB.java b/Core/src/org/sleuthkit/autopsy/coreutils/AppSQLiteDB.java new file mode 100644 index 0000000000..fe9a86c0a8 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/coreutils/AppSQLiteDB.java @@ -0,0 +1,268 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2019 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.sleuthkit.autopsy.coreutils; + +import java.io.Closeable; +import java.io.File; +import java.io.IOException; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.logging.Level; +import org.sleuthkit.autopsy.casemodule.Case; +import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; +import org.sleuthkit.autopsy.casemodule.services.FileManager; +import org.sleuthkit.autopsy.casemodule.services.Services; +import org.sleuthkit.autopsy.datamodel.ContentUtils; +import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.datamodel.DataSource; +import org.sleuthkit.datamodel.ReadContentInputStream; +import org.sleuthkit.datamodel.SleuthkitCase; +import org.sleuthkit.datamodel.TskCoreException; + +/** + * An abstraction around an SQLite app DB found in a data source. + * This class makes a copy of it, opens a SQLite connection to it + * and runs queries on it. + */ +public final class AppSQLiteDB implements Closeable { + private final Logger logger = Logger.getLogger(AppSQLiteDB.class.getName()); + + private final AbstractFile dbAbstractFile; // AbstractFile for the DB file + + private Connection connection = null; + private Statement statement = null; + + private AppSQLiteDB(AbstractFile dbAbstractFile, File dbFileCopy) { + this.dbAbstractFile = dbAbstractFile; + + try { + Class.forName("org.sqlite.JDBC"); //NON-NLS //load JDBC driver + connection = DriverManager.getConnection("jdbc:sqlite:" + dbFileCopy.getPath()); //NON-NLS + statement = connection.createStatement(); + } catch (ClassNotFoundException | SQLException e) { + logger.log(Level.SEVERE, "Error opening database " + dbFileCopy.getPath(), e); //NON-NLS + connection = null; + statement = null; + } + } + + + /** + * Looks for the given SQLIte database filename, with matching path substring. + * It makes a copy of each matching file, and creates an instance of + * AppSQLiteDB to help query the DB. + * + * A list of AppSQLiteDB instances is returned, one for each + * match found., + * . + * @param dataSource data source to search in + * @param dbName db file name to search + * @param parentPathSubstr path substring to match + * + * @return AbstractFile for the DB if the database file is found. + * Returns NULL if no such database is found. + */ + public static Collection findAppDatabases(DataSource dataSource, String dbName, String parentPathSubstr) { + return AppSQLiteDB.findAppDatabases(dataSource, dbName, false, parentPathSubstr); + } + + /** + * Looks for the given SQLIte database filename, with matching path substring. + * It looks for exact name or a pattern match based on + * It makes a copy of each matching file, and creates an instance of + * AppSQLiteDB to help query the DB. + * + * A list of AppSQLiteDB instances is returned, one for each + * match found., + * . + * @param dataSource data source to search in + * @param dbName db file name to search + * @param matchPattern whether to look for a pattern match or an exact match + * @param parentPathSubstr path substring to match + * + * @return AbstractFile for the DB if the database file is found. + * Returns NULL if no such database is found. + */ + public static Collection findAppDatabases(DataSource dataSource, String dbName, boolean matchPattern, String parentPathSubstr) { + + List appDbs = new ArrayList<> (); + Case openCase; + + try { + openCase = Case.getCurrentCaseThrows(); + } catch (NoCurrentCaseException ex) { + Logger.getLogger(AppSQLiteDB.class.getName()).log(Level.SEVERE, "Exception while getting open case.", ex); //NON-NLS + return appDbs; + } + + List absFiles; + long fileId = 0; + String localDiskPath = ""; + try { + SleuthkitCase skCase = openCase.getSleuthkitCase(); + String parentPath = parentPathSubstr.replace("\\", "/"); + parentPath = SleuthkitCase.escapeSingleQuotes(parentPath); + String whereClause; + if (matchPattern) { + whereClause = String.format("LOWER(name) LIKE LOWER(\'%%%1$s%%\') AND LOWER(name) NOT LIKE LOWER(\'%%journal%%\') AND LOWER(parent_path) LIKE LOWER(\'%%%2$s%%\') AND data_source_obj_id = " + dataSource.getId(), dbName, parentPath); + } else { + whereClause = String.format("LOWER(name) LIKE LOWER(\'%1$s\') AND LOWER(parent_path) LIKE LOWER(\'%%%2$s%%\') AND data_source_obj_id = " + dataSource.getId(), dbName, parentPath); + } + absFiles = skCase.findAllFilesWhere(whereClause); + for (AbstractFile absFile : absFiles) { + try { + localDiskPath = openCase.getTempDirectory() + + File.separator + absFile.getId() + absFile.getName(); + File jFile = new java.io.File(localDiskPath); + fileId = absFile.getId(); + ContentUtils.writeToFile(absFile, jFile); + + //Find and copy both WAL and SHM meta files + findAndCopySQLiteMetaFile(absFile, absFile.getName() + "-wal"); + findAndCopySQLiteMetaFile(absFile, absFile.getName() + "-shm"); + + appDbs.add(new AppSQLiteDB(absFile, jFile) ); + } catch (ReadContentInputStream.ReadContentInputStreamException ex) { + Logger.getLogger(AppSQLiteDB.class.getName()).log(Level.WARNING, String.format("Error reading content from file '%s' (id=%d).", absFile.getName(), fileId), ex); //NON-NLS + } catch (IOException | NoCurrentCaseException | TskCoreException ex) { + Logger.getLogger(AppSQLiteDB.class.getName()).log(Level.SEVERE, String.format("Error writing content from file '%s' (id=%d) to '%s'.", absFile.getName(), fileId, localDiskPath), ex); //NON-NLS + } + } + } catch (TskCoreException e) { + Logger.getLogger(AppSQLiteDB.class.getName()).log(Level.SEVERE, "Error finding application DB file.", e); //NON-NLS + } + return appDbs; + } + + public AbstractFile getDBFile() { + return this.dbAbstractFile; + } + + /** + * Checks if the specified table exists in the given database file. + * + * @param tableName table name to check + * + * @return + */ + public boolean tableExists(String tableName) { + // RAMAN TBD + return false; + + } + + /** + * Checks if the specified column exists. + * + * @param tableName table name to check + * @param columnName column name to check + * @return + */ + public boolean columnExists(String tableName, String columnName) { + // RAMAN TBD + return false; + } + + + + + /** + * Runs the given query on the database and returns result set. + + * @param queryStr SQL string for the query to run + * + * @return ResultSet from running the query. + * + * @throws TskCoreException in case of an error. + * + */ + public ResultSet runQuery(String queryStr) throws TskCoreException { + ResultSet resultSet = null; + try { + resultSet = statement.executeQuery(queryStr); //NON-NLS + } + catch (SQLException ex) { + throw new TskCoreException("Error running app SQLite query. " + ex.getMessage(), ex); + } + + return resultSet; + } + + /** + * Closes the DB connection + * + * @throws IOException + */ + @Override + public void close() throws IOException { + + // Close the DB connection + try { + statement.close(); + connection.close(); + } catch (SQLException e) { + logger.log(Level.SEVERE, "Error closing the database", e); //NON-NLS + } + } + + + + /** + * Searches for a meta file associated with the give SQLite database. If + * found, it copies this file into the temp directory of the current case. + * + * @param sqliteFile file being processed + * @param metaFileName name of meta file to look for + * + * @throws NoCurrentCaseException Case has been closed. + * @throws TskCoreException fileManager cannot find AbstractFile + * files. + * @throws IOException Issue during writing to file. + */ + private static void findAndCopySQLiteMetaFile(AbstractFile sqliteFile, + String metaFileName) throws NoCurrentCaseException, TskCoreException, IOException { + + Case openCase = Case.getCurrentCaseThrows(); + SleuthkitCase sleuthkitCase = openCase.getSleuthkitCase(); + Services services = new Services(sleuthkitCase); + FileManager fileManager = services.getFileManager(); + + List metaFiles = fileManager.findFiles( + sqliteFile.getDataSource(), metaFileName, + sqliteFile.getParent().getName()); + + if (metaFiles != null) { + for (AbstractFile metaFile : metaFiles) { + String localDiskPath = openCase.getTempDirectory() + + File.separator + sqliteFile.getId() + metaFile.getName(); + File localMetaFile = new File(localDiskPath); + if (!localMetaFile.exists()) { + ContentUtils.writeToFile(metaFile, localMetaFile); + } + } + } + } +} diff --git a/InternalPythonModules/android/imo.py b/InternalPythonModules/android/imo.py new file mode 100644 index 0000000000..433b818950 --- /dev/null +++ b/InternalPythonModules/android/imo.py @@ -0,0 +1,127 @@ +""" +Autopsy Forensic Browser + +Copyright 2019 Basis Technology Corp. +Contact: carrier sleuthkit org + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from java.io import File +from java.lang import Class +from java.lang import ClassNotFoundException +from java.lang import Long +from java.lang import String +from java.sql import ResultSet +from java.sql import SQLException +from java.sql import Statement +from java.util.logging import Level +from java.util import ArrayList +from org.apache.commons.codec.binary import Base64 +from org.sleuthkit.autopsy.casemodule import Case +from org.sleuthkit.autopsy.coreutils import Logger +from org.sleuthkit.autopsy.coreutils import MessageNotifyUtil +from org.sleuthkit.autopsy.coreutils import AppSQLiteDB +from org.sleuthkit.autopsy.coreutils import AppDBParserHelper +from org.sleuthkit.autopsy.datamodel import ContentUtils +from org.sleuthkit.autopsy.ingest import IngestJobContext +from org.sleuthkit.datamodel import AbstractFile +from org.sleuthkit.datamodel import BlackboardArtifact +from org.sleuthkit.datamodel import BlackboardAttribute +from org.sleuthkit.datamodel import Content +from org.sleuthkit.datamodel import TskCoreException +from org.sleuthkit.datamodel import Account + +import traceback +import general + +""" +Finds the SQLite DB for IMO, parses the DB for contacts & messages, +and adds artifacts to the case. +""" +class IMOAnalyzer(general.AndroidComponentAnalyzer): + def __init__(self): + self._logger = Logger.getLogger(self.__class__.__name__) + + def analyze(self, dataSource, fileManager, context): + selfAccountId = None + accountDbs = AppSQLiteDB.findAppDatabases(dataSource, "accountdb.db", "com.imo.android.imous") + for accountDb in accountDbs: + try: + accountResultSet = accountDb.runQuery("SELECT uid, name FROM account") + if accountResultSet: + # We can determine the IMO user ID of the device owner. + # Therefore we can create and use a app account and use that + # as a 'self' account instead of a Device account + if not selfAccountId: + selfAccountId = accountResultSet.getString("name") + + except SQLException as ex: + self._logger.log(Level.SEVERE, "Error processing query result for account", ex) + finally: + accountDb.close() + + friendsDbs = AppSQLiteDB.findAppDatabases(dataSource, "imofriends.db", "com.imo.android.imous") + for friendsDb in friendsDbs: + try: + friendsDBHelper = AppDBParserHelper("IMO Parser", friendsDb.getDBFile(), + Account.Type.IMO, Account.Type.IMO, selfAccountId ) + contactsResultSet = friendsDb.runQuery("SELECT buid, name FROM friends") + if contactsResultSet is not None: + while contactsResultSet.next(): + friendsDBHelper.addContact( contactsResultSet.getString("name"), ## name for account + contactsResultSet.getString("name"), ## contact name + "", ## phone + "", ## home phone + "", ## mobile + "") ## email + queryString = "SELECT imdata, last_message, timestamp, message_type, message_read, name FROM messages "\ + "INNER JOIN friends ON friends.buid = messages.buid" + messagesResultSet = friendsDb.runQuery(queryString) + if messagesResultSet is not None: + while messagesResultSet.next(): + direction = "" + fromAddress = None + toAdddress = None + if (messagesResultSet.getInt("message_type") == 1): + direction = "Incoming" + fromAddress = messagesResultSet.getString("name") + else: + direction = "Outgoing" + toAddress = messagesResultSet.getString("name") + + timeStamp = messagesResultSet.getLong("timestamp") / 1000000000 + messageArtifact = friendsDBHelper.addMessage( + contactsResultSet.getString("name"), + "IMO Message", + direction, + fromAddress, + toAddress, + timeStamp, + messagesResultSet.getInt("message_read"), + "", + messagesResultSet.getString("last_message"), + "", + "" ) + + # TBD: parse the imdata JSON structure to figure out if there is an attachment. + # If one exists, add the attachment as a derived file and a child of the message artifact. + + + except SQLException as ex: + self._logger.log(Level.SEVERE, "Error processing query result for IMO friends", ex) + finally: + friendsDb.close() + + + diff --git a/InternalPythonModules/android/module.py b/InternalPythonModules/android/module.py index 07ef78ed8d..6430ec82be 100644 --- a/InternalPythonModules/android/module.py +++ b/InternalPythonModules/android/module.py @@ -46,6 +46,7 @@ import googlemaplocation import tangomessage import textmessage import wwfmessage +import imo class AndroidModuleFactory(IngestModuleFactoryAdapter): @@ -87,7 +88,10 @@ class AndroidIngestModule(DataSourceIngestModule): errors = [] fileManager = Case.getCurrentCase().getServices().getFileManager() - analyzers = [contact.ContactAnalyzer(), calllog.CallLogAnalyzer(), textmessage.TextMessageAnalyzer(), tangomessage.TangoMessageAnalyzer(), wwfmessage.WWFMessageAnalyzer(), googlemaplocation.GoogleMapLocationAnalyzer(), browserlocation.BrowserLocationAnalyzer(), cachelocation.CacheLocationAnalyzer()] + analyzers = [contact.ContactAnalyzer(), calllog.CallLogAnalyzer(), textmessage.TextMessageAnalyzer(), + tangomessage.TangoMessageAnalyzer(), wwfmessage.WWFMessageAnalyzer(), + googlemaplocation.GoogleMapLocationAnalyzer(), browserlocation.BrowserLocationAnalyzer(), + cachelocation.CacheLocationAnalyzer(), imo.IMOAnalyzer()] self.log(Level.INFO, "running " + str(len(analyzers)) + " analyzers") progressBar.switchToDeterminate(len(analyzers)) From 1115f5ca4710c52edba7334043004e59de0d62cc Mon Sep 17 00:00:00 2001 From: Joe Ho Date: Wed, 4 Sep 2019 09:33:24 -0400 Subject: [PATCH 047/102] Fix codacy error --- .../autopsy/logicalimager/dsp/AddLogicalImageTask.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java index fc72965bf6..7bef200321 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java @@ -427,9 +427,9 @@ final class AddLogicalImageTask implements Runnable { query = String.format("data_source_obj_id = '%s' AND meta_addr = '%s' AND name = '%s'", // NON-NLS dataSourceObjId.toString(), fileMetaAddressStr, filename.replace("'", "''")); } else { - parentPath = "/" + ROOT_STR + "/" + vhdFilename + "/" + parentPath; + String newParentPath = "/" + ROOT_STR + "/" + vhdFilename + "/" + parentPath; query = String.format("name = '%s' AND parent_path = '%s'", // NON-NLS - filename.replace("'", "''"), parentPath.replace("'", "''")); + filename.replace("'", "''"), newParentPath.replace("'", "''")); } return query; } From 8b2e51ccc1a5d3b4c5454aa115b1811c842117c1 Mon Sep 17 00:00:00 2001 From: Joe Ho Date: Wed, 4 Sep 2019 09:52:42 -0400 Subject: [PATCH 048/102] Fix PR comments --- .../datamodel/utils/LocalFileImporter.java | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/utils/LocalFileImporter.java b/Core/src/org/sleuthkit/autopsy/datamodel/utils/LocalFileImporter.java index 155f167953..95a47d5399 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/utils/LocalFileImporter.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/utils/LocalFileImporter.java @@ -76,8 +76,8 @@ public class LocalFileImporter { * Will not fail if the fileOnDisk does not exist. * * @param fileOnDisk The local file on disk - * @param nameInImage The name to use in the data source - * @param pathInImage The path to use in the data source + * @param name The name to use in the data source + * @param parentPath The path to use in the data source * @param ctime Change time * @param crtime Created time * @param atime Access time @@ -88,12 +88,12 @@ public class LocalFileImporter { * * @throws TskCoreException */ - public AbstractFile addLocalFile(File fileOnDisk, String nameInImage, String pathInImage, + public AbstractFile addLocalFile(File fileOnDisk, String name, String parentPath, Long ctime, Long crtime, Long atime, Long mtime, DataSource dataSource) throws TskCoreException { // Get the parent folder, creating it and any of its parent folders if necessary - SpecialDirectory parentDir = getLocalFilesDir(new File(pathInImage), dataSource); + SpecialDirectory parentDir = getOrMakeDirInDataSource(new File(parentPath), dataSource); SleuthkitCase.CaseDbTransaction trans = null; try { @@ -110,7 +110,7 @@ public class LocalFileImporter { } // Create the new file - AbstractFile file = sleuthkitCase.addLocalFile(nameInImage, fileOnDisk.getAbsolutePath(), size, + AbstractFile file = sleuthkitCase.addLocalFile(name, fileOnDisk.getAbsolutePath(), size, ctime, crtime, atime, mtime, true, TskData.EncodingType.NONE, parentDir, trans); @@ -131,38 +131,38 @@ public class LocalFileImporter { } /** - * Returns the SpecialDirectory object corresponding to the given file, creating + * Returns the SpecialDirectory object corresponding to the given directory, creating * it and its parents as needed. * - * @param file The file to get the SpecialDirectory for + * @param directory The file to get the SpecialDirectory for * @param dataSource The data source * * @return The SpecialDirectory object corresponding to the given file * * @throws TskCoreException */ - private SpecialDirectory getLocalFilesDir(File file, Content dataSource) throws TskCoreException { - if ((file == null) || file.getPath().isEmpty()) { + private SpecialDirectory getOrMakeDirInDataSource(File directory, Content dataSource) throws TskCoreException { + if ((directory == null) || directory.getPath().isEmpty()) { throw new TskCoreException("Can not create directory from null path"); } // Check if we've already created it - if (localFileDirMap.containsKey(file.toString())) { - return localFileDirMap.get(file.toString()); + if (localFileDirMap.containsKey(directory.toString())) { + return localFileDirMap.get(directory.toString()); } - File parent = file.getParentFile(); + File parent = directory.getParentFile(); if (parent == null) { // This is the root of the path and it isn't in the map, so create it - SpecialDirectory dir = createLocalFilesDir(dataSource.getId(), file.getName()); - localFileDirMap.put(file.getName(), dir); + SpecialDirectory dir = createLocalFilesDir(dataSource.getId(), directory.getName()); + localFileDirMap.put(directory.getName(), dir); return dir; } else { // Create everything above this in the tree, and then add the parent folder - SpecialDirectory parentDir = getLocalFilesDir(parent, dataSource); - SpecialDirectory dir = createLocalFilesDir(parentDir.getId(), file.getName()); - localFileDirMap.put(file.getPath(), dir); + SpecialDirectory parentDir = getOrMakeDirInDataSource(parent, dataSource); + SpecialDirectory dir = createLocalFilesDir(parentDir.getId(), directory.getName()); + localFileDirMap.put(directory.getPath(), dir); return dir; } } From 22d47485c9089862dea8fa0c98ccede03f951ced Mon Sep 17 00:00:00 2001 From: Kelly Kelly Date: Wed, 4 Sep 2019 10:35:45 -0400 Subject: [PATCH 049/102] Changed TSK_FLAGS to TSK_FLAG --- .../recentactivity/ExtractRegistry.java | 241 ++++++++++-------- 1 file changed, 133 insertions(+), 108 deletions(-) diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java index a129bea794..be75bb7101 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java @@ -90,7 +90,7 @@ import org.sleuthkit.datamodel.TskCoreException; "Progress_Message_Analyze_Registry=Analyzing Registry Files" }) class ExtractRegistry extends Extract { - + private static final String USERNAME_KEY = "Username"; //NON-NLS private static final String SID_KEY = "SID"; //NON-NLS private static final String RID_KEY = "RID"; //NON-NLS @@ -116,7 +116,7 @@ class ExtractRegistry extends Extract { private static final String SERVER_TRUST_ACCOUNT = "Server trust account"; private static final String ACCOUNT_AUTO_LOCKED = "Account auto locked"; private static final String PASSWORD_HINT = "Password Hint"; - + private static final String[] PASSWORD_SETTINGS_FLAGS = {PWD_DOES_NOT_EXPIRE_KEY, PWD_NOT_REQUIRED_KEY}; private static final String[] ACCOUNT_SETTINGS_FLAGS = {ACCOUNT_AUTO_LOCKED, HOME_DIRECTORY_REQUIRED_KEY, ACCOUNT_DISABLED_KEY}; private static final String[] ACCOUNT_TYPE_FLAGS = {NORMAL_ACCOUNT_KEY, SERVER_TRUST_ACCOUNT, WORKSTATION_TRUST_ACCOUNT, INTERDOMAIN_TRUST_ACCOUNT_KEY, MNS_LOGON_ACCOUNT_KEY, TEMPORARY_DUPLICATE_ACCOUNT}; @@ -849,7 +849,7 @@ class ExtractRegistry extends Extract { break; } } // for - + postArtifacts(usbBBartifacts); postArtifacts(wifiBBartifacts); return true; @@ -894,17 +894,17 @@ class ExtractRegistry extends Extract { if (line.contains(SECTION_DIVIDER) && previousLine != null && previousLine.contains(userInfoSection)) { readUsers(bufferedReader, userSet); } - - if(line.contains(SECTION_DIVIDER) && previousLine != null && previousLine.contains("Group Membership Information")) { - groupMap = readGroups(bufferedReader); + + if (line.contains(SECTION_DIVIDER) && previousLine != null && previousLine.contains("Group Membership Information")) { + groupMap = readGroups(bufferedReader); } - + previousLine = line; line = bufferedReader.readLine(); } - Map> userInfoMap = new HashMap<>(); + Map> userInfoMap = new HashMap<>(); //load all the user info which was read into a map - for (HashMap userInfo : userSet) { + for (HashMap userInfo : userSet) { userInfoMap.put(userInfo.get(SID_KEY), userInfo); } //get all existing OS account artifacts @@ -915,7 +915,7 @@ class ExtractRegistry extends Extract { BlackboardAttribute existingUserId = osAccount.getAttribute(new BlackboardAttribute.Type(ATTRIBUTE_TYPE.TSK_USER_ID)); if (existingUserId != null) { String userID = existingUserId.getValueString().trim(); - HashMap userInfo = userInfoMap.remove(userID); + HashMap userInfo = userInfoMap.remove(userID); //if the existing user id matches a user id which we parsed information for check if that information exists and if it doesn't add it if (userInfo != null) { osAccount.addAttributes(getAttributesForAccount(userInfo, groupMap.get(userID), true)); @@ -924,7 +924,7 @@ class ExtractRegistry extends Extract { } } //add remaining userinfos as accounts; - for (HashMap userInfo: userInfoMap.values()) { + for (HashMap userInfo : userInfoMap.values()) { BlackboardArtifact bbart = regAbstractFile.newArtifact(ARTIFACT_TYPE.TSK_OS_ACCOUNT); bbart.addAttributes(getAttributesForAccount(userInfo, groupMap.get(userInfo.get(SID_KEY)), false)); // index the artifact for keyword search @@ -943,183 +943,150 @@ class ExtractRegistry extends Extract { } return false; } - - Collection getAttributesForAccount(HashMap userInfo, List groupList, boolean existingUser) throws ParseException { + + /** + * Creates the attribute list for the given user information and group list. + * + * @param userInfo Map of key\value pairs of user information + * @param groupList List of the groups that user belongs + * @param existingUser + * + * @return List + * + * @throws ParseException + */ + Collection getAttributesForAccount(HashMap userInfo, List groupList, boolean existingUser) throws ParseException { Collection bbattributes = new ArrayList<>(); SimpleDateFormat regRipperTimeFormat = new SimpleDateFormat("EEE MMM dd HH:mm:ss yyyy 'Z'"); regRipperTimeFormat.setTimeZone(getTimeZone("GMT")); - - if (! existingUser) { + + if (!existingUser) { bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_USER_ID, getRAModuleName(), userInfo.get(SID_KEY))); bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_USER_NAME, - this.moduleName, userInfo.get(USERNAME_KEY))); + this.moduleName, userInfo.get(USERNAME_KEY))); } - + String value = userInfo.get(ACCOUNT_CREATED_KEY); if (value != null && !value.isEmpty() && !value.equals(NEVER_DATE)) { bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_CREATED, getRAModuleName(), regRipperTimeFormat.parse(value).getTime() / MS_IN_SEC)); } - + value = userInfo.get(LAST_LOGIN_KEY); - if (value != null && !value.isEmpty() && !value.equals(NEVER_DATE) ) { + if (value != null && !value.isEmpty() && !value.equals(NEVER_DATE)) { bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_ACCESSED, getRAModuleName(), regRipperTimeFormat.parse(value).getTime() / MS_IN_SEC)); } - + value = userInfo.get(LOGIN_COUNT_KEY); - if(value != null && !value.isEmpty()) { + if (value != null && !value.isEmpty()) { bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_COUNT, getRAModuleName(), Integer.parseInt(value))); } - + value = userInfo.get(ACCOUNT_TYPE_KEY); - if(value != null && !value.isEmpty()) { + if (value != null && !value.isEmpty()) { bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_ACCOUNT_TYPE, getRAModuleName(), value)); } - + value = userInfo.get(USER_COMMENT_KEY); - if(value != null && !value.isEmpty()) { + if (value != null && !value.isEmpty()) { bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DESCRIPTION, getRAModuleName(), value)); } - + value = userInfo.get(NAME_KEY); - if(value != null && !value.isEmpty()) { + if (value != null && !value.isEmpty()) { bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_NAME, getRAModuleName(), value)); } - + value = userInfo.get(INTERNET_NAME_KEY); - if(value != null && !value.isEmpty()) { + if (value != null && !value.isEmpty()) { bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_EMAIL, getRAModuleName(), value)); } - + value = userInfo.get(FULL_NAME_KEY); - if(value != null && !value.isEmpty()) { + if (value != null && !value.isEmpty()) { bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DISPLAY_NAME, getRAModuleName(), value)); } - + value = userInfo.get(PWD_RESET_KEY); - if(value != null && !value.isEmpty() && !value.equals(NEVER_DATE)) { + if (value != null && !value.isEmpty() && !value.equals(NEVER_DATE)) { bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_PASSWORD_RESET, getRAModuleName(), regRipperTimeFormat.parse(value).getTime() / MS_IN_SEC)); } - + value = userInfo.get(PASSWORD_HINT); - if(value != null && !value.isEmpty()) { + if (value != null && !value.isEmpty()) { bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PASSWORD_HINT, getRAModuleName(), value)); } - + value = userInfo.get(PWD_FAILE_KEY); if (value != null && !value.isEmpty() && !value.equals(NEVER_DATE)) { bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_PASSWORD_FAIL, getRAModuleName(), regRipperTimeFormat.parse(value).getTime() / MS_IN_SEC)); } - + String settingString = ""; - for (String setting: PASSWORD_SETTINGS_FLAGS) { + for (String setting : PASSWORD_SETTINGS_FLAGS) { if (userInfo.containsKey(setting)) { settingString += setting + ", "; } } - + if (!settingString.isEmpty()) { settingString = settingString.substring(0, settingString.length() - 2); bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PASSWORD_SETTINGS, getRAModuleName(), settingString)); } - + settingString = ""; - for (String setting: ACCOUNT_SETTINGS_FLAGS) { + for (String setting : ACCOUNT_SETTINGS_FLAGS) { if (userInfo.containsKey(setting)) { settingString += setting + ", "; } } - + if (!settingString.isEmpty()) { settingString = settingString.substring(0, settingString.length() - 2); bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_ACCOUNT_SETTINGS, getRAModuleName(), settingString)); } - + settingString = ""; - for (String setting: ACCOUNT_TYPE_FLAGS) { + for (String setting : ACCOUNT_TYPE_FLAGS) { if (userInfo.containsKey(setting)) { settingString += setting + ", "; } } - + if (!settingString.isEmpty()) { settingString = settingString.substring(0, settingString.length() - 2); - bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_FLAGS, + bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_FLAG, getRAModuleName(), settingString)); - } - + } + if (groupList != null && groupList.size() > 0) { String groups = new String(); - for (String group: groupList) { + for (String group : groupList) { groups += group + ", "; } - groups = groups.substring(0, groups.length() - 2); + bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_GROUPS, - getRAModuleName(), groups)); + getRAModuleName(), groups.substring(0, groups.length() - 2))); } - + return bbattributes; } - Map> readGroups(BufferedReader bufferedReader) throws IOException { - HashMap> groupMap = new HashMap<>(); - - String line = bufferedReader.readLine(); - - int userCount = 0; - String groupName = null; - - while (line != null && !line.contains(SECTION_DIVIDER)) { - - if(line.contains("Group Name")) { - String value = line.replaceAll("Group Name\\s*?:", "").trim(); - groupName = (value.replaceAll("\\[\\d*?\\]", "")).trim(); - int startIndex = value.indexOf('['); - int endIndex = value.indexOf(']'); - - if(startIndex != -1 && endIndex != -1) { - String countStr = value.substring(startIndex+1, endIndex); - userCount = Integer.parseInt(countStr); - } - } else if(line.matches("Users\\s*?:")) { - for(int i = 0; i < userCount; i++) { - line = bufferedReader.readLine(); - if(line != null) { - String sid = line.trim(); - List groupList = groupMap.get(sid); - if(groupList == null) { - groupList = new ArrayList<>(); - groupMap.put(sid, groupList); - } - - groupList.add(groupName); - } - } - - groupName = null; - } - - line = bufferedReader.readLine(); - } - - return groupMap; - } - /** * Read the User Information section of the SAM regripper plugin's output * and collect user account information from the file. @@ -1131,7 +1098,7 @@ class ExtractRegistry extends Extract { * * @throws IOException */ - private void readUsers(BufferedReader bufferedReader, Set> users) throws IOException { + private void readUsers(BufferedReader bufferedReader, Set> users) throws IOException { String line = bufferedReader.readLine(); //read until end of file or next section divider String userName = ""; @@ -1145,12 +1112,12 @@ class ExtractRegistry extends Extract { user_rid = userNameAndIdString.substring(userNameAndIdString.lastIndexOf('['), userNameAndIdString.lastIndexOf(']')); } else if (line.contains(SID_KEY) && !userName.isEmpty()) { Map.Entry entry = getSAMKeyValue(line); - - HashMap userInfo = new HashMap<>(); + + HashMap userInfo = new HashMap<>(); userInfo.put(USERNAME_KEY, userName); userInfo.put(RID_KEY, user_rid); userInfo.put(entry.getKey(), entry.getValue()); - + //continue reading this users information until end of file or a blank line between users line = bufferedReader.readLine(); while (line != null && !line.isEmpty()) { @@ -1159,38 +1126,96 @@ class ExtractRegistry extends Extract { line = bufferedReader.readLine(); } users.add(userInfo); - + userName = ""; } line = bufferedReader.readLine(); } } + /** + * Maps the user groups to the sid that are a part of them. + * + * @param bufferedReader + * + * @return A map if sid and the groups they map too + * + * @throws IOException + */ + Map> readGroups(BufferedReader bufferedReader) throws IOException { + HashMap> groupMap = new HashMap<>(); + + String line = bufferedReader.readLine(); + + int userCount = 0; + String groupName = null; + + while (line != null && !line.contains(SECTION_DIVIDER)) { + + if (line.contains("Group Name")) { + String value = line.replaceAll("Group Name\\s*?:", "").trim(); + groupName = (value.replaceAll("\\[\\d*?\\]", "")).trim(); + int startIndex = value.indexOf('['); + int endIndex = value.indexOf(']'); + + if (startIndex != -1 && endIndex != -1) { + String countStr = value.substring(startIndex + 1, endIndex); + userCount = Integer.parseInt(countStr); + } + } else if (line.matches("Users\\s*?:")) { + for (int i = 0; i < userCount; i++) { + line = bufferedReader.readLine(); + if (line != null) { + String sid = line.trim(); + List groupList = groupMap.get(sid); + if (groupList == null) { + groupList = new ArrayList<>(); + groupMap.put(sid, groupList); + } + groupList.add(groupName); + } + } + groupName = null; + } + line = bufferedReader.readLine(); + } + return groupMap; + } + + /** + * Gets the key value from user account strings of the format + * key:value or + * --> value + * + * @param line String to parse + * + * @return key value pair + */ private Map.Entry getSAMKeyValue(String line) { int index = line.indexOf(':'); Map.Entry returnValue = null; String key = null; String value = null; - + if (index != -1) { key = line.substring(0, index).trim(); if (index + 1 < line.length()) { - value = line.substring(index+1).trim(); + value = line.substring(index + 1).trim(); } else { value = ""; } - + return new AbstractMap.SimpleEntry<>(key, value); - + } else if (line.contains("-->")) { key = line.replace("-->", "").trim(); value = "true"; } - + if (key != null) { - returnValue = new AbstractMap.SimpleEntry<>(key, value); + returnValue = new AbstractMap.SimpleEntry<>(key, value); } - + return returnValue; } From 3c7510fa2c1cc23ea32f13571461f8de87e01314 Mon Sep 17 00:00:00 2001 From: Joe Ho Date: Wed, 4 Sep 2019 11:06:42 -0400 Subject: [PATCH 050/102] Fix PR comments --- .../logicalimager/configuration/Bundle.properties-MERGED | 6 +++++- .../logicalimager/configuration/ConfigVisualPanel1.java | 9 +++++---- .../autopsy/logicalimager/dsp/Bundle.properties-MERGED | 2 ++ 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/Bundle.properties-MERGED index 55d0634c72..23009e88c1 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/Bundle.properties-MERGED @@ -26,8 +26,12 @@ ConfigVisualPanel1.chooseFileTitle=Select a Logical Imager configuration # {0} - filename ConfigVisualPanel1.configFileIsEmpty=Configuration file {0} is empty ConfigVisualPanel1.configurationError=Configuration error +# {0} - root +# {1} - description +# {2} - size with unit +# {3} - file system +ConfigVisualPanel1.driveListItem={0} ({1}) ({2}) - File system: {3} ConfigVisualPanel1.fileNameExtensionFilter=Configuration JSON File -ConfigVisualPanel1.fileSystem=File system ConfigVisualPanel1.invalidConfigJson=Invalid config JSON: ConfigVisualPanel1.messageLabel.noExternalDriveFound=No drive found ConfigVisualPanel1.selectConfigurationFile=Select location diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/ConfigVisualPanel1.java b/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/ConfigVisualPanel1.java index fc683be97a..f6336771bd 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/ConfigVisualPanel1.java +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/configuration/ConfigVisualPanel1.java @@ -267,7 +267,8 @@ final class ConfigVisualPanel1 extends JPanel { */ @NbBundle.Messages({ "ConfigVisualPanel1.messageLabel.noExternalDriveFound=No drive found", - "ConfigVisualPanel1.fileSystem=File system" + "# {0} - root", "# {1} - description", "# {2} - size with unit", "# {3} - file system", + "ConfigVisualPanel1.driveListItem={0} ({1}) ({2}) - File system: {3}" }) private void refreshDriveList() { List listData = new ArrayList<>(); @@ -282,7 +283,7 @@ final class ConfigVisualPanel1 extends JPanel { long spaceInBytes = root.getTotalSpace(); String sizeWithUnit = DriveListUtils.humanReadableByteCount(spaceInBytes, false); String fileSystem = getFileSystemName(root.toString()); - listData.add(root + " (" + description + ") (" + sizeWithUnit + ") - " + Bundle.ConfigVisualPanel1_fileSystem() + ": " + fileSystem); + listData.add(Bundle.ConfigVisualPanel1_driveListItem(root, description, sizeWithUnit, fileSystem)); if (firstRemovableDrive == -1) { try { FileStore fileStore = Files.getFileStore(root.toPath()); @@ -467,8 +468,8 @@ final class ConfigVisualPanel1 extends JPanel { */ boolean isPanelValid() { return !StringUtils.isBlank(getConfigPath()) - && (getFileSystemName(getConfigPath().substring(0, 3)).equals("NTFS") // NON-NLS - || getFileSystemName(getConfigPath().substring(0, 3)).equals("exFAT")) // NON-NLS + && !(getFileSystemName(getConfigPath().substring(0, 3)).equals("FAT") // NON-NLS + || getFileSystemName(getConfigPath().substring(0, 3)).equals("FAT32")) // NON-NLS && ((configureDriveRadioButton.isSelected() && !StringUtils.isBlank(driveList.getSelectedValue())) || (configureFolderRadioButton.isSelected() && (!configFileTextField.getText().isEmpty()))); } diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/Bundle.properties-MERGED index 76b82c57ad..dfcde750b9 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/Bundle.properties-MERGED @@ -5,6 +5,8 @@ AddLogicalImageTask.addingInterestingFiles=Adding search results as interesting files # {0} - file AddLogicalImageTask.addingToReport=Adding {0} to report +# {0} - target image path +AddLogicalImageTask.cannotFindDataSourceObjId=Cannot find obj_id in tsk_image_names for {0} # {0} - SearchResults.txt # {1} - directory AddLogicalImageTask.cannotFindFiles=Cannot find {0} in {1} From a118e0f881a86369179e32260549922a6912efd7 Mon Sep 17 00:00:00 2001 From: Joe Ho Date: Wed, 4 Sep 2019 11:53:09 -0400 Subject: [PATCH 051/102] Fix codacy error --- .../corecomponents/DataResultViewerTable.java | 59 +++++++++---------- .../dsp/AddLogicalImageTask.java | 15 ++--- 2 files changed, 37 insertions(+), 37 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/corecomponents/DataResultViewerTable.java b/Core/src/org/sleuthkit/autopsy/corecomponents/DataResultViewerTable.java index 1c3a371d9f..6d5788fd5f 100644 --- a/Core/src/org/sleuthkit/autopsy/corecomponents/DataResultViewerTable.java +++ b/Core/src/org/sleuthkit/autopsy/corecomponents/DataResultViewerTable.java @@ -434,7 +434,7 @@ public class DataResultViewerTable extends AbstractDataResultViewer { } setColumnWidths(); - + /* * Load column sorting information from preferences file and apply it to * columns. @@ -516,7 +516,7 @@ public class DataResultViewerTable extends AbstractDataResultViewer { protected void setColumnWidths() { if (rootNode.getChildren().getNodesCount() != 0) { final Graphics graphics = outlineView.getGraphics(); - + if (graphics != null) { // Current width of the outlineView double outlineViewWidth = outlineView.getSize().getWidth(); @@ -526,10 +526,10 @@ public class DataResultViewerTable extends AbstractDataResultViewer { int margin = 4; int padding = 8; - + int totalColumnWidth = 0; - int cntMaxSizeColumns =0; - + int cntMaxSizeColumns = 0; + // Calulate the width for each column keeping track of the number // of columns that were set to columnwidthLimit. for (int column = 0; column < outline.getModel().getColumnCount(); column++) { @@ -552,40 +552,40 @@ public class DataResultViewerTable extends AbstractDataResultViewer { columnWidth = Math.min(columnWidth, columnWidthLimit); columnWidths.add(columnWidth); - + totalColumnWidth += columnWidth; - - if( columnWidth == columnWidthLimit) { + + if (columnWidth == columnWidthLimit) { cntMaxSizeColumns++; } } - + // Figure out how much extra, if any can be given to the columns // so that the table is as wide as outlineViewWidth. If cntMaxSizeColumns // is greater than 0 divide the extra space between the columns // that could use more space. Otherwise divide evenly amoung // all columns. int extraWidth = 0; - + if (totalColumnWidth < outlineViewWidth) { - if (cntMaxSizeColumns > 0) { - extraWidth = (int) ((outlineViewWidth - totalColumnWidth)/cntMaxSizeColumns); + if (cntMaxSizeColumns > 0) { + extraWidth = (int) ((outlineViewWidth - totalColumnWidth) / cntMaxSizeColumns); } else { - extraWidth = (int) ((outlineViewWidth - totalColumnWidth)/columnWidths.size()); + extraWidth = (int) ((outlineViewWidth - totalColumnWidth) / columnWidths.size()); } } - - for(int column = 0; column < columnWidths.size(); column++) { + + for (int column = 0; column < columnWidths.size(); column++) { int columnWidth = columnWidths.get(column); - - if(cntMaxSizeColumns > 0) { - if(columnWidth >= ((column == 0) ? 350 : 300)) { + + if (cntMaxSizeColumns > 0) { + if (columnWidth >= ((column == 0) ? 350 : 300)) { columnWidth += extraWidth; } } else { columnWidth += extraWidth; } - + outline.getColumnModel().getColumn(column).setPreferredWidth(columnWidth); } } @@ -896,23 +896,22 @@ public class DataResultViewerTable extends AbstractDataResultViewer { "DataResultViewerTable.goToPageTextField.msgDlg=Please enter a valid page number between 1 and {0}", "DataResultViewerTable.goToPageTextField.err=Invalid page number"}) void gotoPage() { - int saveCurrentPage = currentPage; try { + int saveCurrentPage = currentPage; currentPage = Integer.decode(gotoPageTextField.getText()); + if (currentPage > totalPages || currentPage < 1) { + currentPage = saveCurrentPage; + JOptionPane.showMessageDialog(DataResultViewerTable.this, + Bundle.DataResultViewerTable_goToPageTextField_msgDlg(totalPages), + Bundle.DataResultViewerTable_goToPageTextField_err(), + JOptionPane.WARNING_MESSAGE); + return; + } + postPageChangeEvent(); } catch (NumberFormatException e) { //ignore input return; } - - if (currentPage > totalPages || currentPage < 1) { - currentPage = saveCurrentPage; - JOptionPane.showMessageDialog(DataResultViewerTable.this, - Bundle.DataResultViewerTable_goToPageTextField_msgDlg(totalPages), - Bundle.DataResultViewerTable_goToPageTextField_err(), - JOptionPane.WARNING_MESSAGE); - return; - } - postPageChangeEvent(); } /** diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java index 8d56b676cc..7d9115741d 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java @@ -217,14 +217,14 @@ final class AddLogicalImageTask implements Runnable { try { if (createVHD) { // Wait for addMultipleImageTask to finish, via its privateCallback - while (privateCallback.inProgress()) { + while (privateCallback.isInProgress()) { try { Thread.sleep(1000); } catch (InterruptedException ex) { // Got the addMultipleImageTask stop (cancel) LOGGER.log(Level.INFO, "AddMultipleImageTask interrupted", ex); // NON-NLS // now wait for addMultipleImageTask to revert and finally callback - while (privateCallback.inProgress()) { + while (privateCallback.isInProgress()) { try { Thread.sleep(1000); } catch (InterruptedException ex2) { @@ -296,12 +296,10 @@ final class AddLogicalImageTask implements Runnable { multipleImageThread.interrupt(); addMultipleImageTask.cancelTask(); } - if (!createVHD) { + if (!createVHD && !addingInterestingFiles) { // Don't delete destination directory once we started adding interesting files. // At this point the database and destination directory are complete. - if (!addingInterestingFiles) { - deleteDestinationDirectory(); - } + deleteDestinationDirectory(); } } @@ -478,6 +476,9 @@ final class AddLogicalImageTask implements Runnable { } } + /** + * AddDataSourceCallback private class for adding VHD source + */ private class AddDataSourceCallback extends DataSourceProcessorCallback { private List errorMessages; private List newDataSources; @@ -510,7 +511,7 @@ final class AddLogicalImageTask implements Runnable { return result; } - private boolean inProgress() { + private boolean isInProgress() { return inProgress; } } From b19032bcc43b5175e2755d1f29ea928598d69f5d Mon Sep 17 00:00:00 2001 From: Kelly Kelly Date: Thu, 5 Sep 2019 10:34:44 -0400 Subject: [PATCH 052/102] Update cvt prop file with new labels --- .../communications/relationships/Bundle.properties-MERGED | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/communications/relationships/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/communications/relationships/Bundle.properties-MERGED index 67971ffa88..f79bdaa464 100755 --- a/Core/src/org/sleuthkit/autopsy/communications/relationships/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/communications/relationships/Bundle.properties-MERGED @@ -60,6 +60,6 @@ MessageViewer.backButton.AccessibleContext.accessibleDescription= MessageViewer.backButton.text=Threads MessageViewer.showAllButton.text=All Messages SummaryViewer.thumbnailCntLabel.text=Media Attachments: -SummaryViewer.attachmentsLable.text=Attachments: +SummaryViewer.attachmentsLable.text=Total Attachments: SummaryViewer.thumbnailsDataLabel.text=attachments -SummaryViewer.attachmentDataLabel.text=jLabel1 +SummaryViewer.attachmentDataLabel.text=count From 37e1f043ed8b659c5eccb461a714edde2c2929e1 Mon Sep 17 00:00:00 2001 From: Joe Ho Date: Fri, 6 Sep 2019 13:29:23 -0400 Subject: [PATCH 053/102] Initial check in --- .../logicalimager/dsp/AddLogicalImageTask.java | 11 ++++++++--- .../logicalimager/dsp/Bundle.properties-MERGED | 2 ++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java index 72600f720c..f259b6bfb9 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java @@ -23,6 +23,7 @@ import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStreamReader; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; @@ -190,6 +191,7 @@ final class AddLogicalImageTask extends AddMultipleImageTask { private void addInterestingFiles(File src, Path resultsPath) throws IOException, TskCoreException { Map> imagePaths = currentCase.getSleuthkitCase().getImagePaths(); Map imagePathToObjIdMap = imagePathsToDataSourceObjId(imagePaths); + long totalFiles = Files.lines(resultsPath).count() - 1; // skip the header line try (BufferedReader br = new BufferedReader(new InputStreamReader( new FileInputStream(resultsPath.toFile()), "UTF8"))) { // NON-NLS @@ -198,9 +200,9 @@ final class AddLogicalImageTask extends AddMultipleImageTask { int lineNumber = 2; while ((line = br.readLine()) != null) { String[] fields = line.split("\t", -1); // NON-NLS - if (fields.length != 9) { - throw new IOException(Bundle.AddLogicalImageTask_notEnoughFields(lineNumber, fields.length, 9)); - } +// if (fields.length != 9) { +// throw new IOException(Bundle.AddLogicalImageTask_notEnoughFields(lineNumber, fields.length, 9)); +// } String vhdFilename = fields[0]; String targetImagePath = Paths.get(src.toString(), vhdFilename).toString(); @@ -218,6 +220,9 @@ final class AddLogicalImageTask extends AddMultipleImageTask { String filename = fields[7]; // String parentPath = fields[8]; + if (lineNumber % 100 == 0) { + progressMonitor.setProgressText(String.format("Adding interesting file %d of %d", lineNumber, totalFiles)); + } String query = String.format("data_source_obj_id = '%s' AND meta_addr = '%s' AND name = '%s'", // NON-NLS dataSourceObjId.toString(), fileMetaAddressStr, filename); List matchedFiles = Case.getCurrentCase().getSleuthkitCase().findAllFilesWhere(query); diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/Bundle.properties-MERGED index 76b82c57ad..dfcde750b9 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/Bundle.properties-MERGED @@ -5,6 +5,8 @@ AddLogicalImageTask.addingInterestingFiles=Adding search results as interesting files # {0} - file AddLogicalImageTask.addingToReport=Adding {0} to report +# {0} - target image path +AddLogicalImageTask.cannotFindDataSourceObjId=Cannot find obj_id in tsk_image_names for {0} # {0} - SearchResults.txt # {1} - directory AddLogicalImageTask.cannotFindFiles=Cannot find {0} in {1} From c9a763bd2cb9d04706a0ae670ab15b809d4796aa Mon Sep 17 00:00:00 2001 From: Joe Ho Date: Fri, 6 Sep 2019 14:20:45 -0400 Subject: [PATCH 054/102] Fix createVHD query --- .../autopsy/logicalimager/dsp/AddLogicalImageTask.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java index 3151ddccde..e5556b0c02 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java @@ -333,7 +333,7 @@ final class AddLogicalImageTask implements Runnable { } String[] fields = line.split("\t", -1); // NON-NLS if (fields.length != 14) { - throw new IOException(Bundle.AddLogicalImageTask_notEnoughFields(lineNumber, fields.length, 9)); + throw new IOException(Bundle.AddLogicalImageTask_notEnoughFields(lineNumber, fields.length, 14)); } String vhdFilename = fields[0]; // String fileSystemOffsetStr = fields[1]; @@ -506,7 +506,7 @@ final class AddLogicalImageTask implements Runnable { if (createVHD) { Map> imagePaths = currentCase.getSleuthkitCase().getImagePaths(); Map imagePathToObjIdMap = imagePathsToDataSourceObjId(imagePaths); - String targetImagePath = Paths.get(src.toString(), vhdFilename).toString(); + String targetImagePath = Paths.get(dest.toString(), vhdFilename).toString(); Long dataSourceObjId = imagePathToObjIdMap.get(targetImagePath); if (dataSourceObjId == null) { throw new TskCoreException(Bundle.AddLogicalImageTask_cannotFindDataSourceObjId(targetImagePath)); From 94d8a10cb579949dd4c4f3f247904c92aaca3c6c Mon Sep 17 00:00:00 2001 From: Raman Date: Fri, 6 Sep 2019 16:14:50 -0400 Subject: [PATCH 055/102] 5394: App DB parser helper - Added support for messages with multiple recipients. --- .../autopsy/coreutils/AppDBParserHelper.java | 164 ++++++++++++++---- .../autopsy/coreutils/AppSQLiteDB.java | 29 +--- InternalPythonModules/android/imo.py | 36 ++-- 3 files changed, 155 insertions(+), 74 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/coreutils/AppDBParserHelper.java b/Core/src/org/sleuthkit/autopsy/coreutils/AppDBParserHelper.java index 34c0fefc48..32848c6437 100644 --- a/Core/src/org/sleuthkit/autopsy/coreutils/AppDBParserHelper.java +++ b/Core/src/org/sleuthkit/autopsy/coreutils/AppDBParserHelper.java @@ -18,8 +18,10 @@ */ package org.sleuthkit.autopsy.coreutils; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.List; import java.util.logging.Level; import org.apache.commons.lang3.StringUtils; import org.sleuthkit.autopsy.casemodule.Case; @@ -45,6 +47,16 @@ public final class AppDBParserHelper { private static final Logger logger = Logger.getLogger(AppDBParserHelper.class.getName()); + /** + * Enum for message read status + */ + public enum MessageReadStatusEnum { + + UNKNOWN, /// read status is unknown + UNREAD, /// message has not been read + READ /// message has been read + } + private final AbstractFile dbAbstractFile; private final String moduleName; @@ -253,11 +265,10 @@ public final class AppDBParserHelper { * Also creates an account instance for the sender/receiver, and creates a * relationship between the self account and the sender/receiver account. * - * @param otherAccountUniqueID unique id for the sender/receiver account * @param messageType message type * @param direction message direction - * @param fromAddress sender address, may be empty - * @param toAddress recipient address, may be empty + * @param fromAddress sender address, may be null + * @param toAddress recipient address, may be null * @param dateTime date/time of message, * @param readStatus message read or not * @param subject message subject, may be empty @@ -266,10 +277,13 @@ public final class AppDBParserHelper { * * @return message artifact */ - public BlackboardArtifact addMessage(String otherAccountUniqueID, - String messageType, String direction, String fromAddress, String toAddress, - long dateTime, int readStatus, String subject, String messageText, String threadId) { - return addMessage(otherAccountUniqueID, messageType, direction, + public BlackboardArtifact addMessage( + String messageType, String direction, + Account.Address fromAddress, + Account.Address toAddress, + long dateTime, MessageReadStatusEnum readStatus, + String subject, String messageText, String threadId) { + return addMessage(messageType, direction, fromAddress, toAddress, dateTime, readStatus, subject, messageText, threadId, Collections.emptyList()); @@ -281,7 +295,6 @@ public final class AppDBParserHelper { * Also creates an account instance for the sender/receiver, and creates a * relationship between the self account and the sender/receiver account. * - * @param otherAccountUniqueID unique id for the sender/receiver account * @param messageType message type * @param direction message direction * @param fromAddress sender address, may be empty @@ -296,12 +309,75 @@ public final class AppDBParserHelper { * * @return message artifact */ - public BlackboardArtifact addMessage(String otherAccountUniqueID, - String messageType, String direction, String fromAddress, - String toAddress, long dateTime, int readStatus, String subject, - String messageText, String threadId, - Collection otherAttributesList) { + public BlackboardArtifact addMessage( String messageType, String direction, + Account.Address fromAddress, + Account.Address toAddress, + long dateTime, MessageReadStatusEnum readStatus, String subject, + String messageText, String threadId, + Collection otherAttributesList) { + return addMessage(messageType, direction, + fromAddress, + Arrays.asList(toAddress), + dateTime, readStatus, + subject, messageText, threadId, + otherAttributesList); + } + + /** + * Adds a TSK_MESSAGE artifact. + * + * Also creates an account instance for the sender/receiver, and creates a + * relationship between the self account and the sender/receiver account. + * + * This method is for messages with a multiple recipients. + * + * @param messageType message type + * @param direction message direction + * @param fromAddress sender address, may be null + * @param recipientsList recipient address list, may be null or empty list + * @param dateTime date/time of message, + * @param readStatus message read or not + * @param subject message subject, may be empty + * @param messageText message body, may be empty + * @param threadId, message thread id + * + * + * @return message artifact + */ + public BlackboardArtifact addMessage( String messageType, String direction, + Account.Address fromAddress, + List recipientsList, + long dateTime, MessageReadStatusEnum readStatus, + String subject, String messageText, String threadId) { + return addMessage( messageType, direction, + fromAddress, recipientsList, + dateTime, readStatus, + subject, messageText, threadId, + Collections.emptyList()); + } + + + public BlackboardArtifact addMessage( String messageType, String direction, + Account.Address fromAddress, + List recipientsList, + long dateTime, MessageReadStatusEnum readStatus, + String subject, String messageText, + String threadId, + Collection otherAttributesList) { + + + // Create a comma separated string of recipients + String toAddresses = null; + if (recipientsList != null && (!recipientsList.isEmpty())) { + StringBuilder toAddressesSb = new StringBuilder(); + for(Account.Address recipient : recipientsList) { + toAddressesSb = toAddressesSb.length() > 0 ? toAddressesSb.append(",").append(recipient.getDisplayName()) : toAddressesSb.append(recipient.getDisplayName()); + } + toAddresses = toAddressesSb.toString(); + } + + // Created message artifact. BlackboardArtifact msgArtifact = null; try { // Create TSK_MESSAGE artifact @@ -309,10 +385,10 @@ public final class AppDBParserHelper { if (dateTime > 0) { msgArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME, moduleName, dateTime)); } - if (readStatus > 0) { - msgArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_READ_STATUS, moduleName, readStatus)); + if (readStatus != MessageReadStatusEnum.UNKNOWN) { + msgArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_READ_STATUS, moduleName, (readStatus == MessageReadStatusEnum.READ) ? 1 : 0)); } - + // Add basic attribute, if the correspond value is specified if (!StringUtils.isEmpty(messageType)) { msgArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_MESSAGE_TYPE, moduleName, messageType)); @@ -320,13 +396,13 @@ public final class AppDBParserHelper { if (!StringUtils.isEmpty(direction)) { msgArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DIRECTION, moduleName, direction)); } - if (!StringUtils.isEmpty(fromAddress)) { - msgArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_FROM, moduleName, fromAddress)); + if (fromAddress != null && !StringUtils.isEmpty(fromAddress.getDisplayName())) { + msgArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_FROM, moduleName, fromAddress.getDisplayName())); } - if (!StringUtils.isEmpty(toAddress)) { - msgArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_TO, moduleName, toAddress)); + if (toAddresses != null && !StringUtils.isEmpty(toAddresses)) { + msgArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_TO, moduleName, toAddresses)); } - + if (!StringUtils.isEmpty(subject)) { msgArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_SUBJECT, moduleName, subject)); } @@ -336,23 +412,40 @@ public final class AppDBParserHelper { if (!StringUtils.isEmpty(threadId)) { msgArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_THREAD_ID, moduleName, threadId)); } - - + + // Add other specified attributes for (BlackboardAttribute otherAttribute: otherAttributesList) { msgArtifact.addAttribute(otherAttribute); } - - // Find/Create an account instance for the sender/recipient - AccountFileInstance contactAccountInstance = createAccountInstance(accountsType, otherAccountUniqueID); - - // Create a relationship between selfAccount and contactAccount - if (selfAccountInstance != null) { - addRelationship (selfAccountInstance, contactAccountInstance, msgArtifact, Relationship.Type.MESSAGE, 0 ); + + // Find/create an account instance for sender + if (fromAddress != null) { + AccountFileInstance senderAccountInstance = createAccountInstance(accountsType, fromAddress.getUniqueID()); + + // Create a relationship between selfAccount and sender account + if (selfAccountInstance != null) { + addRelationship (selfAccountInstance, senderAccountInstance, msgArtifact, Relationship.Type.MESSAGE, dateTime ); + } } - + + // Find/create an account instance for each recipient + if (recipientsList != null) { + for(Account.Address recipient : recipientsList) { + + AccountFileInstance recipientAccountInstance = createAccountInstance(accountsType, recipient.getUniqueID()); + + // Create a relationship between selfAccount and recipient account + if (selfAccountInstance != null) { + addRelationship (selfAccountInstance, recipientAccountInstance, msgArtifact, Relationship.Type.MESSAGE, dateTime ); + } + } + } + // post artifact Case.getCurrentCase().getSleuthkitCase().getBlackboard().postArtifact(msgArtifact, this.moduleName); + + } catch (TskCoreException ex) { logger.log(Level.SEVERE, "Unable to add message artifact", ex); //NON-NLS return null; @@ -362,9 +455,9 @@ public final class AppDBParserHelper { } // return the artifact - return msgArtifact; + return msgArtifact; } - + /** * Adds a TSK_CALLLOG artifact. * @@ -463,11 +556,6 @@ public final class AppDBParserHelper { } - // TBD - public void addAttachment(BlackboardArtifact parentArtifact) { - // RAMAN TBD - } - /** * Adds a TSK_WEB_BOOKMARK artifact. * diff --git a/Core/src/org/sleuthkit/autopsy/coreutils/AppSQLiteDB.java b/Core/src/org/sleuthkit/autopsy/coreutils/AppSQLiteDB.java index fe9a86c0a8..f38a9d8b9e 100644 --- a/Core/src/org/sleuthkit/autopsy/coreutils/AppSQLiteDB.java +++ b/Core/src/org/sleuthkit/autopsy/coreutils/AppSQLiteDB.java @@ -70,25 +70,6 @@ public final class AppSQLiteDB implements Closeable { } - /** - * Looks for the given SQLIte database filename, with matching path substring. - * It makes a copy of each matching file, and creates an instance of - * AppSQLiteDB to help query the DB. - * - * A list of AppSQLiteDB instances is returned, one for each - * match found., - * . - * @param dataSource data source to search in - * @param dbName db file name to search - * @param parentPathSubstr path substring to match - * - * @return AbstractFile for the DB if the database file is found. - * Returns NULL if no such database is found. - */ - public static Collection findAppDatabases(DataSource dataSource, String dbName, String parentPathSubstr) { - return AppSQLiteDB.findAppDatabases(dataSource, dbName, false, parentPathSubstr); - } - /** * Looks for the given SQLIte database filename, with matching path substring. * It looks for exact name or a pattern match based on @@ -100,13 +81,13 @@ public final class AppSQLiteDB implements Closeable { * . * @param dataSource data source to search in * @param dbName db file name to search - * @param matchPattern whether to look for a pattern match or an exact match + * @param matchExactName whether to look for exact file name or a pattern match * @param parentPathSubstr path substring to match * * @return AbstractFile for the DB if the database file is found. * Returns NULL if no such database is found. */ - public static Collection findAppDatabases(DataSource dataSource, String dbName, boolean matchPattern, String parentPathSubstr) { + public static Collection findAppDatabases(DataSource dataSource, String dbName, boolean matchExactName, String parentPathSubstr) { List appDbs = new ArrayList<> (); Case openCase; @@ -126,10 +107,10 @@ public final class AppSQLiteDB implements Closeable { String parentPath = parentPathSubstr.replace("\\", "/"); parentPath = SleuthkitCase.escapeSingleQuotes(parentPath); String whereClause; - if (matchPattern) { - whereClause = String.format("LOWER(name) LIKE LOWER(\'%%%1$s%%\') AND LOWER(name) NOT LIKE LOWER(\'%%journal%%\') AND LOWER(parent_path) LIKE LOWER(\'%%%2$s%%\') AND data_source_obj_id = " + dataSource.getId(), dbName, parentPath); - } else { + if (matchExactName) { whereClause = String.format("LOWER(name) LIKE LOWER(\'%1$s\') AND LOWER(parent_path) LIKE LOWER(\'%%%2$s%%\') AND data_source_obj_id = " + dataSource.getId(), dbName, parentPath); + } else { + whereClause = String.format("LOWER(name) LIKE LOWER(\'%%%1$s%%\') AND LOWER(name) NOT LIKE LOWER(\'%%journal%%\') AND LOWER(parent_path) LIKE LOWER(\'%%%2$s%%\') AND data_source_obj_id = " + dataSource.getId(), dbName, parentPath); } absFiles = skCase.findAllFilesWhere(whereClause); for (AbstractFile absFile : absFiles) { diff --git a/InternalPythonModules/android/imo.py b/InternalPythonModules/android/imo.py index 433b818950..2407ff552d 100644 --- a/InternalPythonModules/android/imo.py +++ b/InternalPythonModules/android/imo.py @@ -55,7 +55,7 @@ class IMOAnalyzer(general.AndroidComponentAnalyzer): def analyze(self, dataSource, fileManager, context): selfAccountId = None - accountDbs = AppSQLiteDB.findAppDatabases(dataSource, "accountdb.db", "com.imo.android.imous") + accountDbs = AppSQLiteDB.findAppDatabases(dataSource, "accountdb.db", True, "com.imo.android.imous") for accountDb in accountDbs: try: accountResultSet = accountDb.runQuery("SELECT uid, name FROM account") @@ -71,7 +71,7 @@ class IMOAnalyzer(general.AndroidComponentAnalyzer): finally: accountDb.close() - friendsDbs = AppSQLiteDB.findAppDatabases(dataSource, "imofriends.db", "com.imo.android.imous") + friendsDbs = AppSQLiteDB.findAppDatabases(dataSource, "imofriends.db", True, "com.imo.android.imous") for friendsDb in friendsDbs: try: friendsDBHelper = AppDBParserHelper("IMO Parser", friendsDb.getDBFile(), @@ -79,40 +79,52 @@ class IMOAnalyzer(general.AndroidComponentAnalyzer): contactsResultSet = friendsDb.runQuery("SELECT buid, name FROM friends") if contactsResultSet is not None: while contactsResultSet.next(): - friendsDBHelper.addContact( contactsResultSet.getString("name"), ## name for account + friendsDBHelper.addContact( contactsResultSet.getString("buid"), ## unique id for account contactsResultSet.getString("name"), ## contact name "", ## phone "", ## home phone "", ## mobile "") ## email - queryString = "SELECT imdata, last_message, timestamp, message_type, message_read, name FROM messages "\ + queryString = "SELECT messages.buid AS buid, imdata, last_message, timestamp, message_type, message_read, name FROM messages "\ "INNER JOIN friends ON friends.buid = messages.buid" messagesResultSet = friendsDb.runQuery(queryString) if messagesResultSet is not None: while messagesResultSet.next(): direction = "" fromAddress = None - toAdddress = None + toAddress = None + name = messagesResultSet.getString("name") + uniqueId = messagesResultSet.getString("buid") + if (messagesResultSet.getInt("message_type") == 1): direction = "Incoming" - fromAddress = messagesResultSet.getString("name") + fromAddress = Account.Address(uniqueId, name) else: direction = "Outgoing" - toAddress = messagesResultSet.getString("name") + toAddress = Account.Address(uniqueId, name) + + + message_read = messagesResultSet.getInt("message_read") + if (message_read == 1): + msgReadStatus = AppDBParserHelper.MessageReadStatusEnum.READ + elif (message_read == 0): + msgReadStatus = AppDBParserHelper.MessageReadStatusEnum.UNREAD + else: + msgReadStatus = AppDBParserHelper.MessageReadStatusEnum.UNKNOWN timeStamp = messagesResultSet.getLong("timestamp") / 1000000000 + + messageArtifact = friendsDBHelper.addMessage( - contactsResultSet.getString("name"), "IMO Message", direction, fromAddress, toAddress, timeStamp, - messagesResultSet.getInt("message_read"), - "", + msgReadStatus, + "", # subject messagesResultSet.getString("last_message"), - "", - "" ) + "") # thread id # TBD: parse the imdata JSON structure to figure out if there is an attachment. # If one exists, add the attachment as a derived file and a child of the message artifact. From 892a33e942ff5d7f4d9d974f1bc4274336eef96a Mon Sep 17 00:00:00 2001 From: Joe Ho Date: Fri, 6 Sep 2019 16:25:12 -0400 Subject: [PATCH 056/102] Update Bundle.properties-MERGED --- .../autopsy/logicalimager/dsp/Bundle.properties-MERGED | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/Bundle.properties-MERGED index fd08147b08..b5c2e7963e 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/Bundle.properties-MERGED @@ -2,7 +2,13 @@ # To change this template file, choose Tools | Templates # and open the template in the editor. +# {0} - file number +# {1} - total files +AddLogicalImageTask.addingExtractedFile=Adding extracted file {0} of {1} AddLogicalImageTask.addingExtractedFiles=Adding extracted files +# {0} - file number +# {1} - total files +AddLogicalImageTask.addingInterestingFile=Adding interesting file {0} of {1} AddLogicalImageTask.addingInterestingFiles=Adding search results as interesting files # {0} - file AddLogicalImageTask.addingToReport=Adding {0} to report @@ -21,6 +27,7 @@ AddLogicalImageTask.doneAddingInterestingFiles=Done adding search results as int # {0} - file AddLogicalImageTask.doneAddingToReport=Done adding {0} to report AddLogicalImageTask.doneCopying=Done copying +AddLogicalImageTask.errorAddingExtractedFiles=Error adding extracted files # {0} - reason AddLogicalImageTask.failedToAddInterestingFiles=Failed to add interesting files: {0} # {0} - file @@ -29,6 +36,8 @@ AddLogicalImageTask.failedToAddReport=Failed to add report {0}. Reason= {1} # {0} - src # {1} - dest AddLogicalImageTask.failedToCopyDirectory=Failed to copy directory {0} to {1} +# {0} - reason +AddLogicalImageTask.failedToGetTotalFilesCount=Failed to get total files count: {0} # {0} - file AddLogicalImageTask.failToGetCanonicalPath=Fail to get canonical path for {0} AddLogicalImageTask.ingestionCancelled=Ingestion cancelled From e9fe824865ce5cbb4bf28f6520c4efae92955b7f Mon Sep 17 00:00:00 2001 From: Joe Ho Date: Fri, 6 Sep 2019 16:27:39 -0400 Subject: [PATCH 057/102] Update AddLogicalImageTask.java --- .../logicalimager/dsp/AddLogicalImageTask.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java index 7b3fe30dd8..3112385450 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java @@ -411,14 +411,14 @@ final class AddLogicalImageTask implements Runnable { //addLocalFile here fileImporter.addLocalFile( - Paths.get(src.toString(), extractedFilePath).toFile(), - filename, - parentPath, - Long.parseLong(ctime), - Long.parseLong(crtime), - Long.parseLong(atime), - Long.parseLong(mtime), - localFilesDataSource); + Paths.get(src.toString(), extractedFilePath).toFile(), + filename, + parentPath, + Long.parseLong(ctime), + Long.parseLong(crtime), + Long.parseLong(atime), + Long.parseLong(mtime), + localFilesDataSource); lineNumber++; } // end reading file From d384937471bf32e4ec69be2d6e3c47a69351ce84 Mon Sep 17 00:00:00 2001 From: Joe Ho Date: Fri, 6 Sep 2019 17:07:15 -0400 Subject: [PATCH 058/102] Fix codacy error --- .../dsp/AddLogicalImageTask.java | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java index 3112385450..e018a352cc 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java @@ -70,7 +70,6 @@ final class AddLogicalImageTask implements Runnable { private final DataSourceProcessorProgressMonitor progressMonitor; private final Blackboard blackboard; private final Case currentCase; - private Map> imagePaths; private Map imagePathToObjIdMap; private long totalFiles; @@ -176,10 +175,6 @@ final class AddLogicalImageTask implements Runnable { } } - AddMultipleImageTask addMultipleImageTask = null; - List newDataSources = new ArrayList<>(); - boolean createVHD; - Path resultsPath = Paths.get(dest.toString(), resultsFilename); try { totalFiles = Files.lines(resultsPath).count() - 1; // skip the header line @@ -189,6 +184,10 @@ final class AddLogicalImageTask implements Runnable { return; } + AddMultipleImageTask addMultipleImageTask = null; + List newDataSources = new ArrayList<>(); + boolean createVHD; + if (imagePaths.isEmpty()) { createVHD = false; // No VHD in src directory, try ingest the root directory using Logical File Set @@ -282,15 +281,15 @@ final class AddLogicalImageTask implements Runnable { } private Map imagePathsToDataSourceObjId(Map> imagePaths) { - Map imagePathToObjIdMap = new HashMap<>(); + Map imagePathToObjId = new HashMap<>(); for (Map.Entry> entry : imagePaths.entrySet()) { Long key = entry.getKey(); List names = entry.getValue(); for (String name : names) { - imagePathToObjIdMap.put(name, key); + imagePathToObjId.put(name, key); } } - return imagePathToObjIdMap; + return imagePathToObjId; } @Messages({ @@ -299,8 +298,8 @@ final class AddLogicalImageTask implements Runnable { "# {0} - file number", "# {1} - total files", "AddLogicalImageTask.addingInterestingFile=Adding interesting file {0} of {1}" }) private void addInterestingFiles(Path resultsPath, boolean createVHD) throws IOException, TskCoreException { - imagePaths = currentCase.getSleuthkitCase().getImagePaths(); - imagePathToObjIdMap = imagePathsToDataSourceObjId(imagePaths); + Map> objIdToimagePathsMap = currentCase.getSleuthkitCase().getImagePaths(); + imagePathToObjIdMap = imagePathsToDataSourceObjId(objIdToimagePathsMap); try (BufferedReader br = new BufferedReader(new InputStreamReader( new FileInputStream(resultsPath.toFile()), "UTF8"))) { // NON-NLS From 143007e565199f7b7d93601be61f58f04d617303 Mon Sep 17 00:00:00 2001 From: Raman Date: Mon, 9 Sep 2019 11:45:59 -0400 Subject: [PATCH 059/102] Added attachDatabase/detachDatabase apis. --- .../autopsy/coreutils/AppSQLiteDB.java | 99 ++++++++++++++++++- 1 file changed, 97 insertions(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/coreutils/AppSQLiteDB.java b/Core/src/org/sleuthkit/autopsy/coreutils/AppSQLiteDB.java index f38a9d8b9e..f514072433 100644 --- a/Core/src/org/sleuthkit/autopsy/coreutils/AppSQLiteDB.java +++ b/Core/src/org/sleuthkit/autopsy/coreutils/AppSQLiteDB.java @@ -108,9 +108,9 @@ public final class AppSQLiteDB implements Closeable { parentPath = SleuthkitCase.escapeSingleQuotes(parentPath); String whereClause; if (matchExactName) { - whereClause = String.format("LOWER(name) LIKE LOWER(\'%1$s\') AND LOWER(parent_path) LIKE LOWER(\'%%%2$s%%\') AND data_source_obj_id = " + dataSource.getId(), dbName, parentPath); + whereClause = String.format("LOWER(name) = LOWER('%s') AND LOWER(parent_path) LIKE LOWER('%%%s%%') AND data_source_obj_id = %s", dbName, parentPath, dataSource.getId()); } else { - whereClause = String.format("LOWER(name) LIKE LOWER(\'%%%1$s%%\') AND LOWER(name) NOT LIKE LOWER(\'%%journal%%\') AND LOWER(parent_path) LIKE LOWER(\'%%%2$s%%\') AND data_source_obj_id = " + dataSource.getId(), dbName, parentPath); + whereClause = String.format("LOWER(name) LIKE LOWER('%%%s%%') AND LOWER(name) NOT LIKE LOWER('%%journal%%') AND LOWER(parent_path) LIKE LOWER('%%%s%%') AND data_source_obj_id = %s", dbName, parentPath, dataSource.getId() ); } absFiles = skCase.findAllFilesWhere(whereClause); for (AbstractFile absFile : absFiles) { @@ -142,6 +142,101 @@ public final class AppSQLiteDB implements Closeable { return this.dbAbstractFile; } + /** + * Attaches a database to the current connection. + * + * Finds the specified database file in the specified folder. + * If found, makes copy of the database in the case folder and + * run ATTACH DATABASE sql. + * + * @param dataSource data source in which to look file the db file + * @param dbName name of db file to look for + * @param matchExactName specified whether the name is an exact name or a pattern + * @param dbPath path in which to look for the db file + * @param dbAlias alias name to attach the database as + * + * @return abstract file for the matching db file. + * + * @throws TskCoreException in case of an error. + */ + public AbstractFile attachDatabase(DataSource dataSource, String dbName, + boolean matchExactName, String dbPath, String dbAlias) throws TskCoreException { + + Case openCase; + try { + openCase = Case.getCurrentCaseThrows(); + } catch (NoCurrentCaseException ex) { + throw new TskCoreException("Exception while getting open case.", ex); + } + + List absFiles; + long fileId = 0; + String localFilePath = ""; + try { + SleuthkitCase skCase = openCase.getSleuthkitCase(); + String parentPath = dbPath.replace("\\", "/"); + parentPath = SleuthkitCase.escapeSingleQuotes(parentPath); + String whereClause; + if (matchExactName) { + whereClause = String.format("LOWER(name) = LOWER('%s') AND LOWER(parent_path) = LOWER('%s') AND data_source_obj_id = %s", dbName, parentPath, dataSource.getId()); //NON-NLS + } else { + whereClause = String.format("LOWER(name) LIKE LOWER('%%%s%%') AND LOWER(name) NOT LIKE LOWER('%%journal%%') AND LOWER(parent_path) = LOWER('%s') AND data_source_obj_id = %s", dbName, parentPath, dataSource.getId()); //NON-NLS + } + absFiles = skCase.findAllFilesWhere(whereClause); + for (AbstractFile absFile : absFiles) { + try { + localFilePath = openCase.getTempDirectory() + + File.separator + absFile.getId() + absFile.getName(); + File jFile = new java.io.File(localFilePath); + fileId = absFile.getId(); + ContentUtils.writeToFile(absFile, jFile); + + //Find and copy both WAL and SHM meta files + findAndCopySQLiteMetaFile(absFile, absFile.getName() + "-wal"); + findAndCopySQLiteMetaFile(absFile, absFile.getName() + "-shm"); + + //run the ATTACH DATABASE sql command + try { + String attachDbSql = String.format("ATTACH DATABASE '%s' AS '%s'", localFilePath, dbAlias); //NON-NLS + statement.executeUpdate(attachDbSql); + } + catch (SQLException ex) { + throw new TskCoreException("Error running ATTACH DATABASE SQL. " + ex.getMessage(), ex); + } + + return absFile; + } catch (ReadContentInputStream.ReadContentInputStreamException ex) { + Logger.getLogger(AppSQLiteDB.class.getName()).log(Level.WARNING, String.format("Error reading content from file '%s' (id=%d).", absFile.getName(), fileId), ex); //NON-NLS + } catch (IOException | NoCurrentCaseException ex) { + Logger.getLogger(AppSQLiteDB.class.getName()).log(Level.SEVERE, String.format("Error writing content from file '%s' (id=%d) to '%s'.", absFile.getName(), fileId, localFilePath), ex); //NON-NLS + } + } + } catch (TskCoreException e) { + Logger.getLogger(AppSQLiteDB.class.getName()).log(Level.SEVERE, "Error finding application DB file.", e); //NON-NLS + } + + return null; + } + + /** + * Detaches the specified database from the connection + * + * @param dbAlias alias for database to detach + * + * @throws TskCoreException + */ + public void detachDatabase(String dbAlias) throws TskCoreException { + + try { + String detachDbSql = String.format("DETACH DATABASE '%s'", dbAlias); + statement.executeUpdate(detachDbSql); //NON-NLS + } + catch (SQLException ex) { + throw new TskCoreException("Error running DETACH DATABASE SQL. " + ex.getMessage(), ex); + } + } + + /** * Checks if the specified table exists in the given database file. * From 042975b980e6c085b8da1d3ac5b2823e357b914a Mon Sep 17 00:00:00 2001 From: Eugene Livis Date: Mon, 9 Sep 2019 13:27:17 -0400 Subject: [PATCH 060/102] Added a utility method to utf-8 sanitize file names --- .../org/sleuthkit/autopsy/coreutils/FileUtil.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/Core/src/org/sleuthkit/autopsy/coreutils/FileUtil.java b/Core/src/org/sleuthkit/autopsy/coreutils/FileUtil.java index 5b432124ba..4a7e6da65f 100644 --- a/Core/src/org/sleuthkit/autopsy/coreutils/FileUtil.java +++ b/Core/src/org/sleuthkit/autopsy/coreutils/FileUtil.java @@ -20,6 +20,7 @@ package org.sleuthkit.autopsy.coreutils; import java.io.File; import java.io.IOException; +import java.nio.charset.Charset; import java.util.logging.Level; import org.openide.filesystems.FileObject; import java.nio.file.Files; @@ -171,6 +172,18 @@ public class FileUtil { //with underscores. We are only keeping \ as it could be part of the path. return fileName.replaceAll("[\\p{Cntrl}/:\"*?<>|]+", "_"); } + + /** + * UTF-8 sanitize and escape special characters in a file name or a file name component + * + * @param fileName to escape + * + * @return Sanitized string + */ + public static String utf8SanitizeFileName(String fileName) { + Charset charset = Charset.forName("UTF-8"); + return charset.decode(charset.encode(escapeFileName(fileName))).toString(); + } /** * Test if the current user has read and write access to the dirPath. From 9344eb4c98e1b07d548959d292acf6909783e29f Mon Sep 17 00:00:00 2001 From: Raman Date: Tue, 10 Sep 2019 08:13:48 -0400 Subject: [PATCH 061/102] Address review comments. --- .../autopsy/coreutils/AppSQLiteDB.java | 280 +++++++++--------- InternalPythonModules/android/imo.py | 6 +- 2 files changed, 136 insertions(+), 150 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/coreutils/AppSQLiteDB.java b/Core/src/org/sleuthkit/autopsy/coreutils/AppSQLiteDB.java index f514072433..404936e087 100644 --- a/Core/src/org/sleuthkit/autopsy/coreutils/AppSQLiteDB.java +++ b/Core/src/org/sleuthkit/autopsy/coreutils/AppSQLiteDB.java @@ -31,6 +31,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.logging.Level; +import org.openide.util.Exceptions; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; import org.sleuthkit.autopsy.casemodule.services.FileManager; @@ -52,27 +53,45 @@ public final class AppSQLiteDB implements Closeable { private final AbstractFile dbAbstractFile; // AbstractFile for the DB file - private Connection connection = null; - private Statement statement = null; + private final Connection connection; + private final Statement statement; - private AppSQLiteDB(AbstractFile dbAbstractFile, File dbFileCopy) { - this.dbAbstractFile = dbAbstractFile; + + /** + * Class to abstract the abstract file for a DB file and its on disk copy + * + */ + private static final class AppSQLiteDBFileBundle { + private final AbstractFile dbAbstractFile; + private final File dbFileCopy; - try { - Class.forName("org.sqlite.JDBC"); //NON-NLS //load JDBC driver - connection = DriverManager.getConnection("jdbc:sqlite:" + dbFileCopy.getPath()); //NON-NLS - statement = connection.createStatement(); - } catch (ClassNotFoundException | SQLException e) { - logger.log(Level.SEVERE, "Error opening database " + dbFileCopy.getPath(), e); //NON-NLS - connection = null; - statement = null; + AppSQLiteDBFileBundle(AbstractFile dbAbstractFile, File dbFileCopy) { + this.dbAbstractFile = dbAbstractFile; + this.dbFileCopy = dbFileCopy; } + + AbstractFile getAbstractFile() { + return dbAbstractFile; + } + + File getFileCopy() { + return dbFileCopy; + } + + } + + private AppSQLiteDB(AppSQLiteDBFileBundle appSQLiteDBFileBundle) throws ClassNotFoundException, SQLException { + this.dbAbstractFile = appSQLiteDBFileBundle.getAbstractFile(); + + Class.forName("org.sqlite.JDBC"); //NON-NLS //load JDBC driver + connection = DriverManager.getConnection("jdbc:sqlite:" + appSQLiteDBFileBundle.getFileCopy().getPath()); //NON-NLS + statement = connection.createStatement(); } /** * Looks for the given SQLIte database filename, with matching path substring. - * It looks for exact name or a pattern match based on + * It looks for exact name or a pattern match based on a input parameter. * It makes a copy of each matching file, and creates an instance of * AppSQLiteDB to help query the DB. * @@ -87,54 +106,24 @@ public final class AppSQLiteDB implements Closeable { * @return AbstractFile for the DB if the database file is found. * Returns NULL if no such database is found. */ - public static Collection findAppDatabases(DataSource dataSource, String dbName, boolean matchExactName, String parentPathSubstr) { + public static Collection findAppDatabases(DataSource dataSource, + String dbName, boolean matchExactName, String parentPathSubstr) { List appDbs = new ArrayList<> (); - Case openCase; - try { - openCase = Case.getCurrentCaseThrows(); - } catch (NoCurrentCaseException ex) { - Logger.getLogger(AppSQLiteDB.class.getName()).log(Level.SEVERE, "Exception while getting open case.", ex); //NON-NLS - return appDbs; - } - - List absFiles; - long fileId = 0; - String localDiskPath = ""; - try { - SleuthkitCase skCase = openCase.getSleuthkitCase(); - String parentPath = parentPathSubstr.replace("\\", "/"); - parentPath = SleuthkitCase.escapeSingleQuotes(parentPath); - String whereClause; - if (matchExactName) { - whereClause = String.format("LOWER(name) = LOWER('%s') AND LOWER(parent_path) LIKE LOWER('%%%s%%') AND data_source_obj_id = %s", dbName, parentPath, dataSource.getId()); - } else { - whereClause = String.format("LOWER(name) LIKE LOWER('%%%s%%') AND LOWER(name) NOT LIKE LOWER('%%journal%%') AND LOWER(parent_path) LIKE LOWER('%%%s%%') AND data_source_obj_id = %s", dbName, parentPath, dataSource.getId() ); - } - absFiles = skCase.findAllFilesWhere(whereClause); - for (AbstractFile absFile : absFiles) { + Collection dbFileBundles = findAndCopySQLiteDB( dataSource, dbName, matchExactName, parentPathSubstr, false); + dbFileBundles.forEach((dbFileBundle) -> { try { - localDiskPath = openCase.getTempDirectory() - + File.separator + absFile.getId() + absFile.getName(); - File jFile = new java.io.File(localDiskPath); - fileId = absFile.getId(); - ContentUtils.writeToFile(absFile, jFile); - - //Find and copy both WAL and SHM meta files - findAndCopySQLiteMetaFile(absFile, absFile.getName() + "-wal"); - findAndCopySQLiteMetaFile(absFile, absFile.getName() + "-shm"); - - appDbs.add(new AppSQLiteDB(absFile, jFile) ); - } catch (ReadContentInputStream.ReadContentInputStreamException ex) { - Logger.getLogger(AppSQLiteDB.class.getName()).log(Level.WARNING, String.format("Error reading content from file '%s' (id=%d).", absFile.getName(), fileId), ex); //NON-NLS - } catch (IOException | NoCurrentCaseException | TskCoreException ex) { - Logger.getLogger(AppSQLiteDB.class.getName()).log(Level.SEVERE, String.format("Error writing content from file '%s' (id=%d) to '%s'.", absFile.getName(), fileId, localDiskPath), ex); //NON-NLS + AppSQLiteDB appSQLiteDB = new AppSQLiteDB(dbFileBundle); + appDbs.add(appSQLiteDB); + } catch (ClassNotFoundException | SQLException ex) { + Logger.getLogger(AppSQLiteDB.class.getName()).log(Level.SEVERE, String.format("Failed to open a DB connection for file = '%s' and path = '%s'.", dbFileBundle.dbAbstractFile.getName(), dbFileBundle.getFileCopy().getPath()), ex); //NON-NLS } - } - } catch (TskCoreException e) { - Logger.getLogger(AppSQLiteDB.class.getName()).log(Level.SEVERE, "Error finding application DB file.", e); //NON-NLS + }); + } catch (TskCoreException ex) { + Logger.getLogger(AppSQLiteDB.class.getName()).log(Level.SEVERE, String.format("Error finding App database files with name = '%s' and path = '%s'.", dbName, parentPathSubstr), ex); //NON-NLS } + return appDbs; } @@ -156,66 +145,98 @@ public final class AppSQLiteDB implements Closeable { * @param dbAlias alias name to attach the database as * * @return abstract file for the matching db file. - * - * @throws TskCoreException in case of an error. + * null if no match is found. + * + * @throws SQLException in case of an SQL error */ public AbstractFile attachDatabase(DataSource dataSource, String dbName, - boolean matchExactName, String dbPath, String dbAlias) throws TskCoreException { + boolean matchExactName, String dbPath, String dbAlias) throws SQLException { + try { + Collection dbFileBundles = findAndCopySQLiteDB(dataSource, dbName, matchExactName, dbPath, true); + for (AppSQLiteDBFileBundle dbFileBundle: dbFileBundles) { + String attachDbSql = String.format("ATTACH DATABASE '%s' AS '%s'", dbFileBundle.getFileCopy().getPath(), dbAlias); //NON-NLS + statement.executeUpdate(attachDbSql); + + return dbFileBundle.getAbstractFile(); + } + } catch (TskCoreException ex) { + Logger.getLogger(AppSQLiteDB.class.getName()).log(Level.SEVERE, String.format("Error attaching to App database files with name = '%s' and path = '%s'.", dbName, dbPath), ex); //NON-NLS + } + + return null; + } + + /** + * Finds database file with the specified name, makes a copy of the file in the case directory, + * and returns the AbstractFile as well as the file copy. + * + * @param dataSource data source to search in + * @param dbName db file name to search + * @param matchExactName whether to look for exact file name or a pattern match + * @param dbPath path to match + * @param matchExactName whether to look for exact path name or a substring match + * + * @return a collection of AppSQLiteDBFileBundle + * + * @throws TskCoreException + */ + private static Collection findAndCopySQLiteDB(DataSource dataSource, String dbName, + boolean matchExactName, String dbPath, boolean matchExactPath) throws TskCoreException { + + List dbFileBundles = new ArrayList<> (); + Case openCase; - Case openCase; try { openCase = Case.getCurrentCaseThrows(); } catch (NoCurrentCaseException ex) { - throw new TskCoreException("Exception while getting open case.", ex); + throw new TskCoreException("Failed to get current case.", ex); } List absFiles; long fileId = 0; - String localFilePath = ""; - try { - SleuthkitCase skCase = openCase.getSleuthkitCase(); - String parentPath = dbPath.replace("\\", "/"); - parentPath = SleuthkitCase.escapeSingleQuotes(parentPath); - String whereClause; - if (matchExactName) { - whereClause = String.format("LOWER(name) = LOWER('%s') AND LOWER(parent_path) = LOWER('%s') AND data_source_obj_id = %s", dbName, parentPath, dataSource.getId()); //NON-NLS - } else { - whereClause = String.format("LOWER(name) LIKE LOWER('%%%s%%') AND LOWER(name) NOT LIKE LOWER('%%journal%%') AND LOWER(parent_path) = LOWER('%s') AND data_source_obj_id = %s", dbName, parentPath, dataSource.getId()); //NON-NLS - } - absFiles = skCase.findAllFilesWhere(whereClause); - for (AbstractFile absFile : absFiles) { - try { - localFilePath = openCase.getTempDirectory() - + File.separator + absFile.getId() + absFile.getName(); - File jFile = new java.io.File(localFilePath); - fileId = absFile.getId(); - ContentUtils.writeToFile(absFile, jFile); - - //Find and copy both WAL and SHM meta files - findAndCopySQLiteMetaFile(absFile, absFile.getName() + "-wal"); - findAndCopySQLiteMetaFile(absFile, absFile.getName() + "-shm"); - - //run the ATTACH DATABASE sql command - try { - String attachDbSql = String.format("ATTACH DATABASE '%s' AS '%s'", localFilePath, dbAlias); //NON-NLS - statement.executeUpdate(attachDbSql); - } - catch (SQLException ex) { - throw new TskCoreException("Error running ATTACH DATABASE SQL. " + ex.getMessage(), ex); - } + String localDiskPath = ""; - return absFile; - } catch (ReadContentInputStream.ReadContentInputStreamException ex) { - Logger.getLogger(AppSQLiteDB.class.getName()).log(Level.WARNING, String.format("Error reading content from file '%s' (id=%d).", absFile.getName(), fileId), ex); //NON-NLS - } catch (IOException | NoCurrentCaseException ex) { - Logger.getLogger(AppSQLiteDB.class.getName()).log(Level.SEVERE, String.format("Error writing content from file '%s' (id=%d) to '%s'.", absFile.getName(), fileId, localFilePath), ex); //NON-NLS - } - } - } catch (TskCoreException e) { - Logger.getLogger(AppSQLiteDB.class.getName()).log(Level.SEVERE, "Error finding application DB file.", e); //NON-NLS + SleuthkitCase skCase = openCase.getSleuthkitCase(); + String parentPath = dbPath.replace("\\", "/"); + parentPath = SleuthkitCase.escapeSingleQuotes(parentPath); + + String whereClause; + if (matchExactName) { + whereClause = String.format("LOWER(name) = LOWER('%s')", dbName); + } else { + whereClause = String.format("LOWER(name) LIKE LOWER('%%%s%%') AND LOWER(name) NOT LIKE LOWER('%%journal%%')", dbName ); + } + if (matchExactPath) { + whereClause += String.format(" AND LOWER(parent_path) = LOWER('%s')", parentPath ); + } else { + whereClause += String.format(" AND LOWER(parent_path) LIKE LOWER('%%%s%%')", parentPath ); + } + whereClause += String.format(" AND data_source_obj_id = %s", dataSource.getId()); + + absFiles = skCase.findAllFilesWhere(whereClause); + for (AbstractFile absFile : absFiles) { + try { + localDiskPath = openCase.getTempDirectory() + + File.separator + absFile.getId() + absFile.getName(); + File jFile = new java.io.File(localDiskPath); + fileId = absFile.getId(); + ContentUtils.writeToFile(absFile, jFile); + + //Find and copy both WAL and SHM meta files + findAndCopySQLiteMetaFile(absFile, absFile.getName() + "-wal"); + findAndCopySQLiteMetaFile(absFile, absFile.getName() + "-shm"); + + AppSQLiteDBFileBundle dbFileBundle = new AppSQLiteDBFileBundle(absFile, jFile); + dbFileBundles.add(dbFileBundle); + + } catch (ReadContentInputStream.ReadContentInputStreamException ex) { + Logger.getLogger(AppSQLiteDB.class.getName()).log(Level.WARNING, String.format("Error reading content from file '%s' (id=%d).", absFile.getName(), fileId), ex); //NON-NLS + } catch (IOException | NoCurrentCaseException | TskCoreException ex) { + Logger.getLogger(AppSQLiteDB.class.getName()).log(Level.SEVERE, String.format("Error creating AppSQLiteDB for file '%s' (id=%d) to copied to '%s'.", absFile.getName(), fileId, localDiskPath), ex); //NON-NLS + } } - return null; + return dbFileBundles; } /** @@ -223,48 +244,14 @@ public final class AppSQLiteDB implements Closeable { * * @param dbAlias alias for database to detach * - * @throws TskCoreException + * @throws SQLException */ - public void detachDatabase(String dbAlias) throws TskCoreException { - - try { - String detachDbSql = String.format("DETACH DATABASE '%s'", dbAlias); - statement.executeUpdate(detachDbSql); //NON-NLS - } - catch (SQLException ex) { - throw new TskCoreException("Error running DETACH DATABASE SQL. " + ex.getMessage(), ex); - } + public void detachDatabase(String dbAlias) throws SQLException { + String detachDbSql = String.format("DETACH DATABASE '%s'", dbAlias); + statement.executeUpdate(detachDbSql); //NON-NLS } - /** - * Checks if the specified table exists in the given database file. - * - * @param tableName table name to check - * - * @return - */ - public boolean tableExists(String tableName) { - // RAMAN TBD - return false; - - } - - /** - * Checks if the specified column exists. - * - * @param tableName table name to check - * @param columnName column name to check - * @return - */ - public boolean columnExists(String tableName, String columnName) { - // RAMAN TBD - return false; - } - - - - /** * Runs the given query on the database and returns result set. @@ -272,18 +259,15 @@ public final class AppSQLiteDB implements Closeable { * * @return ResultSet from running the query. * - * @throws TskCoreException in case of an error. + * @throws SQLException in case of an error. * */ - public ResultSet runQuery(String queryStr) throws TskCoreException { + public ResultSet runQuery(String queryStr) throws SQLException { ResultSet resultSet = null; - try { + + if (null != queryStr) { resultSet = statement.executeQuery(queryStr); //NON-NLS - } - catch (SQLException ex) { - throw new TskCoreException("Error running app SQLite query. " + ex.getMessage(), ex); - } - + } return resultSet; } diff --git a/InternalPythonModules/android/imo.py b/InternalPythonModules/android/imo.py index 2407ff552d..d887a6da53 100644 --- a/InternalPythonModules/android/imo.py +++ b/InternalPythonModules/android/imo.py @@ -129,9 +129,11 @@ class IMOAnalyzer(general.AndroidComponentAnalyzer): # TBD: parse the imdata JSON structure to figure out if there is an attachment. # If one exists, add the attachment as a derived file and a child of the message artifact. - + except SQLException as ex: - self._logger.log(Level.SEVERE, "Error processing query result for IMO friends", ex) + self._logger.log(Level.SEVERE, "Error processing query result for IMO friends", ex) + except TskCoreException as ex: + self._logger.log(Level.SEVERE, "Failed to create AppDBParserHelper for adding artifacts.", ex) finally: friendsDb.close() From 9b3fc008471ad0c03c9d4aee390e11baaf1dd9ec Mon Sep 17 00:00:00 2001 From: Raman Date: Tue, 10 Sep 2019 10:13:34 -0400 Subject: [PATCH 062/102] Updated javadocs. --- .../src/org/sleuthkit/autopsy/coreutils/AppSQLiteDB.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/coreutils/AppSQLiteDB.java b/Core/src/org/sleuthkit/autopsy/coreutils/AppSQLiteDB.java index 404936e087..900e8aa45e 100644 --- a/Core/src/org/sleuthkit/autopsy/coreutils/AppSQLiteDB.java +++ b/Core/src/org/sleuthkit/autopsy/coreutils/AppSQLiteDB.java @@ -31,7 +31,6 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.logging.Level; -import org.openide.util.Exceptions; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; import org.sleuthkit.autopsy.casemodule.services.FileManager; @@ -45,8 +44,8 @@ import org.sleuthkit.datamodel.TskCoreException; /** * An abstraction around an SQLite app DB found in a data source. - * This class makes a copy of it, opens a SQLite connection to it - * and runs queries on it. + * This class makes a copy of it, along with any meta files (WAL, SHM), + * opens a SQLite connection to it, and runs queries on it. */ public final class AppSQLiteDB implements Closeable { private final Logger logger = Logger.getLogger(AppSQLiteDB.class.getName()); @@ -103,8 +102,8 @@ public final class AppSQLiteDB implements Closeable { * @param matchExactName whether to look for exact file name or a pattern match * @param parentPathSubstr path substring to match * - * @return AbstractFile for the DB if the database file is found. - * Returns NULL if no such database is found. + * @return A list of abstract files matching the specified name and path. + * Returns an empty list if no matching database is found. */ public static Collection findAppDatabases(DataSource dataSource, String dbName, boolean matchExactName, String parentPathSubstr) { From ce841f2c3b677f9250c88eba25f9081a4201b3b2 Mon Sep 17 00:00:00 2001 From: Joe Ho Date: Tue, 10 Sep 2019 10:34:12 -0400 Subject: [PATCH 063/102] Fix PR comments --- .../autopsy/logicalimager/dsp/AddLogicalImageTask.java | 7 +++---- .../autopsy/logicalimager/dsp/Bundle.properties-MERGED | 5 ++--- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java index e018a352cc..7cce2416f9 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java @@ -295,7 +295,7 @@ final class AddLogicalImageTask implements Runnable { @Messages({ "# {0} - line number", "# {1} - fields length", "# {2} - expected length", "AddLogicalImageTask.notEnoughFields=File does not contain enough fields at line {0}, got {1}, expecting {2}", "# {0} - target image path", "AddLogicalImageTask.cannotFindDataSourceObjId=Cannot find obj_id in tsk_image_names for {0}", - "# {0} - file number", "# {1} - total files", "AddLogicalImageTask.addingInterestingFile=Adding interesting file {0} of {1}" + "# {0} - file number", "# {1} - total files", "AddLogicalImageTask.addingInterestingFile=Adding interesting file ({0}/{1})" }) private void addInterestingFiles(Path resultsPath, boolean createVHD) throws IOException, TskCoreException { Map> objIdToimagePathsMap = currentCase.getSleuthkitCase().getImagePaths(); @@ -361,8 +361,7 @@ final class AddLogicalImageTask implements Runnable { } @Messages({ - "# {0} - file number", "# {1} - total files", "AddLogicalImageTask.addingExtractedFile=Adding extracted file {0} of {1}", - "AddLogicalImageTask.errorAddingExtractedFiles=Error adding extracted files" + "# {0} - file number", "# {1} - total files", "AddLogicalImageTask.addingExtractedFile=Adding extracted file ({0}/{1})" }) private void addExtractedFiles(File src, Path resultsPath, List newDataSources) throws TskCoreException, IOException { SleuthkitCase skCase = Case.getCurrentCase().getSleuthkitCase(); @@ -428,7 +427,7 @@ final class AddLogicalImageTask implements Runnable { } catch (NumberFormatException | TskCoreException ex) { LOGGER.log(Level.SEVERE, "Error adding extracted files", ex); // NON-NLS rollbackTransaction(trans); - throw new TskCoreException(Bundle.AddLogicalImageTask_errorAddingExtractedFiles(), ex); + throw new TskCoreException("Error adding extracted files", ex); } } diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/Bundle.properties-MERGED index b5c2e7963e..6636cdda9f 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/Bundle.properties-MERGED @@ -4,11 +4,11 @@ # {0} - file number # {1} - total files -AddLogicalImageTask.addingExtractedFile=Adding extracted file {0} of {1} +AddLogicalImageTask.addingExtractedFile=Adding extracted file ({0}/{1}) AddLogicalImageTask.addingExtractedFiles=Adding extracted files # {0} - file number # {1} - total files -AddLogicalImageTask.addingInterestingFile=Adding interesting file {0} of {1} +AddLogicalImageTask.addingInterestingFile=Adding interesting file ({0}/{1}) AddLogicalImageTask.addingInterestingFiles=Adding search results as interesting files # {0} - file AddLogicalImageTask.addingToReport=Adding {0} to report @@ -27,7 +27,6 @@ AddLogicalImageTask.doneAddingInterestingFiles=Done adding search results as int # {0} - file AddLogicalImageTask.doneAddingToReport=Done adding {0} to report AddLogicalImageTask.doneCopying=Done copying -AddLogicalImageTask.errorAddingExtractedFiles=Error adding extracted files # {0} - reason AddLogicalImageTask.failedToAddInterestingFiles=Failed to add interesting files: {0} # {0} - file From a8ad6eaa3d95078a92f8efec085fbeb4f4ead3d6 Mon Sep 17 00:00:00 2001 From: Ethan Roseman Date: Tue, 10 Sep 2019 12:12:51 -0400 Subject: [PATCH 064/102] Adding idea project files to .gitignore --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index fd160c9744..45c3b68db4 100644 --- a/.gitignore +++ b/.gitignore @@ -82,7 +82,8 @@ hs_err_pid*.log /RecentActivity/release/ /CentralRepository/release/ -/.idea/ +.idea/ +*.iml *.img *.vhd From 7a71e361e30c42deb5bca45262de3f8e316aa856 Mon Sep 17 00:00:00 2001 From: Joe Ho Date: Tue, 10 Sep 2019 12:25:07 -0400 Subject: [PATCH 065/102] Undo the wait cursor changes. --- .../autopsy/corecomponents/DataResultViewerTable.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/corecomponents/DataResultViewerTable.java b/Core/src/org/sleuthkit/autopsy/corecomponents/DataResultViewerTable.java index 6d5788fd5f..e2babc903a 100644 --- a/Core/src/org/sleuthkit/autopsy/corecomponents/DataResultViewerTable.java +++ b/Core/src/org/sleuthkit/autopsy/corecomponents/DataResultViewerTable.java @@ -880,16 +880,12 @@ public class DataResultViewerTable extends AbstractDataResultViewer { void nextPage() { currentPage++; - DataResultViewerTable.this.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR)); postPageChangeEvent(); - DataResultViewerTable.this.setCursor(null); } void previousPage() { currentPage--; - DataResultViewerTable.this.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR)); postPageChangeEvent(); - DataResultViewerTable.this.setCursor(null); } @NbBundle.Messages({"# {0} - totalPages", From 8344422690f69c491aee05a722c90102eab5c424 Mon Sep 17 00:00:00 2001 From: Joe Ho Date: Tue, 10 Sep 2019 13:30:17 -0400 Subject: [PATCH 066/102] Revert DataResultViewerTable.java --- .../corecomponents/DataResultViewerTable.java | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/corecomponents/DataResultViewerTable.java b/Core/src/org/sleuthkit/autopsy/corecomponents/DataResultViewerTable.java index e2babc903a..50aef1a14e 100644 --- a/Core/src/org/sleuthkit/autopsy/corecomponents/DataResultViewerTable.java +++ b/Core/src/org/sleuthkit/autopsy/corecomponents/DataResultViewerTable.java @@ -81,12 +81,12 @@ import org.sleuthkit.autopsy.corecomponentinterfaces.DataResultViewer; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; import org.sleuthkit.autopsy.coreutils.ThreadConfined; +import org.sleuthkit.autopsy.datamodel.NodeProperty; +import org.sleuthkit.autopsy.datamodel.NodeSelectionInfo; import org.sleuthkit.autopsy.datamodel.BaseChildFactory; import org.sleuthkit.autopsy.datamodel.BaseChildFactory.PageChangeEvent; import org.sleuthkit.autopsy.datamodel.BaseChildFactory.PageCountChangeEvent; import org.sleuthkit.autopsy.datamodel.BaseChildFactory.PageSizeChangeEvent; -import org.sleuthkit.autopsy.datamodel.NodeProperty; -import org.sleuthkit.autopsy.datamodel.NodeSelectionInfo; /** * A tabular result viewer that displays the children of the given root node @@ -893,21 +893,21 @@ public class DataResultViewerTable extends AbstractDataResultViewer { "DataResultViewerTable.goToPageTextField.err=Invalid page number"}) void gotoPage() { try { - int saveCurrentPage = currentPage; currentPage = Integer.decode(gotoPageTextField.getText()); - if (currentPage > totalPages || currentPage < 1) { - currentPage = saveCurrentPage; - JOptionPane.showMessageDialog(DataResultViewerTable.this, - Bundle.DataResultViewerTable_goToPageTextField_msgDlg(totalPages), - Bundle.DataResultViewerTable_goToPageTextField_err(), - JOptionPane.WARNING_MESSAGE); - return; - } - postPageChangeEvent(); } catch (NumberFormatException e) { //ignore input return; } + + if (currentPage > totalPages || currentPage < 1) { + currentPage = 1; + JOptionPane.showMessageDialog(DataResultViewerTable.this, + Bundle.DataResultViewerTable_goToPageTextField_msgDlg(totalPages), + Bundle.DataResultViewerTable_goToPageTextField_err(), + JOptionPane.WARNING_MESSAGE); + return; + } + postPageChangeEvent(); } /** From 77f5de5a30d1c47a1b34d51d0bcd264933a6da54 Mon Sep 17 00:00:00 2001 From: Raman Date: Tue, 10 Sep 2019 15:55:37 -0400 Subject: [PATCH 067/102] attachDatabase() only takes the exact DB filename to match. --- .../autopsy/coreutils/AppSQLiteDB.java | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/coreutils/AppSQLiteDB.java b/Core/src/org/sleuthkit/autopsy/coreutils/AppSQLiteDB.java index 900e8aa45e..6812f6daf3 100644 --- a/Core/src/org/sleuthkit/autopsy/coreutils/AppSQLiteDB.java +++ b/Core/src/org/sleuthkit/autopsy/coreutils/AppSQLiteDB.java @@ -139,7 +139,6 @@ public final class AppSQLiteDB implements Closeable { * * @param dataSource data source in which to look file the db file * @param dbName name of db file to look for - * @param matchExactName specified whether the name is an exact name or a pattern * @param dbPath path in which to look for the db file * @param dbAlias alias name to attach the database as * @@ -149,12 +148,14 @@ public final class AppSQLiteDB implements Closeable { * @throws SQLException in case of an SQL error */ public AbstractFile attachDatabase(DataSource dataSource, String dbName, - boolean matchExactName, String dbPath, String dbAlias) throws SQLException { + String dbPath, String dbAlias) throws SQLException { try { - Collection dbFileBundles = findAndCopySQLiteDB(dataSource, dbName, matchExactName, dbPath, true); - for (AppSQLiteDBFileBundle dbFileBundle: dbFileBundles) { + // find and copy DB files with exact name and path. + Collection dbFileBundles = findAndCopySQLiteDB(dataSource, dbName, true, dbPath, true); + if (!dbFileBundles.isEmpty()) { + AppSQLiteDBFileBundle dbFileBundle = dbFileBundles.iterator().next(); String attachDbSql = String.format("ATTACH DATABASE '%s' AS '%s'", dbFileBundle.getFileCopy().getPath(), dbAlias); //NON-NLS - statement.executeUpdate(attachDbSql); + statement.executeUpdate(attachDbSql); return dbFileBundle.getAbstractFile(); } @@ -182,16 +183,14 @@ public final class AppSQLiteDB implements Closeable { private static Collection findAndCopySQLiteDB(DataSource dataSource, String dbName, boolean matchExactName, String dbPath, boolean matchExactPath) throws TskCoreException { - List dbFileBundles = new ArrayList<> (); Case openCase; - try { openCase = Case.getCurrentCaseThrows(); } catch (NoCurrentCaseException ex) { throw new TskCoreException("Failed to get current case.", ex); } - List absFiles; + List dbFileBundles = new ArrayList<> (); long fileId = 0; String localDiskPath = ""; @@ -212,7 +211,7 @@ public final class AppSQLiteDB implements Closeable { } whereClause += String.format(" AND data_source_obj_id = %s", dataSource.getId()); - absFiles = skCase.findAllFilesWhere(whereClause); + List absFiles = skCase.findAllFilesWhere(whereClause); for (AbstractFile absFile : absFiles) { try { localDiskPath = openCase.getTempDirectory() From 4c05fd5a219aa84fc738599cef76405433e44220 Mon Sep 17 00:00:00 2001 From: Eugene Livis Date: Tue, 10 Sep 2019 17:00:41 -0400 Subject: [PATCH 068/102] Added utility method to UTF-8 sanitize a string --- Core/src/org/sleuthkit/autopsy/coreutils/FileUtil.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/coreutils/FileUtil.java b/Core/src/org/sleuthkit/autopsy/coreutils/FileUtil.java index 4a7e6da65f..6422fbcad2 100644 --- a/Core/src/org/sleuthkit/autopsy/coreutils/FileUtil.java +++ b/Core/src/org/sleuthkit/autopsy/coreutils/FileUtil.java @@ -21,6 +21,7 @@ package org.sleuthkit.autopsy.coreutils; import java.io.File; import java.io.IOException; import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import java.util.logging.Level; import org.openide.filesystems.FileObject; import java.nio.file.Files; @@ -181,7 +182,7 @@ public class FileUtil { * @return Sanitized string */ public static String utf8SanitizeFileName(String fileName) { - Charset charset = Charset.forName("UTF-8"); + Charset charset = StandardCharsets.UTF_8; return charset.decode(charset.encode(escapeFileName(fileName))).toString(); } From 2cfa83a50b09e399c071a601b4299c22b0fae698 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Tue, 10 Sep 2019 17:50:05 -0400 Subject: [PATCH 069/102] 5479 move SCO task creation inside SCO check for optimization --- .../autopsy/datamodel/AbstractAbstractFileNode.java | 7 +++---- .../autopsy/datamodel/BlackboardArtifactNode.java | 7 +++---- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java index e02a4ff776..f1573043ef 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java @@ -330,12 +330,11 @@ public abstract class AbstractAbstractFileNode extends A if (EamDb.isEnabled()) { properties.add(new NodeProperty<>(OCCURRENCES.toString(), OCCURRENCES.toString(), VALUE_LOADING, "")); } + // Get the SCO columns data in a background task + backgroundTasksPool.submit(new GetSCOTask( + new WeakReference<>(this), weakPcl)); } - // Get the SCO columns data in a background task - backgroundTasksPool.submit(new GetSCOTask( - new WeakReference<>(this), weakPcl)); - properties.add(new NodeProperty<>(MOD_TIME.toString(), MOD_TIME.toString(), NO_DESCR, ContentUtils.getStringTime(content.getMtime(), content))); properties.add(new NodeProperty<>(CHANGED_TIME.toString(), CHANGED_TIME.toString(), NO_DESCR, ContentUtils.getStringTime(content.getCtime(), content))); properties.add(new NodeProperty<>(ACCESS_TIME.toString(), ACCESS_TIME.toString(), NO_DESCR, ContentUtils.getStringTime(content.getAtime(), content))); diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/BlackboardArtifactNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/BlackboardArtifactNode.java index 0ea8b04660..b063a0f006 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/BlackboardArtifactNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/BlackboardArtifactNode.java @@ -369,12 +369,11 @@ public class BlackboardArtifactNode extends AbstractContentNode(Bundle.BlackboardArtifactNode_createSheet_count_name(), Bundle.BlackboardArtifactNode_createSheet_count_displayName(), VALUE_LOADING, "")); } + // Get the SCO columns data in a background task + backgroundTasksPool.submit(new GetSCOTask( + new WeakReference<>(this), weakPcl)); } - // Get the SCO columns data in a background task - backgroundTasksPool.submit(new GetSCOTask( - new WeakReference<>(this), weakPcl)); - if (artifact.getArtifactTypeID() == ARTIFACT_TYPE.TSK_INTERESTING_ARTIFACT_HIT.getTypeID()) { try { BlackboardAttribute attribute = artifact.getAttribute(new BlackboardAttribute.Type(ATTRIBUTE_TYPE.TSK_ASSOCIATED_ARTIFACT)); From 1a6d40665518ee4a1e8e2b79fe3c2c748d103c55 Mon Sep 17 00:00:00 2001 From: Joe Ho Date: Wed, 11 Sep 2019 10:13:16 -0400 Subject: [PATCH 070/102] AddMultipleImageTask no longer a thread. --- .../dsp/AddLogicalImageTask.java | 38 ++++--------------- .../dsp/AddMultipleImageTask.java | 15 ++++---- 2 files changed, 14 insertions(+), 39 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java index e5556b0c02..e8c11492a6 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java @@ -204,8 +204,13 @@ final class AddLogicalImageTask implements Runnable { try { privateCallback = new AddDataSourceCallback(); addMultipleImageTask = new AddMultipleImageTask(deviceId, imagePaths, timeZone , progressMonitor, privateCallback); - multipleImageThread = new Thread(addMultipleImageTask); - multipleImageThread.start(); + addMultipleImageTask.run(); + if (privateCallback.getResult() == DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS) { + // TODO: Delete destination directory when 5453 (VHD file is not closed upon revert) is fixed. + // bait out + callback.done(privateCallback.getResult(), privateCallback.getErrorMessages(), privateCallback.getNewDataSources()); + return; + } } catch (NoCurrentCaseException ex) { String msg = Bundle.AddLogicalImageTask_noCurrentCase(); errorList.add(msg); @@ -215,34 +220,6 @@ final class AddLogicalImageTask implements Runnable { } try { - if (createVHD) { - // Wait for addMultipleImageTask to finish, via its privateCallback - while (privateCallback.isInProgress()) { - try { - Thread.sleep(1000); - } catch (InterruptedException ex) { - // Got the addMultipleImageTask stop (cancel) - LOGGER.log(Level.INFO, "AddMultipleImageTask interrupted", ex); // NON-NLS - // now wait for addMultipleImageTask to revert and finally callback - while (privateCallback.isInProgress()) { - try { - Thread.sleep(1000); - } catch (InterruptedException ex2) { - LOGGER.log(Level.INFO, "AddMultipleImageTask interrupted 2", ex2); // NON-NLS - } - } - } - } - // TODO: Delete destination directory when 5453 (VHD file is not closed upon revert) is fixed. - // if (cancelled) { - // deleteDestinationDirectory(); - // } - if (privateCallback.getResult() == DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS) { - // bait out - callback.done(privateCallback.getResult(), privateCallback.getErrorMessages(), privateCallback.getNewDataSources()); - return; - } - } progressMonitor.setProgressText(Bundle.AddLogicalImageTask_addingInterestingFiles()); addingInterestingFiles = true; addInterestingFiles(dest, Paths.get(dest.toString(), resultsFilename), createVHD); @@ -293,7 +270,6 @@ final class AddLogicalImageTask implements Runnable { LOGGER.log(Level.WARNING, "AddLogicalImageTask cancelled, processing may be incomplete"); // NON-NLS cancelled = true; if (addMultipleImageTask != null) { - multipleImageThread.interrupt(); addMultipleImageTask.cancelTask(); } if (!createVHD && !addingInterestingFiles) { diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddMultipleImageTask.java b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddMultipleImageTask.java index 681f7733da..f5f64665cf 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddMultipleImageTask.java +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddMultipleImageTask.java @@ -213,15 +213,14 @@ class AddMultipleImageTask implements Runnable { tskAddImageProcessStopped = true; if (addImageProcess != null) { try { - /* - * All this does is set a flag that will make the TSK add - * image process exit when the flag is checked between - * processing steps. The state of the flag is not - * accessible, so record it here so that it is known that - * the revert method of the process object needs to be - * called. - */ addImageProcess.stop(); + addImageProcess.revert(); + if (tskAddImageProcessStopped) { + List errorMessages = new ArrayList<>(); + List emptyDataSources = new ArrayList<>(); + errorMessages.add(Bundle.AddMultipleImageTask_cancelled()); + callback.done(DataSourceProcessorResult.CRITICAL_ERRORS, errorMessages, emptyDataSources); + } } catch (TskCoreException ex) { LOGGER.log(Level.SEVERE, "Cancellation: addImagePRocess.stop failed", ex); // NON-NLS } From fc137e4ab6e3f84342bca38da0fe378253608c1f Mon Sep 17 00:00:00 2001 From: Raman Date: Wed, 11 Sep 2019 10:16:16 -0400 Subject: [PATCH 071/102] Updated addCalllog() api to support group calls. --- .../autopsy/coreutils/AppDBParserHelper.java | 182 ++++++++++++------ 1 file changed, 128 insertions(+), 54 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/coreutils/AppDBParserHelper.java b/Core/src/org/sleuthkit/autopsy/coreutils/AppDBParserHelper.java index 32848c6437..c7bd656b78 100644 --- a/Core/src/org/sleuthkit/autopsy/coreutils/AppDBParserHelper.java +++ b/Core/src/org/sleuthkit/autopsy/coreutils/AppDBParserHelper.java @@ -248,13 +248,15 @@ public final class AppDBParserHelper { * @param relationshipType type of relationship * @param dateTime date/time of relationship */ - private void addRelationship(AccountFileInstance selfAccount, AccountFileInstance otherAccount, + private void addRelationship(AccountFileInstance selfAccountInstance, AccountFileInstance otherAccountInstance, BlackboardArtifact sourceArtifact, Relationship.Type relationshipType, long dateTime) { try { - Case.getCurrentCase().getSleuthkitCase().getCommunicationsManager().addRelationships(selfAccount, - Collections.singletonList(otherAccount), sourceArtifact, relationshipType, dateTime); + if (selfAccountInstance.getAccount() != otherAccountInstance.getAccount()) { + Case.getCurrentCase().getSleuthkitCase().getCommunicationsManager().addRelationships(selfAccountInstance, + Collections.singletonList(otherAccountInstance), sourceArtifact, relationshipType, dateTime); + } } catch (TskCoreException | TskDataException ex) { - logger.log(Level.SEVERE, String.format("Unable to add relationship between account %s and account %s", selfAccount.toString(), otherAccount.toString()), ex); //NON-NLS + logger.log(Level.SEVERE, String.format("Unable to add relationship between account %s and account %s", selfAccountInstance.toString(), otherAccountInstance.toString()), ex); //NON-NLS } } @@ -366,17 +368,6 @@ public final class AppDBParserHelper { String threadId, Collection otherAttributesList) { - - // Create a comma separated string of recipients - String toAddresses = null; - if (recipientsList != null && (!recipientsList.isEmpty())) { - StringBuilder toAddressesSb = new StringBuilder(); - for(Account.Address recipient : recipientsList) { - toAddressesSb = toAddressesSb.length() > 0 ? toAddressesSb.append(",").append(recipient.getDisplayName()) : toAddressesSb.append(recipient.getDisplayName()); - } - toAddresses = toAddressesSb.toString(); - } - // Created message artifact. BlackboardArtifact msgArtifact = null; try { @@ -399,6 +390,8 @@ public final class AppDBParserHelper { if (fromAddress != null && !StringUtils.isEmpty(fromAddress.getDisplayName())) { msgArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_FROM, moduleName, fromAddress.getDisplayName())); } + // Create a comma separated string of recipients + String toAddresses = addressListToString(recipientsList); if (toAddresses != null && !StringUtils.isEmpty(toAddresses)) { msgArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_TO, moduleName, toAddresses)); } @@ -413,7 +406,6 @@ public final class AppDBParserHelper { msgArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_THREAD_ID, moduleName, threadId)); } - // Add other specified attributes for (BlackboardAttribute otherAttribute: otherAttributesList) { msgArtifact.addAttribute(otherAttribute); @@ -444,8 +436,6 @@ public final class AppDBParserHelper { // post artifact Case.getCurrentCase().getSleuthkitCase().getBlackboard().postArtifact(msgArtifact, this.moduleName); - - } catch (TskCoreException ex) { logger.log(Level.SEVERE, "Unable to add message artifact", ex); //NON-NLS return null; @@ -461,24 +451,22 @@ public final class AppDBParserHelper { /** * Adds a TSK_CALLLOG artifact. * - * Also creates an account instance for the caller/receiver, and creates a - * relationship between the self account and the caller/receiver account. + * Also creates an account instance for the caller/callee, and creates a + * relationship between the self account and the caller/callee account. * - * @param otherAccountUniqueID unique id for the caller/receiver account * @param direction call direction - * @param fromPhoneNumber originating phone number, may be empty - * @param toPhoneNumber recipient phone number, may be empty + * @param fromAddress caller address, may be empty + * @param toAddress callee address, may be empty * @param startDateTime start date/time * @param endDateTime end date/time - * @param contactName contact name, may be empty * * @return call log artifact */ - public BlackboardArtifact addCalllog( String otherAccountUniqueID, - String direction, String fromPhoneNumber, String toPhoneNumber, - long startDateTime, long endDateTime, String contactName) { - return addCalllog(otherAccountUniqueID, direction, fromPhoneNumber, toPhoneNumber, - startDateTime, endDateTime, contactName, + public BlackboardArtifact addCalllog( String direction, + Account.Address fromAddress, Account.Address toAddress, + long startDateTime, long endDateTime) { + return addCalllog(direction, fromAddress, toAddress, + startDateTime, endDateTime, Collections.emptyList()); } @@ -486,23 +474,74 @@ public final class AppDBParserHelper { * Adds a TSK_CALLLOG artifact. * * Also creates an account instance for the caller/receiver, and creates a - * relationship between the device owner account and the caller/receiver account. + * relationship between the self account and the caller/receiver account. * - * @param otherAccountUniqueID - * @param direction - * @param fromPhoneNumber - * @param toPhoneNumber - * @param startDateTime - * @param endDateTime - * @param contactName - * @param otherAttributesList + * @param direction call direction + * @param fromAddress caller address, may be empty + * @param toAddress callee address, may be empty + * @param startDateTime start date/time + * @param endDateTime end date/time + * @param otherAttributesList other attributes + * + * @return call log artifact + */ + public BlackboardArtifact addCalllog( String direction, + Account.Address fromAddress, + Account.Address toAddress, + long startDateTime, long endDateTime, + Collection otherAttributesList) { + return addCalllog(direction, + fromAddress, + Arrays.asList(toAddress), + startDateTime, endDateTime, + otherAttributesList); + } + + /** + * Adds a TSK_CALLLOG artifact. + * + * Also creates an account instance for the caller/callees, + * and creates a relationship between the device owner account and the caller account + * as well between the device owner account and each callee account + * + * @param direction call direction + * @param fromAddress caller address, may be empty + * @param toAddressList callee address list, may be empty + * @param startDateTime start date/time + * @param endDateTime end date/time + * + * @return call log artifact + */ + public BlackboardArtifact addCalllog( String direction, + Account.Address fromAddress, + Collection toAddressList, + long startDateTime, long endDateTime) { + + return addCalllog(direction, fromAddress, toAddressList, + startDateTime, endDateTime, + Collections.emptyList()); + } + /** + * Adds a TSK_CALLLOG artifact. + * + * Also creates an account instance for the caller/callees, + * and creates a relationship between the device owner account and the caller account + * as well between the device owner account and each callee account + * + * @param direction call direction + * @param fromAddress caller address, may be empty + * @param toAddressList callee address list, may be empty + * @param startDateTime start date/time + * @param endDateTime end date/time + * @param otherAttributesList other attributes * * @return calllog artifact */ - public BlackboardArtifact addCalllog(String otherAccountUniqueID, - String direction, String fromPhoneNumber, String toPhoneNumber, - long startDateTime, long endDateTime, String contactName, - Collection otherAttributesList) { + public BlackboardArtifact addCalllog( String direction, + Account.Address fromAddress, + Collection toAddressList, + long startDateTime, long endDateTime, + Collection otherAttributesList) { BlackboardArtifact callLogArtifact = null; try { // Create TSK_CALLLOG artifact @@ -519,14 +558,17 @@ public final class AppDBParserHelper { if (!StringUtils.isEmpty(direction)) { callLogArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DIRECTION, moduleName, direction)); } - if (!StringUtils.isEmpty(fromPhoneNumber)) { - callLogArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_FROM, moduleName, fromPhoneNumber)); + if (fromAddress != null) { + callLogArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_FROM, moduleName, fromAddress.getUniqueID())); + if (!StringUtils.isEmpty(fromAddress.getDisplayName())) { + callLogArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_NAME, moduleName, fromAddress.getDisplayName())); + } } - if (!StringUtils.isEmpty(toPhoneNumber)) { - callLogArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_TO, moduleName, toPhoneNumber)); - } - if (!StringUtils.isEmpty(contactName)) { - callLogArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_NAME, moduleName, contactName)); + + // Create a comma separated string of recipients + String toAddresses = addressListToString(toAddressList); + if (!StringUtils.isEmpty(toAddresses)) { + callLogArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_TO, moduleName, toAddresses)); } // Add other specified attributes @@ -534,13 +576,24 @@ public final class AppDBParserHelper { callLogArtifact.addAttribute(otherAttribute); } - // Find/Create an account instance for the sender/recipient - // Create a relationship between selfAccount and contactAccount - AccountFileInstance contactAccountInstance = createAccountInstance(accountsType, otherAccountUniqueID); - if (selfAccountInstance != null) { - addRelationship (selfAccountInstance, contactAccountInstance, callLogArtifact, Relationship.Type.CALL_LOG, 0 ); + // Create a relationship between selfAccount and caller + if (fromAddress != null) { + AccountFileInstance callerAccountInstance = createAccountInstance(accountsType, fromAddress.getUniqueID()); + if (selfAccountInstance != null) { + addRelationship (selfAccountInstance, callerAccountInstance, callLogArtifact, Relationship.Type.CALL_LOG, (startDateTime > 0) ? startDateTime : 0 ); + } } + // Create a relationship between selfAccount and each callee + if (toAddressList != null) { + for(Account.Address callee : toAddressList) { + AccountFileInstance calleeAccountInstance = createAccountInstance(accountsType, callee.getUniqueID()); + if (selfAccountInstance != null) { + addRelationship (selfAccountInstance, calleeAccountInstance, callLogArtifact, Relationship.Type.CALL_LOG, (startDateTime > 0) ? startDateTime : 0 ); + } + } + } + // post artifact Case.getCurrentCase().getSleuthkitCase().getBlackboard().postArtifact(callLogArtifact, this.moduleName); } catch (TskCoreException ex) { @@ -1142,4 +1195,25 @@ public final class AppDBParserHelper { return gpsTrackpointArtifact; } + /** + * Converts a list of addresses into a single comma separated string of + * addresses. + * + * @param addressList + * @return comma separated string of addresses + */ + private String addressListToString(Collection addressList) { + + String toAddresses = ""; + if (addressList != null && (!addressList.isEmpty())) { + StringBuilder toAddressesSb = new StringBuilder(); + for(Account.Address address : addressList) { + String displayAddress = !StringUtils.isEmpty(address.getDisplayName()) ? address.getDisplayName() : address.getUniqueID(); + toAddressesSb = toAddressesSb.length() > 0 ? toAddressesSb.append(",").append(displayAddress) : toAddressesSb.append(displayAddress); + } + toAddresses = toAddressesSb.toString(); + } + + return toAddresses; + } } From 77ed043911992a5c47e3af3a63e37ba958b17060 Mon Sep 17 00:00:00 2001 From: Raman Date: Wed, 11 Sep 2019 13:41:18 -0400 Subject: [PATCH 072/102] Defined enums for communication direction and call media type. --- .../autopsy/coreutils/AppDBParserHelper.java | 128 ++++++++++++++++-- InternalPythonModules/android/imo.py | 12 +- 2 files changed, 122 insertions(+), 18 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/coreutils/AppDBParserHelper.java b/Core/src/org/sleuthkit/autopsy/coreutils/AppDBParserHelper.java index c7bd656b78..0b6f567b87 100644 --- a/Core/src/org/sleuthkit/autopsy/coreutils/AppDBParserHelper.java +++ b/Core/src/org/sleuthkit/autopsy/coreutils/AppDBParserHelper.java @@ -56,7 +56,48 @@ public final class AppDBParserHelper { UNREAD, /// message has not been read READ /// message has been read } + + /** + * Enum for call/message direction + */ + public enum CommunicationDirection + { + UNKNOWN("Unknown"), + INCOMING("Incoming"), + OUTGOING("Outgoing"); + private final String dirStr; + + CommunicationDirection(String dir) { + this.dirStr = dir; + } + + public String getString() { + return dirStr; + } + } + + /** + * Enum for call media type + */ + public enum CallMediaType + { + UNKNOWN("Unknown"), + AUDIO("Audio"), + VIDEO("Video"); + + private final String typeStr; + + CallMediaType(String type) { + this.typeStr = type; + } + + public String getString() { + return typeStr; + } + } + + private final AbstractFile dbAbstractFile; private final String moduleName; @@ -280,7 +321,8 @@ public final class AppDBParserHelper { * @return message artifact */ public BlackboardArtifact addMessage( - String messageType, String direction, + String messageType, + CommunicationDirection direction, Account.Address fromAddress, Account.Address toAddress, long dateTime, MessageReadStatusEnum readStatus, @@ -311,7 +353,8 @@ public final class AppDBParserHelper { * * @return message artifact */ - public BlackboardArtifact addMessage( String messageType, String direction, + public BlackboardArtifact addMessage( String messageType, + CommunicationDirection direction, Account.Address fromAddress, Account.Address toAddress, long dateTime, MessageReadStatusEnum readStatus, String subject, @@ -347,7 +390,8 @@ public final class AppDBParserHelper { * * @return message artifact */ - public BlackboardArtifact addMessage( String messageType, String direction, + public BlackboardArtifact addMessage( String messageType, + CommunicationDirection direction, Account.Address fromAddress, List recipientsList, long dateTime, MessageReadStatusEnum readStatus, @@ -360,7 +404,8 @@ public final class AppDBParserHelper { } - public BlackboardArtifact addMessage( String messageType, String direction, + public BlackboardArtifact addMessage( String messageType, + CommunicationDirection direction, Account.Address fromAddress, List recipientsList, long dateTime, MessageReadStatusEnum readStatus, @@ -384,8 +429,8 @@ public final class AppDBParserHelper { if (!StringUtils.isEmpty(messageType)) { msgArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_MESSAGE_TYPE, moduleName, messageType)); } - if (!StringUtils.isEmpty(direction)) { - msgArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DIRECTION, moduleName, direction)); + if (direction != CommunicationDirection.UNKNOWN) { + msgArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DIRECTION, moduleName, direction.getString())); } if (fromAddress != null && !StringUtils.isEmpty(fromAddress.getDisplayName())) { msgArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_FROM, moduleName, fromAddress.getDisplayName())); @@ -462,11 +507,34 @@ public final class AppDBParserHelper { * * @return call log artifact */ - public BlackboardArtifact addCalllog( String direction, + public BlackboardArtifact addCalllog(CommunicationDirection direction, Account.Address fromAddress, Account.Address toAddress, long startDateTime, long endDateTime) { return addCalllog(direction, fromAddress, toAddress, startDateTime, endDateTime, + CallMediaType.UNKNOWN); + } + + /** + * Adds a TSK_CALLLOG artifact. + * + * Also creates an account instance for the caller/callee, and creates a + * relationship between the self account and the caller/callee account. + * + * @param direction call direction + * @param fromAddress caller address, may be empty + * @param toAddress callee address, may be empty + * @param startDateTime start date/time + * @param endDateTime end date/time + * @param mediaType media type + * + * @return call log artifact + */ + public BlackboardArtifact addCalllog(CommunicationDirection direction, + Account.Address fromAddress, Account.Address toAddress, + long startDateTime, long endDateTime, CallMediaType mediaType) { + return addCalllog(direction, fromAddress, toAddress, + startDateTime, endDateTime, mediaType, Collections.emptyList()); } @@ -481,19 +549,22 @@ public final class AppDBParserHelper { * @param toAddress callee address, may be empty * @param startDateTime start date/time * @param endDateTime end date/time + * @param mediaType media type * @param otherAttributesList other attributes * * @return call log artifact */ - public BlackboardArtifact addCalllog( String direction, + public BlackboardArtifact addCalllog(CommunicationDirection direction, Account.Address fromAddress, Account.Address toAddress, - long startDateTime, long endDateTime, + long startDateTime, long endDateTime, + CallMediaType mediaType, Collection otherAttributesList) { return addCalllog(direction, fromAddress, Arrays.asList(toAddress), startDateTime, endDateTime, + mediaType, otherAttributesList); } @@ -512,15 +583,44 @@ public final class AppDBParserHelper { * * @return call log artifact */ - public BlackboardArtifact addCalllog( String direction, + public BlackboardArtifact addCalllog(CommunicationDirection direction, Account.Address fromAddress, Collection toAddressList, long startDateTime, long endDateTime) { return addCalllog(direction, fromAddress, toAddressList, startDateTime, endDateTime, + CallMediaType.UNKNOWN); + } + + /** + * Adds a TSK_CALLLOG artifact. + * + * Also creates an account instance for the caller/callees, + * and creates a relationship between the device owner account and the caller account + * as well between the device owner account and each callee account + * + * @param direction call direction + * @param fromAddress caller address, may be empty + * @param toAddressList callee address list, may be empty + * @param startDateTime start date/time + * @param endDateTime end date/time + * @param mediaType call media type + * + * @return call log artifact + */ + public BlackboardArtifact addCalllog(CommunicationDirection direction, + Account.Address fromAddress, + Collection toAddressList, + long startDateTime, long endDateTime, + CallMediaType mediaType) { + + return addCalllog(direction, fromAddress, toAddressList, + startDateTime, endDateTime, + mediaType, Collections.emptyList()); } + /** * Adds a TSK_CALLLOG artifact. * @@ -533,14 +633,16 @@ public final class AppDBParserHelper { * @param toAddressList callee address list, may be empty * @param startDateTime start date/time * @param endDateTime end date/time + * @param mediaType called media type * @param otherAttributesList other attributes * * @return calllog artifact */ - public BlackboardArtifact addCalllog( String direction, + public BlackboardArtifact addCalllog(CommunicationDirection direction, Account.Address fromAddress, Collection toAddressList, long startDateTime, long endDateTime, + CallMediaType mediaType, Collection otherAttributesList) { BlackboardArtifact callLogArtifact = null; try { @@ -555,8 +657,8 @@ public final class AppDBParserHelper { callLogArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_END, moduleName, endDateTime)); } - if (!StringUtils.isEmpty(direction)) { - callLogArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DIRECTION, moduleName, direction)); + if (direction != CommunicationDirection.UNKNOWN) { + callLogArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DIRECTION, moduleName, direction.getString())); } if (fromAddress != null) { callLogArtifact.addAttribute(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PHONE_NUMBER_FROM, moduleName, fromAddress.getUniqueID())); diff --git a/InternalPythonModules/android/imo.py b/InternalPythonModules/android/imo.py index d887a6da53..0bb1b913b2 100644 --- a/InternalPythonModules/android/imo.py +++ b/InternalPythonModules/android/imo.py @@ -33,6 +33,8 @@ from org.sleuthkit.autopsy.coreutils import Logger from org.sleuthkit.autopsy.coreutils import MessageNotifyUtil from org.sleuthkit.autopsy.coreutils import AppSQLiteDB from org.sleuthkit.autopsy.coreutils import AppDBParserHelper +from org.sleuthkit.autopsy.coreutils.AppDBParserHelper import MessageReadStatusEnum +from org.sleuthkit.autopsy.coreutils.AppDBParserHelper import CommunicationDirection from org.sleuthkit.autopsy.datamodel import ContentUtils from org.sleuthkit.autopsy.ingest import IngestJobContext from org.sleuthkit.datamodel import AbstractFile @@ -97,20 +99,20 @@ class IMOAnalyzer(general.AndroidComponentAnalyzer): uniqueId = messagesResultSet.getString("buid") if (messagesResultSet.getInt("message_type") == 1): - direction = "Incoming" + direction = CommunicationDirection.INCOMING fromAddress = Account.Address(uniqueId, name) else: - direction = "Outgoing" + direction = CommunicationDirection.OUTGOING toAddress = Account.Address(uniqueId, name) message_read = messagesResultSet.getInt("message_read") if (message_read == 1): - msgReadStatus = AppDBParserHelper.MessageReadStatusEnum.READ + msgReadStatus = MessageReadStatusEnum.READ elif (message_read == 0): - msgReadStatus = AppDBParserHelper.MessageReadStatusEnum.UNREAD + msgReadStatus = MessageReadStatusEnum.UNREAD else: - msgReadStatus = AppDBParserHelper.MessageReadStatusEnum.UNKNOWN + msgReadStatus = MessageReadStatusEnum.UNKNOWN timeStamp = messagesResultSet.getLong("timestamp") / 1000000000 From 97ac6821ce4a1c05f6cc7a11d3f8cafb0d05a4b2 Mon Sep 17 00:00:00 2001 From: Raman Date: Wed, 11 Sep 2019 15:49:26 -0400 Subject: [PATCH 073/102] Updated AppDBParserHelper to take in a Account.Address for self account, instead of just an account id. --- .../sleuthkit/autopsy/coreutils/AppDBParserHelper.java | 6 +++--- InternalPythonModules/android/imo.py | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/coreutils/AppDBParserHelper.java b/Core/src/org/sleuthkit/autopsy/coreutils/AppDBParserHelper.java index 0b6f567b87..538ea07098 100644 --- a/Core/src/org/sleuthkit/autopsy/coreutils/AppDBParserHelper.java +++ b/Core/src/org/sleuthkit/autopsy/coreutils/AppDBParserHelper.java @@ -142,17 +142,17 @@ public final class AppDBParserHelper { * @param dbFile database file being parsed by the module * @param accountsType account types created by this module * @param selfAccountType self account type to be created for this module - * @param selfAccountId account unique id for the self account + * @param selfAccountAddress account unique id for the self account * * @throws TskCoreException */ - public AppDBParserHelper(String moduleName, AbstractFile dbFile, Account.Type accountsType, Account.Type selfAccountType, String selfAccountId) throws TskCoreException { + public AppDBParserHelper(String moduleName, AbstractFile dbFile, Account.Type accountsType, Account.Type selfAccountType, Account.Address selfAccountAddress) throws TskCoreException { this.moduleName = moduleName; this.dbAbstractFile = dbFile; this.accountsType = accountsType; - this.selfAccountInstance = Case.getCurrentCase().getSleuthkitCase().getCommunicationsManager().createAccountFileInstance(selfAccountType, selfAccountId, moduleName, dbFile); + this.selfAccountInstance = Case.getCurrentCase().getSleuthkitCase().getCommunicationsManager().createAccountFileInstance(selfAccountType, selfAccountAddress.getUniqueID(), moduleName, dbFile); } /** diff --git a/InternalPythonModules/android/imo.py b/InternalPythonModules/android/imo.py index 0bb1b913b2..9b3ec78528 100644 --- a/InternalPythonModules/android/imo.py +++ b/InternalPythonModules/android/imo.py @@ -56,7 +56,7 @@ class IMOAnalyzer(general.AndroidComponentAnalyzer): self._logger = Logger.getLogger(self.__class__.__name__) def analyze(self, dataSource, fileManager, context): - selfAccountId = None + selfAccountAddress = None accountDbs = AppSQLiteDB.findAppDatabases(dataSource, "accountdb.db", True, "com.imo.android.imous") for accountDb in accountDbs: try: @@ -65,8 +65,8 @@ class IMOAnalyzer(general.AndroidComponentAnalyzer): # We can determine the IMO user ID of the device owner. # Therefore we can create and use a app account and use that # as a 'self' account instead of a Device account - if not selfAccountId: - selfAccountId = accountResultSet.getString("name") + if not selfAccountAddress: + selfAccountAddress = Account.Address(accountResultSet.getString("uid"), accountResultSet.getString("name")) except SQLException as ex: self._logger.log(Level.SEVERE, "Error processing query result for account", ex) @@ -77,7 +77,7 @@ class IMOAnalyzer(general.AndroidComponentAnalyzer): for friendsDb in friendsDbs: try: friendsDBHelper = AppDBParserHelper("IMO Parser", friendsDb.getDBFile(), - Account.Type.IMO, Account.Type.IMO, selfAccountId ) + Account.Type.IMO, Account.Type.IMO, selfAccountAddress ) contactsResultSet = friendsDb.runQuery("SELECT buid, name FROM friends") if contactsResultSet is not None: while contactsResultSet.next(): From f4a602cb2836bf93e49bc8636ecb65199bb427eb Mon Sep 17 00:00:00 2001 From: esaunders Date: Wed, 11 Sep 2019 16:56:24 -0400 Subject: [PATCH 074/102] Bring Autopsy Travis build up to date with TSK Travis build. --- .travis.yml | 41 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 36 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4bb150cd6b..6a8926993b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,24 +1,55 @@ language: java sudo: required -dist: trusty +dist: bionic os: - linux + env: global: - TSK_HOME=$TRAVIS_BUILD_DIR/sleuthkit/sleuthkit + +addons: + apt: + update: true + packages: + - libafflib-dev + - libewf-dev + - libpq-dev + - autopoint + - libsqlite3-dev + - ant + - libcppunit-dev + - wget + - openjdk-8-jdk + - openjfx=8u161-b12-1ubuntu2 + - libopenjfx-java=8u161-b12-1ubuntu2 + - libopenjfx-jni=8u161-b12-1ubuntu2 + homebrew: + update: true + packages: + - ant + - libewf + - gettext + - cppunit + - afflib + python: - "2.7" -jdk: - - oraclejdk8 + before_install: - git clone https://github.com/sleuthkit/sleuthkit.git sleuthkit/sleuthkit - python setupSleuthkitBranch.py + install: - sudo apt-get install testdisk - - cd sleuthkit/sleuthkit - - sh travis_build.sh + - ./travis_install_libs.sh + script: - set -e + - echo "Building TSK..." + - cd sleuthkit/sleuthkit + - ./bootstrap && ./configure --prefix=/usr && make + - pushd bindings/java/ && ant -q dist-PostgreSQL && popd - echo "Building Autopsy..." && echo -en 'travis_fold:start:script.build\\r' - cd $TRAVIS_BUILD_DIR/ - ant build From 98e860df7ac4de5124da04bba1607cf37230f03f Mon Sep 17 00:00:00 2001 From: esaunders Date: Wed, 11 Sep 2019 17:02:14 -0400 Subject: [PATCH 075/102] Shouldn't have moved the line that cd'd into sleuthkit. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 6a8926993b..3b823acab7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -42,12 +42,12 @@ before_install: install: - sudo apt-get install testdisk + - cd sleuthkit/sleuthkit - ./travis_install_libs.sh script: - set -e - echo "Building TSK..." - - cd sleuthkit/sleuthkit - ./bootstrap && ./configure --prefix=/usr && make - pushd bindings/java/ && ant -q dist-PostgreSQL && popd - echo "Building Autopsy..." && echo -en 'travis_fold:start:script.build\\r' From 082718af2879ad41ee0136bb7ff6e8cab49ceca8 Mon Sep 17 00:00:00 2001 From: esaunders Date: Wed, 11 Sep 2019 17:13:24 -0400 Subject: [PATCH 076/102] Get the java version set up correctly. --- .travis.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.travis.yml b/.travis.yml index 3b823acab7..fc6d41a895 100644 --- a/.travis.yml +++ b/.travis.yml @@ -45,6 +45,14 @@ install: - cd sleuthkit/sleuthkit - ./travis_install_libs.sh +before_script: + - if [ $TRAVIS_OS_NAME = linux ]; then + sudo update-alternatives --set java /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java; + sudo update-alternatives --set javac /usr/lib/jvm/java-8-openjdk-amd64/bin/javac; + export PATH=/usr/bin:$PATH; + unset JAVA_HOME; + fi + script: - set -e - echo "Building TSK..." From 6fd7a5cf61e440d95c399ca2b6a4ba9d5cca4483 Mon Sep 17 00:00:00 2001 From: esaunders Date: Wed, 11 Sep 2019 17:36:59 -0400 Subject: [PATCH 077/102] Add ant-optional package in an attempt to get JUnitTask --- .travis.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.travis.yml b/.travis.yml index fc6d41a895..a3a67e2b46 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,9 @@ dist: bionic os: - linux +jdk: + - openjdk8 + env: global: - TSK_HOME=$TRAVIS_BUILD_DIR/sleuthkit/sleuthkit @@ -18,6 +21,7 @@ addons: - autopoint - libsqlite3-dev - ant + - ant-optional - libcppunit-dev - wget - openjdk-8-jdk @@ -28,6 +32,7 @@ addons: update: true packages: - ant + - ant-optional - libewf - gettext - cppunit From 0437b9561a464de932db09546ef34ddfe755f9e6 Mon Sep 17 00:00:00 2001 From: esaunders Date: Wed, 11 Sep 2019 17:46:05 -0400 Subject: [PATCH 078/102] Remove jdk tag because it causes travis to error out. --- .travis.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index a3a67e2b46..7554bbc6a4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,9 +4,6 @@ dist: bionic os: - linux -jdk: - - openjdk8 - env: global: - TSK_HOME=$TRAVIS_BUILD_DIR/sleuthkit/sleuthkit From f40d95005974bd6d103c30db9f00a94b3f484d20 Mon Sep 17 00:00:00 2001 From: Eugene Livis Date: Wed, 11 Sep 2019 18:02:26 -0400 Subject: [PATCH 079/102] Modified Image Gallery tables to store strings as TEXT instead of VARCHAR(255) --- .../autopsy/imagegallery/datamodel/DrawableDB.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java index b13d6bebf5..e3310007fd 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java @@ -572,12 +572,12 @@ public final class DrawableDB { String sql = "CREATE TABLE if not exists drawable_files " //NON-NLS + "( obj_id BIGINT PRIMARY KEY, " //NON-NLS + " data_source_obj_id BIGINT NOT NULL, " - + " path VARCHAR(255), " //NON-NLS - + " name VARCHAR(255), " //NON-NLS + + " path TEXT, " //NON-NLS + + " name TEXT, " //NON-NLS + " created_time integer, " //NON-NLS + " modified_time integer, " //NON-NLS - + " make VARCHAR(255) DEFAULT NULL, " //NON-NLS - + " model VARCHAR(255) DEFAULT NULL, " //NON-NLS + + " make TEXT DEFAULT NULL, " //NON-NLS + + " model TEXT DEFAULT NULL, " //NON-NLS + " analyzed integer DEFAULT 0)"; //NON-NLS stmt.execute(sql); } catch (SQLException ex) { @@ -588,7 +588,7 @@ public final class DrawableDB { try { String sql = "CREATE TABLE if not exists hash_sets " //NON-NLS + "( hash_set_id INTEGER primary key," //NON-NLS - + " hash_set_name VARCHAR(255) UNIQUE NOT NULL)"; //NON-NLS + + " hash_set_name TEXT UNIQUE NOT NULL)"; //NON-NLS stmt.execute(sql); } catch (SQLException ex) { logger.log(Level.SEVERE, "Failed to create hash_sets table", ex); //NON-NLS @@ -692,8 +692,8 @@ public final class DrawableDB { String tableSchema = "( group_id " + autogenKeyType + " PRIMARY KEY, " //NON-NLS + " data_source_obj_id BIGINT DEFAULT 0, " - + " value VARCHAR(255) not null, " //NON-NLS - + " attribute VARCHAR(255) not null, " //NON-NLS + + " value TEXT not null, " //NON-NLS + + " attribute TEXT not null, " //NON-NLS + " is_analyzed integer DEFAULT 0, " + " UNIQUE(data_source_obj_id, value, attribute) )"; //NON-NLS From 6a64791fe589c48abf1748ab032880ce8a57091e Mon Sep 17 00:00:00 2001 From: Ann Priestman Date: Wed, 11 Sep 2019 21:31:34 -0400 Subject: [PATCH 080/102] Remove legacy artifact ID from tsk_event_descriptions --- test/script/tskdbdiff.py | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/test/script/tskdbdiff.py b/test/script/tskdbdiff.py index a39bb9fc16..5518c97512 100644 --- a/test/script/tskdbdiff.py +++ b/test/script/tskdbdiff.py @@ -322,6 +322,7 @@ class TskDbDiff(object): id_fs_info_table = build_id_fs_info_table(conn.cursor(), isMultiUser) id_objects_table = build_id_objects_table(conn.cursor(), isMultiUser) id_artifact_types_table = build_id_artifact_types_table(conn.cursor(), isMultiUser) + id_legacy_artifact_types = build_id_legacy_artifact_types_table(conn.cursor(), isMultiUser) id_reports_table = build_id_reports_table(conn.cursor(), isMultiUser) id_images_table = build_id_image_names_table(conn.cursor(), isMultiUser) id_obj_path_table = build_id_obj_path_table(id_files_table, id_objects_table, id_artifact_types_table, id_reports_table, id_images_table) @@ -347,7 +348,7 @@ class TskDbDiff(object): if 'INSERT INTO image_gallery_groups_seen' in dump_line: dump_line = '' continue; - dump_line = normalize_db_entry(dump_line, id_obj_path_table, id_vs_parts_table, id_vs_info_table, id_fs_info_table, id_objects_table, id_reports_table, id_images_table) + dump_line = normalize_db_entry(dump_line, id_obj_path_table, id_vs_parts_table, id_vs_info_table, id_fs_info_table, id_objects_table, id_reports_table, id_images_table, id_legacy_artifact_types) db_log.write('%s\n' % dump_line) dump_line = '' postgreSQL_db.close() @@ -361,7 +362,7 @@ class TskDbDiff(object): for line in conn.iterdump(): if 'INSERT INTO "image_gallery_groups_seen"' in line: continue - line = normalize_db_entry(line, id_obj_path_table, id_vs_parts_table, id_vs_info_table, id_fs_info_table, id_objects_table, id_reports_table, id_images_table) + line = normalize_db_entry(line, id_obj_path_table, id_vs_parts_table, id_vs_info_table, id_fs_info_table, id_objects_table, id_reports_table, id_images_table, id_legacy_artifact_types) db_log.write('%s\n' % line) # Now sort the file srtcmdlst = ["sort", dump_file, "-o", dump_file] @@ -414,7 +415,7 @@ class PGSettings(object): return self.password -def normalize_db_entry(line, files_table, vs_parts_table, vs_info_table, fs_info_table, objects_table, reports_table, images_table): +def normalize_db_entry(line, files_table, vs_parts_table, vs_info_table, fs_info_table, objects_table, reports_table, images_table, artifact_table): """ Make testing more consistent and reasonable by doctoring certain db entries. Args: @@ -591,10 +592,15 @@ def normalize_db_entry(line, files_table, vs_parts_table, vs_info_table, fs_info # replace object ids with information that is deterministic file_obj_id = int(fields_list[5]) object_id = int(fields_list[4]) + legacy_artifact_id = 'NULL' + if (fields_list[6] != 'NULL'): + legacy_artifact_id = int(fields_list[6]) if file_obj_id != 'NULL' and file_obj_id in files_table.keys(): fields_list[5] = files_table[file_obj_id] if object_id != 'NULL' and object_id in files_table.keys(): fields_list[4] = files_table[object_id] + if legacy_artifact_id != 'NULL' and legacy_artifact_id in artifact_table.keys(): + fields_list[6] = artifact_table[legacy_artifact_id] newLine = ('INSERT INTO "tsk_event_descriptions" VALUES(' + ','.join(fields_list[1:]) + ');') # remove report_id return newLine else: @@ -689,6 +695,17 @@ def build_id_artifact_types_table(db_cursor, isPostgreSQL): mapping = dict([(row[0], row[1]) for row in sql_select_execute(db_cursor, isPostgreSQL, "SELECT blackboard_artifacts.artifact_obj_id, blackboard_artifact_types.type_name FROM blackboard_artifacts INNER JOIN blackboard_artifact_types ON blackboard_artifact_types.artifact_type_id = blackboard_artifacts.artifact_type_id ")]) return mapping +def build_id_legacy_artifact_types_table(db_cursor, isPostgreSQL): + """Build the map of legacy artifact ids to artifact type. + + Args: + db_cursor: the database cursor + """ + # for each row in the db, take the legacy artifact id then create a tuple in the dictionary + # with the artifact id as the key and artifact type as the value + mapping = dict([(row[0], row[1]) for row in sql_select_execute(db_cursor, isPostgreSQL, "SELECT blackboard_artifacts.artifact_id, blackboard_artifact_types.type_name FROM blackboard_artifacts INNER JOIN blackboard_artifact_types ON blackboard_artifact_types.artifact_type_id = blackboard_artifacts.artifact_type_id ")]) + return mapping + def build_id_reports_table(db_cursor, isPostgreSQL): """Build the map of report object ids to report path. From e4bc470849b537c5dda56e438dd4c543f96bc96a Mon Sep 17 00:00:00 2001 From: Joe Ho Date: Thu, 12 Sep 2019 10:04:14 -0400 Subject: [PATCH 081/102] Fix PR comments --- .../autopsy/logicalimager/dsp/AddLogicalImageTask.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java index 7cce2416f9..bd9c836e88 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java @@ -295,7 +295,7 @@ final class AddLogicalImageTask implements Runnable { @Messages({ "# {0} - line number", "# {1} - fields length", "# {2} - expected length", "AddLogicalImageTask.notEnoughFields=File does not contain enough fields at line {0}, got {1}, expecting {2}", "# {0} - target image path", "AddLogicalImageTask.cannotFindDataSourceObjId=Cannot find obj_id in tsk_image_names for {0}", - "# {0} - file number", "# {1} - total files", "AddLogicalImageTask.addingInterestingFile=Adding interesting file ({0}/{1})" + "# {0} - file number", "# {1} - total files", "AddLogicalImageTask.addingInterestingFile=Adding interesting files ({0}/{1})" }) private void addInterestingFiles(Path resultsPath, boolean createVHD) throws IOException, TskCoreException { Map> objIdToimagePathsMap = currentCase.getSleuthkitCase().getImagePaths(); @@ -361,7 +361,7 @@ final class AddLogicalImageTask implements Runnable { } @Messages({ - "# {0} - file number", "# {1} - total files", "AddLogicalImageTask.addingExtractedFile=Adding extracted file ({0}/{1})" + "# {0} - file number", "# {1} - total files", "AddLogicalImageTask.addingExtractedFile=Adding extracted files ({0}/{1})" }) private void addExtractedFiles(File src, Path resultsPath, List newDataSources) throws TskCoreException, IOException { SleuthkitCase skCase = Case.getCurrentCase().getSleuthkitCase(); From eb2660ec77d1cfb6db3f3be5e02ec4ddc3300823 Mon Sep 17 00:00:00 2001 From: Ann Priestman Date: Thu, 12 Sep 2019 10:23:04 -0400 Subject: [PATCH 082/102] Added image tags to portable case --- .../autopsy/report/Bundle.properties-MERGED | 3 + .../report/PortableCaseReportModule.java | 107 +++++++++++++++++- 2 files changed, 106 insertions(+), 4 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/report/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/report/Bundle.properties-MERGED index 53aa1acfbd..8978a1baf1 100755 --- a/Core/src/org/sleuthkit/autopsy/report/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/report/Bundle.properties-MERGED @@ -31,6 +31,9 @@ PortableCaseReportModule.generateReport.errorCopyingFiles=Error copying tagged f PortableCaseReportModule.generateReport.errorCopyingInterestingFiles=Error copying interesting files PortableCaseReportModule.generateReport.errorCopyingInterestingResults=Error copying interesting results PortableCaseReportModule.generateReport.errorCopyingTags=Error copying tags +PortableCaseReportModule.generateReport.errorCreatingImageTagTable=Error creating image tags table +PortableCaseReportModule.generateReport.errorCreatingReportFolder=Could not make report folder +PortableCaseReportModule.generateReport.errorGeneratingUCOreport=Problem while generating CASE-UCO report # {0} - attribute type name PortableCaseReportModule.generateReport.errorLookingUpAttrType=Error looking up attribute type {0} PortableCaseReportModule.generateReport.interestingItemError=Error loading intersting items diff --git a/Core/src/org/sleuthkit/autopsy/report/PortableCaseReportModule.java b/Core/src/org/sleuthkit/autopsy/report/PortableCaseReportModule.java index 3c712c067f..6cfad5aade 100644 --- a/Core/src/org/sleuthkit/autopsy/report/PortableCaseReportModule.java +++ b/Core/src/org/sleuthkit/autopsy/report/PortableCaseReportModule.java @@ -36,6 +36,7 @@ import org.openide.modules.InstalledFileLocator; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; +import org.sleuthkit.autopsy.casemodule.services.contentviewertags.ContentViewerTagManager; import org.sleuthkit.autopsy.coreutils.FileUtil; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; @@ -179,9 +180,12 @@ class PortableCaseReportModule implements ReportModule { "PortableCaseReportModule.generateReport.errorCopyingArtifacts=Error copying tagged artifacts", "PortableCaseReportModule.generateReport.errorCopyingInterestingFiles=Error copying interesting files", "PortableCaseReportModule.generateReport.errorCopyingInterestingResults=Error copying interesting results", + "PortableCaseReportModule.generateReport.errorCreatingImageTagTable=Error creating image tags table", "# {0} - attribute type name", "PortableCaseReportModule.generateReport.errorLookingUpAttrType=Error looking up attribute type {0}", "PortableCaseReportModule.generateReport.compressingCase=Compressing case...", + "PortableCaseReportModule.generateReport.errorCreatingReportFolder=Could not make report folder", + "PortableCaseReportModule.generateReport.errorGeneratingUCOreport=Problem while generating CASE-UCO report" }) void generateReport(String reportPath, PortableCaseOptions options, ReportProgressPanel progressPanel) { @@ -240,6 +244,14 @@ class PortableCaseReportModule implements ReportModule { return; } + // Set up the table for the image tags + try { + initializeImageTags(progressPanel); + } catch (TskCoreException ex) { + handleError("Error creating image tag table", Bundle.PortableCaseReportModule_generateReport_errorCreatingImageTagTable(), ex, progressPanel); // NON-NLS + return; + } + // Copy the selected tags progressPanel.updateStatusLabel(Bundle.PortableCaseReportModule_generateReport_copyingTags()); try { @@ -358,7 +370,7 @@ class PortableCaseReportModule implements ReportModule { File reportsFolder = Paths.get(caseFolder.toString(), "Reports").toFile(); if(!reportsFolder.mkdir()) { - handleError("Could not make report folder", "Could not make report folder", null, progressPanel); // NON-NLS + handleError("Could not make report folder", Bundle.PortableCaseReportModule_generateReport_errorCreatingReportFolder(), null, progressPanel); // NON-NLS return; } @@ -366,7 +378,7 @@ class PortableCaseReportModule implements ReportModule { CaseUcoFormatExporter.export(tagNames, setNames, reportsFolder, progressPanel); } catch (IOException | SQLException | NoCurrentCaseException | TskCoreException ex) { handleError("Problem while generating CASE-UCO report", - "Problem while generating CASE-UCO report", ex, progressPanel); // NON-NLS + Bundle.PortableCaseReportModule_generateReport_errorGeneratingUCOreport(), ex, progressPanel); // NON-NLS } // Compress the case (if desired) @@ -484,6 +496,22 @@ class PortableCaseReportModule implements ReportModule { currentCaseDbManager.select("max(examiner_id) as max_id from tsk_examiners", new StoreMaxIdCallback("tsk_examiners")); // NON-NLS } + /** + * Set up the image tag table in the portable case + * + * @param progressPanel + * + * @throws TskCoreException + */ + private void initializeImageTags(ReportProgressPanel progressPanel) throws TskCoreException { + + // Create the image tags table in the portable case + CaseDbAccessManager portableDbAccessManager = portableSkCase.getCaseDbAccessManager(); + if (! portableDbAccessManager.tableExists(ContentViewerTagManager.TABLE_NAME)) { + portableDbAccessManager.createTable(ContentViewerTagManager.TABLE_NAME, ContentViewerTagManager.TABLE_SCHEMA_SQLITE); + } + } + /** * Add all files with a given tag to the portable case. * @@ -496,7 +524,7 @@ class PortableCaseReportModule implements ReportModule { // Get all the tags in the current case List tags = currentCase.getServices().getTagsManager().getContentTagsByTagName(oldTagName); - + // Copy the files into the portable case and tag for (ContentTag tag : tags) { @@ -507,17 +535,88 @@ class PortableCaseReportModule implements ReportModule { Content content = tag.getContent(); if (content instanceof AbstractFile) { + + // Get the image tag data associated with this tag (empty string if there is none) + String appData = getImageTagDataForContentTag(tag); + long newFileId = copyContentToPortableCase(content, progressPanel); // Tag the file if (! oldTagNameToNewTagName.containsKey(tag.getName())) { throw new TskCoreException("TagName map is missing entry for ID " + tag.getName().getId() + " with display name " + tag.getName().getDisplayName()); // NON-NLS } - portableSkCase.addContentTag(newIdToContent.get(newFileId), oldTagNameToNewTagName.get(tag.getName()), tag.getComment(), tag.getBeginByteOffset(), tag.getEndByteOffset()); + ContentTag newContentTag = portableSkCase.addContentTag(newIdToContent.get(newFileId), oldTagNameToNewTagName.get(tag.getName()), tag.getComment(), tag.getBeginByteOffset(), tag.getEndByteOffset()); + if (! appData.isEmpty()) { + addImageTagToPortableCase(newContentTag, appData); + } } } } + /** + * Gets the image tag data for a given content tag + * + * @param tag The ContentTag in the current case + * + * @return The app_data string for this content tag or an empty string if there was none + * + * @throws TskCoreException + */ + private String getImageTagDataForContentTag(ContentTag tag) throws TskCoreException { + + GetImageTagCallback callback = new GetImageTagCallback(); + String query = "* FROM " + ContentViewerTagManager.TABLE_NAME + " WHERE content_tag_id = " + tag.getId(); + currentCase.getSleuthkitCase().getCaseDbAccessManager().select(query, callback); + return callback.getAppData(); + } + + /** + * CaseDbAccessManager callback to get the app_data string for the image tag + */ + private static class GetImageTagCallback implements CaseDbAccessManager.CaseDbAccessQueryCallback { + + private static final Logger logger = Logger.getLogger(PortableCaseReportModule.class.getName()); + private String appData = ""; + + @Override + public void process(ResultSet rs) { + try { + while (rs.next()) { + try { + appData = rs.getString("app_data"); // NON-NLS + } catch (SQLException ex) { + logger.log(Level.WARNING, "Unable to get app_data from result set", ex); // NON-NLS + } + } + } catch (SQLException ex) { + logger.log(Level.WARNING, "Failed to get next result for app_data", ex); // NON-NLS + } + } + + /** + * Get the app_data string + * + * @return the app_data string + */ + String getAppData() { + return appData; + } + } + + /** + * Add an image tag to the portable case. + * + * @param newContentTag The content tag in the portable case + * @param appData The string to copy into app_data + * + * @throws TskCoreException + */ + private void addImageTagToPortableCase(ContentTag newContentTag, String appData) throws TskCoreException { + String insert = "(content_tag_id, app_data) VALUES (" + newContentTag.getId() + ", '" + appData + "')"; + portableSkCase.getCaseDbAccessManager().insert(ContentViewerTagManager.TABLE_NAME, insert); + } + + /** * Add all artifacts with a given tag to the portable case. * From 1b4a70bdeca1cdb74c0e980aae3b45d2f5ecea8d Mon Sep 17 00:00:00 2001 From: Kelly Kelly Date: Thu, 12 Sep 2019 11:26:20 -0400 Subject: [PATCH 083/102] Fixed codacy issues --- .../recentactivity/ExtractRegistry.java | 24 ++++++++----------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java index 3a71680207..6e4d0fc4c7 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java @@ -67,8 +67,6 @@ import static java.util.TimeZone.getTimeZone; import org.openide.util.Lookup; import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProgress; import org.sleuthkit.autopsy.ingest.IngestModule.IngestModuleException; -import org.sleuthkit.autopsy.ingest.IngestServices; -import org.sleuthkit.autopsy.ingest.ModuleDataEvent; import org.sleuthkit.autopsy.keywordsearchservice.KeywordSearchService; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.BlackboardArtifact; @@ -876,7 +874,7 @@ class ExtractRegistry extends Extract { String userInfoSection = "User Information"; String previousLine = null; String line = bufferedReader.readLine(); - Set> userSet = new HashSet<>(); + Set> userSet = new HashSet<>(); Map> groupMap = null; while (line != null) { if (line.contains(SECTION_DIVIDER) && previousLine != null && previousLine.contains(userInfoSection)) { @@ -890,9 +888,9 @@ class ExtractRegistry extends Extract { previousLine = line; line = bufferedReader.readLine(); } - Map> userInfoMap = new HashMap<>(); + Map> userInfoMap = new HashMap<>(); //load all the user info which was read into a map - for (HashMap userInfo : userSet) { + for (Map userInfo : userSet) { userInfoMap.put(userInfo.get(SID_KEY), userInfo); } //get all existing OS account artifacts @@ -903,7 +901,7 @@ class ExtractRegistry extends Extract { BlackboardAttribute existingUserId = osAccount.getAttribute(new BlackboardAttribute.Type(ATTRIBUTE_TYPE.TSK_USER_ID)); if (existingUserId != null) { String userID = existingUserId.getValueString().trim(); - HashMap userInfo = userInfoMap.remove(userID); + Map userInfo = userInfoMap.remove(userID); //if the existing user id matches a user id which we parsed information for check if that information exists and if it doesn't add it if (userInfo != null) { osAccount.addAttributes(getAttributesForAccount(userInfo, groupMap.get(userID), true)); @@ -912,7 +910,7 @@ class ExtractRegistry extends Extract { } } //add remaining userinfos as accounts; - for (HashMap userInfo : userInfoMap.values()) { + for (Map userInfo : userInfoMap.values()) { BlackboardArtifact bbart = regAbstractFile.newArtifact(ARTIFACT_TYPE.TSK_OS_ACCOUNT); bbart.addAttributes(getAttributesForAccount(userInfo, groupMap.get(userInfo.get(SID_KEY)), false)); // index the artifact for keyword search @@ -943,7 +941,7 @@ class ExtractRegistry extends Extract { * * @throws ParseException */ - Collection getAttributesForAccount(HashMap userInfo, List groupList, boolean existingUser) throws ParseException { + Collection getAttributesForAccount(Map userInfo, List groupList, boolean existingUser) throws ParseException { Collection bbattributes = new ArrayList<>(); SimpleDateFormat regRipperTimeFormat = new SimpleDateFormat("EEE MMM dd HH:mm:ss yyyy 'Z'"); @@ -1062,8 +1060,8 @@ class ExtractRegistry extends Extract { getRAModuleName(), settingString)); } - if (groupList != null && groupList.size() > 0) { - String groups = new String(); + if (groupList != null && groupList.isEmpty()) { + String groups = ""; for (String group : groupList) { groups += group + ", "; } @@ -1086,7 +1084,7 @@ class ExtractRegistry extends Extract { * * @throws IOException */ - private void readUsers(BufferedReader bufferedReader, Set> users) throws IOException { + private void readUsers(BufferedReader bufferedReader, Set> users) throws IOException { String line = bufferedReader.readLine(); //read until end of file or next section divider String userName = ""; @@ -1131,7 +1129,7 @@ class ExtractRegistry extends Extract { * @throws IOException */ Map> readGroups(BufferedReader bufferedReader) throws IOException { - HashMap> groupMap = new HashMap<>(); + Map> groupMap = new HashMap<>(); String line = bufferedReader.readLine(); @@ -1193,8 +1191,6 @@ class ExtractRegistry extends Extract { value = ""; } - return new AbstractMap.SimpleEntry<>(key, value); - } else if (line.contains("-->")) { key = line.replace("-->", "").trim(); value = "true"; From d92943581d0466a8eeab399952407749e0fd6ce7 Mon Sep 17 00:00:00 2001 From: Joe Ho Date: Thu, 12 Sep 2019 11:54:07 -0400 Subject: [PATCH 084/102] Fix Codacy error --- .../sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java | 1 - 1 file changed, 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java index e8c11492a6..33fd3de19e 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java @@ -73,7 +73,6 @@ final class AddLogicalImageTask implements Runnable { private volatile boolean cancelled; private boolean addingInterestingFiles; private AddMultipleImageTask addMultipleImageTask; - private Thread multipleImageThread; private boolean createVHD; AddLogicalImageTask(String deviceId, From 6aa6d66d60a6d774d64b66276133bd934e1ffad1 Mon Sep 17 00:00:00 2001 From: Raman Date: Thu, 12 Sep 2019 11:55:10 -0400 Subject: [PATCH 085/102] Address log levels. --- InternalPythonModules/android/imo.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/InternalPythonModules/android/imo.py b/InternalPythonModules/android/imo.py index 9b3ec78528..e089248e5f 100644 --- a/InternalPythonModules/android/imo.py +++ b/InternalPythonModules/android/imo.py @@ -133,9 +133,9 @@ class IMOAnalyzer(general.AndroidComponentAnalyzer): except SQLException as ex: - self._logger.log(Level.SEVERE, "Error processing query result for IMO friends", ex) + self._logger.log(Level.WARNING, "Error processing query result for IMO friends", ex) except TskCoreException as ex: - self._logger.log(Level.SEVERE, "Failed to create AppDBParserHelper for adding artifacts.", ex) + self._logger.log(Level.WARNING, "Failed to create AppDBParserHelper for adding artifacts.", ex) finally: friendsDb.close() From 4362b9b0edb318a8f0a91b955d13cb4d8014fcb7 Mon Sep 17 00:00:00 2001 From: Joe Ho Date: Thu, 12 Sep 2019 11:57:36 -0400 Subject: [PATCH 086/102] revert DataResultViewerTable.java --- .../corecomponents/DataResultViewerTable.java | 38 +++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/corecomponents/DataResultViewerTable.java b/Core/src/org/sleuthkit/autopsy/corecomponents/DataResultViewerTable.java index 50aef1a14e..71f801282b 100644 --- a/Core/src/org/sleuthkit/autopsy/corecomponents/DataResultViewerTable.java +++ b/Core/src/org/sleuthkit/autopsy/corecomponents/DataResultViewerTable.java @@ -434,7 +434,7 @@ public class DataResultViewerTable extends AbstractDataResultViewer { } setColumnWidths(); - + /* * Load column sorting information from preferences file and apply it to * columns. @@ -516,7 +516,7 @@ public class DataResultViewerTable extends AbstractDataResultViewer { protected void setColumnWidths() { if (rootNode.getChildren().getNodesCount() != 0) { final Graphics graphics = outlineView.getGraphics(); - + if (graphics != null) { // Current width of the outlineView double outlineViewWidth = outlineView.getSize().getWidth(); @@ -526,10 +526,10 @@ public class DataResultViewerTable extends AbstractDataResultViewer { int margin = 4; int padding = 8; - + int totalColumnWidth = 0; - int cntMaxSizeColumns = 0; - + int cntMaxSizeColumns =0; + // Calulate the width for each column keeping track of the number // of columns that were set to columnwidthLimit. for (int column = 0; column < outline.getModel().getColumnCount(); column++) { @@ -552,40 +552,40 @@ public class DataResultViewerTable extends AbstractDataResultViewer { columnWidth = Math.min(columnWidth, columnWidthLimit); columnWidths.add(columnWidth); - + totalColumnWidth += columnWidth; - - if (columnWidth == columnWidthLimit) { + + if( columnWidth == columnWidthLimit) { cntMaxSizeColumns++; } } - + // Figure out how much extra, if any can be given to the columns // so that the table is as wide as outlineViewWidth. If cntMaxSizeColumns // is greater than 0 divide the extra space between the columns // that could use more space. Otherwise divide evenly amoung // all columns. int extraWidth = 0; - + if (totalColumnWidth < outlineViewWidth) { - if (cntMaxSizeColumns > 0) { - extraWidth = (int) ((outlineViewWidth - totalColumnWidth) / cntMaxSizeColumns); + if (cntMaxSizeColumns > 0) { + extraWidth = (int) ((outlineViewWidth - totalColumnWidth)/cntMaxSizeColumns); } else { - extraWidth = (int) ((outlineViewWidth - totalColumnWidth) / columnWidths.size()); + extraWidth = (int) ((outlineViewWidth - totalColumnWidth)/columnWidths.size()); } } - - for (int column = 0; column < columnWidths.size(); column++) { + + for(int column = 0; column < columnWidths.size(); column++) { int columnWidth = columnWidths.get(column); - - if (cntMaxSizeColumns > 0) { - if (columnWidth >= ((column == 0) ? 350 : 300)) { + + if(cntMaxSizeColumns > 0) { + if(columnWidth >= ((column == 0) ? 350 : 300)) { columnWidth += extraWidth; } } else { columnWidth += extraWidth; } - + outline.getColumnModel().getColumn(column).setPreferredWidth(columnWidth); } } From 55976bb56cbecd1ea17c3f527261d2c3a238357a Mon Sep 17 00:00:00 2001 From: Kelly Kelly Date: Thu, 12 Sep 2019 12:04:34 -0400 Subject: [PATCH 087/102] fixed codacy issues --- .../org/sleuthkit/autopsy/thunderbirdparser/EMLParser.java | 4 +--- .../autopsy/thunderbirdparser/MimeJ4MessageParser.java | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/EMLParser.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/EMLParser.java index f7f456dd50..14a2d73993 100755 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/EMLParser.java +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/EMLParser.java @@ -32,8 +32,6 @@ import org.sleuthkit.datamodel.ReadContentInputStream; */ class EMLParser extends MimeJ4MessageParser { - private static final Logger logger = Logger.getLogger(EMLParser.class.getName()); - /** * If the extention of the AbstractFile is eml and 'To:' is found close to * the beginning of the file, then its probably an eml file. @@ -45,7 +43,7 @@ class EMLParser extends MimeJ4MessageParser { */ static boolean isEMLFile(AbstractFile abFile, byte[] buffer) { String ext = abFile.getNameExtension(); - boolean isEMLFile = ext != null ? ext.equals("eml") : false; + boolean isEMLFile = ext != null && ext.equals("eml"); if (isEMLFile) { isEMLFile = (new String(buffer)).contains("To:"); //NON-NLS } diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/MimeJ4MessageParser.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/MimeJ4MessageParser.java index 14482008a9..1114d71eba 100755 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/MimeJ4MessageParser.java +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/MimeJ4MessageParser.java @@ -50,7 +50,7 @@ import org.sleuthkit.datamodel.TskData; /** * Super class for email parsers that can use the james.mime4J.Message objects. */ -abstract class MimeJ4MessageParser { +class MimeJ4MessageParser { private static final Logger logger = Logger.getLogger(MimeJ4MessageParser.class.getName()); From b5301e689a8b6e683f9453af774db97f59c330c5 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Thu, 12 Sep 2019 15:14:24 -0400 Subject: [PATCH 088/102] Fix incorrect use of Autopsy Logger in RegressionTest class --- .../src/org/sleuthkit/autopsy/testing/RegressionTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Testing/test/qa-functional/src/org/sleuthkit/autopsy/testing/RegressionTest.java b/Testing/test/qa-functional/src/org/sleuthkit/autopsy/testing/RegressionTest.java index 0a7feaa46c..8f1ef48a4c 100644 --- a/Testing/test/qa-functional/src/org/sleuthkit/autopsy/testing/RegressionTest.java +++ b/Testing/test/qa-functional/src/org/sleuthkit/autopsy/testing/RegressionTest.java @@ -20,7 +20,7 @@ package org.sleuthkit.autopsy.testing; import java.io.File; import java.io.IOException; -import org.sleuthkit.autopsy.coreutils.Logger; +import java.util.logging.Logger; import junit.framework.Test; import junit.framework.TestCase; import org.netbeans.jemmy.Timeouts; From b08f4a7119d2f2725e5ad98ed612a7a85bbda1ec Mon Sep 17 00:00:00 2001 From: Joe Ho Date: Thu, 12 Sep 2019 15:55:21 -0400 Subject: [PATCH 089/102] Resolve merge conflicts --- .../autopsy/logicalimager/dsp/AddLogicalImageTask.java | 5 +++-- .../autopsy/logicalimager/dsp/Bundle.properties-MERGED | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java index 656ca050a7..df3c0f8d14 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java @@ -75,6 +75,8 @@ final class AddLogicalImageTask implements Runnable { private boolean addingInterestingFiles; private AddMultipleImageTask addMultipleImageTask; private boolean createVHD; + private long totalFiles; + private Map imagePathToObjIdMap; AddLogicalImageTask(String deviceId, String timeZone, @@ -181,9 +183,8 @@ final class AddLogicalImageTask implements Runnable { return; } - AddMultipleImageTask addMultipleImageTask = null; + AddDataSourceCallback privateCallback = null; List newDataSources = new ArrayList<>(); - boolean createVHD; if (imagePaths.isEmpty()) { createVHD = false; diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/Bundle.properties-MERGED index 6e15648a88..ebe9de0684 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/Bundle.properties-MERGED @@ -4,11 +4,11 @@ # {0} - file number # {1} - total files -AddLogicalImageTask.addingExtractedFile=Adding extracted file ({0}/{1}) +AddLogicalImageTask.addingExtractedFile=Adding extracted files ({0}/{1}) AddLogicalImageTask.addingExtractedFiles=Adding extracted files # {0} - file number # {1} - total files -AddLogicalImageTask.addingInterestingFile=Adding interesting file ({0}/{1}) +AddLogicalImageTask.addingInterestingFile=Adding interesting files ({0}/{1}) AddLogicalImageTask.addingInterestingFiles=Adding search results as interesting files # {0} - file AddLogicalImageTask.addingToReport=Adding {0} to report From 7b5ac994a70f03011e328166e9826daefc23aaed Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Thu, 12 Sep 2019 17:53:34 -0400 Subject: [PATCH 090/102] Fix incorrect use of Autopsy Logger in AutopsyTestCases class --- .../src/org/sleuthkit/autopsy/testing/AutopsyTestCases.java | 4 ++-- .../src/org/sleuthkit/autopsy/testing/RegressionTest.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Testing/src/org/sleuthkit/autopsy/testing/AutopsyTestCases.java b/Testing/src/org/sleuthkit/autopsy/testing/AutopsyTestCases.java index dd877b3a11..8ad09e4921 100644 --- a/Testing/src/org/sleuthkit/autopsy/testing/AutopsyTestCases.java +++ b/Testing/src/org/sleuthkit/autopsy/testing/AutopsyTestCases.java @@ -31,8 +31,8 @@ import java.util.ArrayList; import java.util.Date; import java.util.List; import java.util.Random; +import java.util.logging.Logger; import java.util.logging.Level; -import org.sleuthkit.autopsy.coreutils.Logger; import javax.imageio.ImageIO; import javax.swing.JDialog; import javax.swing.text.JTextComponent; @@ -66,7 +66,7 @@ import org.sleuthkit.datamodel.TskData; public class AutopsyTestCases { - private static final Logger logger = Logger.getLogger(AutopsyTestCases.class.getName()); + private static final Logger logger = Logger.getLogger(AutopsyTestCases.class.getName()); // DO NOT USE AUTOPSY LOGGER private long start; /** diff --git a/Testing/test/qa-functional/src/org/sleuthkit/autopsy/testing/RegressionTest.java b/Testing/test/qa-functional/src/org/sleuthkit/autopsy/testing/RegressionTest.java index 8f1ef48a4c..6f6e04d7bd 100644 --- a/Testing/test/qa-functional/src/org/sleuthkit/autopsy/testing/RegressionTest.java +++ b/Testing/test/qa-functional/src/org/sleuthkit/autopsy/testing/RegressionTest.java @@ -40,7 +40,7 @@ import org.netbeans.junit.NbModuleSuite; */ public class RegressionTest extends TestCase { - private static final Logger logger = Logger.getLogger(RegressionTest.class.getName()); + private static final Logger logger = Logger.getLogger(RegressionTest.class.getName()); // DO NOT USE AUTOPSY LOGGER private static final AutopsyTestCases autopsyTests = new AutopsyTestCases(Boolean.parseBoolean(System.getProperty("isMultiUser"))); /** From 9a22b39a9aa3e87faf27a6747c4cc0eb402ff7ff Mon Sep 17 00:00:00 2001 From: Ann Priestman Date: Fri, 13 Sep 2019 09:59:09 -0400 Subject: [PATCH 091/102] Codacy --- .../sleuthkit/autopsy/report/PortableCaseReportModule.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/report/PortableCaseReportModule.java b/Core/src/org/sleuthkit/autopsy/report/PortableCaseReportModule.java index 6cfad5aade..0910b2a2ea 100644 --- a/Core/src/org/sleuthkit/autopsy/report/PortableCaseReportModule.java +++ b/Core/src/org/sleuthkit/autopsy/report/PortableCaseReportModule.java @@ -536,9 +536,6 @@ class PortableCaseReportModule implements ReportModule { Content content = tag.getContent(); if (content instanceof AbstractFile) { - // Get the image tag data associated with this tag (empty string if there is none) - String appData = getImageTagDataForContentTag(tag); - long newFileId = copyContentToPortableCase(content, progressPanel); // Tag the file @@ -546,6 +543,10 @@ class PortableCaseReportModule implements ReportModule { throw new TskCoreException("TagName map is missing entry for ID " + tag.getName().getId() + " with display name " + tag.getName().getDisplayName()); // NON-NLS } ContentTag newContentTag = portableSkCase.addContentTag(newIdToContent.get(newFileId), oldTagNameToNewTagName.get(tag.getName()), tag.getComment(), tag.getBeginByteOffset(), tag.getEndByteOffset()); + + // Get the image tag data associated with this tag (empty string if there is none) + // and save it if present + String appData = getImageTagDataForContentTag(tag); if (! appData.isEmpty()) { addImageTagToPortableCase(newContentTag, appData); } From b0238448515837fb9ab232d6617fbb4aa027f862 Mon Sep 17 00:00:00 2001 From: Brian Carrier Date: Fri, 13 Sep 2019 11:10:01 -0400 Subject: [PATCH 092/102] Update Doxyfile --- docs/doxygen-dev/Doxyfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/doxygen-dev/Doxyfile b/docs/doxygen-dev/Doxyfile index 231f5716fc..b8412f61ef 100755 --- a/docs/doxygen-dev/Doxyfile +++ b/docs/doxygen-dev/Doxyfile @@ -58,7 +58,7 @@ PROJECT_LOGO = # entered, it will be relative to the location where doxygen was started. If # left blank the current directory will be used. -OUTPUT_DIRECTORY = dev-docs +OUTPUT_DIRECTORY = build-docs # If the CREATE_SUBDIRS tag is set to YES then doxygen will create 4096 sub- # directories (in 2 levels) under the output directory of each output format and From 27c4ead7ed0e4670f5c2cceb87ac08d66d491c24 Mon Sep 17 00:00:00 2001 From: Brian Carrier Date: Fri, 13 Sep 2019 11:12:39 -0400 Subject: [PATCH 093/102] Update for new doxygen build folder --- build.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.xml b/build.xml index 6d7a1a0315..1f00a24f79 100644 --- a/build.xml +++ b/build.xml @@ -77,7 +77,7 @@ - + @@ -265,7 +265,7 @@ - + From 9db0cf2b7e6488abfd66bc02c9e9bf9de7f9273c Mon Sep 17 00:00:00 2001 From: Joe Ho Date: Fri, 13 Sep 2019 13:39:54 -0400 Subject: [PATCH 094/102] Fix PR comments. Use thread.join to wait for AddMultipleImageTask thread. --- .../dsp/AddLogicalImageTask.java | 23 ++++---- .../dsp/AddMultipleImageTask.java | 57 +++++++++---------- .../dsp/LogicalImagerDSProcessor.java | 3 +- 3 files changed, 43 insertions(+), 40 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java index df3c0f8d14..16481c26a8 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java @@ -72,9 +72,10 @@ final class AddLogicalImageTask implements Runnable { private final Case currentCase; private volatile boolean cancelled; - private boolean addingInterestingFiles; + private volatile boolean addingInterestingFiles; private AddMultipleImageTask addMultipleImageTask; - private boolean createVHD; + private Thread multipleImageThread; + private volatile boolean createVHD; private long totalFiles; private Map imagePathToObjIdMap; @@ -183,7 +184,8 @@ final class AddLogicalImageTask implements Runnable { return; } - AddDataSourceCallback privateCallback = null; + addMultipleImageTask = null; + AddDataSourceCallback privateCallback = new AddDataSourceCallback(); List newDataSources = new ArrayList<>(); if (imagePaths.isEmpty()) { @@ -213,13 +215,14 @@ final class AddLogicalImageTask implements Runnable { createVHD = true; // ingest the VHDs try { - privateCallback = new AddDataSourceCallback(); addMultipleImageTask = new AddMultipleImageTask(deviceId, imagePaths, timeZone , progressMonitor, privateCallback); - addMultipleImageTask.run(); - if (privateCallback.getResult() == DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS) { - // TODO: Delete destination directory when 5453 (VHD file is not closed upon revert) is fixed. - // bait out - callback.done(privateCallback.getResult(), privateCallback.getErrorMessages(), privateCallback.getNewDataSources()); + multipleImageThread = new Thread(addMultipleImageTask); + multipleImageThread.start(); + try { + multipleImageThread.join(); + } catch (InterruptedException ex) { + LOGGER.log(Level.SEVERE, "Add Image interrupted", ex); // NON-NLS + callback.done(DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS, errorList, emptyDataSources); return; } } catch (NoCurrentCaseException ex) { @@ -235,7 +238,7 @@ final class AddLogicalImageTask implements Runnable { addingInterestingFiles = true; addInterestingFiles(Paths.get(dest.toString(), resultsFilename), createVHD); progressMonitor.setProgressText(Bundle.AddLogicalImageTask_doneAddingInterestingFiles()); - if (addMultipleImageTask != null && privateCallback != null) { + if (createVHD) { callback.done(privateCallback.getResult(), privateCallback.getErrorMessages(), privateCallback.getNewDataSources()); } else { callback.done(DataSourceProcessorCallback.DataSourceProcessorResult.NO_ERRORS, errorList, newDataSources); diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddMultipleImageTask.java b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddMultipleImageTask.java index f5f64665cf..f19330015a 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddMultipleImageTask.java +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddMultipleImageTask.java @@ -213,14 +213,15 @@ class AddMultipleImageTask implements Runnable { tskAddImageProcessStopped = true; if (addImageProcess != null) { try { + /* + * All this does is set a flag that will make the TSK add + * image process exit when the flag is checked between + * processing steps. The state of the flag is not + * accessible, so record it here so that it is known that + * the revert method of the process object needs to be + * called. + */ addImageProcess.stop(); - addImageProcess.revert(); - if (tskAddImageProcessStopped) { - List errorMessages = new ArrayList<>(); - List emptyDataSources = new ArrayList<>(); - errorMessages.add(Bundle.AddMultipleImageTask_cancelled()); - callback.done(DataSourceProcessorResult.CRITICAL_ERRORS, errorMessages, emptyDataSources); - } } catch (TskCoreException ex) { LOGGER.log(Level.SEVERE, "Cancellation: addImagePRocess.stop failed", ex); // NON-NLS } @@ -295,34 +296,32 @@ class AddMultipleImageTask implements Runnable { return; } - if (!tskAddImageProcessStopped) { - /* - * Try to commit the results of the add image process, retrieve the new - * image from the case database, and add it to the list of new data - * sources to be returned via the callback. - */ - try { - long imageId = addImageProcess.commit(); - Image dataSource = currentCase.getSleuthkitCase().getImageById(imageId); - newDataSources.add(dataSource); + /* + * Try to commit the results of the add image process, retrieve the new + * image from the case database, and add it to the list of new data + * sources to be returned via the callback. + */ + try { + long imageId = addImageProcess.commit(); + Image dataSource = currentCase.getSleuthkitCase().getImageById(imageId); + newDataSources.add(dataSource); - /* + /* * Verify the size of the new image. Note that it may not be what is * expected, but at least part of it was added to the case. - */ - String verificationError = dataSource.verifyImageSize(); - if (!verificationError.isEmpty()) { - errorMessages.add(Bundle.AddMultipleImageTask_nonCriticalErrorAdding(imageFilePath, deviceId, verificationError)); - } - } catch (TskCoreException ex) { - /* + */ + String verificationError = dataSource.verifyImageSize(); + if (!verificationError.isEmpty()) { + errorMessages.add(Bundle.AddMultipleImageTask_nonCriticalErrorAdding(imageFilePath, deviceId, verificationError)); + } + } catch (TskCoreException ex) { + /* * The add image process commit failed or querying the case database * for the newly added image failed. Either way, this is a critical * error. - */ - errorMessages.add(Bundle.AddMultipleImageTask_criticalErrorAdding(imageFilePath, deviceId, ex.getLocalizedMessage())); - criticalErrorOccurred = true; - } + */ + errorMessages.add(Bundle.AddMultipleImageTask_criticalErrorAdding(imageFilePath, deviceId, ex.getLocalizedMessage())); + criticalErrorOccurred = true; } } } diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/LogicalImagerDSProcessor.java b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/LogicalImagerDSProcessor.java index 4cd8ff162e..a3a2632374 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/LogicalImagerDSProcessor.java +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/LogicalImagerDSProcessor.java @@ -36,6 +36,7 @@ import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessor; import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorCallback; import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorProgressMonitor; +import org.sleuthkit.autopsy.coreutils.TimeStampUtils; import org.sleuthkit.datamodel.Content; /** @@ -173,7 +174,7 @@ public final class LogicalImagerDSProcessor implements DataSourceProcessor { JOptionPane.YES_NO_OPTION); if (showConfirmDialog == YES_OPTION) { // Get unique dest directory - String uniqueDirectory = imageDirPath.getFileName() + "_" + UUID.randomUUID(); + String uniqueDirectory = imageDirPath.getFileName() + "_" + TimeStampUtils.createTimeStamp(); dest = Paths.get(logicalImagerDir.toString(), uniqueDirectory).toFile(); } else { String msg = Bundle.LogicalImagerDSProcessor_directoryAlreadyExists(dest.toString()); From c9d3b6309f21fe41410610322d8a184f444d1859 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Fri, 13 Sep 2019 13:45:15 -0400 Subject: [PATCH 095/102] 5504 add red x to carved file icon --- .../sleuthkit/autopsy/datamodel/FileNode.java | 2 +- .../autopsy/datamodel/LayoutFileNode.java | 2 +- .../autopsy/datamodel/SlackFileNode.java | 2 +- .../autopsy/images/carved-file-x-icon-16.png | Bin 0 -> 6611 bytes 4 files changed, 3 insertions(+), 3 deletions(-) create mode 100644 Core/src/org/sleuthkit/autopsy/images/carved-file-x-icon-16.png diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/FileNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/FileNode.java index 7b3a55c20b..c194268d9f 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/FileNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/FileNode.java @@ -128,7 +128,7 @@ public class FileNode extends AbstractFsContentNode { private void setIcon(AbstractFile file) { if (file.isDirNameFlagSet(TSK_FS_NAME_FLAG_ENUM.UNALLOC)) { if (file.getType().equals(TSK_DB_FILES_TYPE_ENUM.CARVED)) { - this.setIconBaseWithExtension("org/sleuthkit/autopsy/images/carved-file-icon-16.png"); //NON-NLS + this.setIconBaseWithExtension("org/sleuthkit/autopsy/images/carved-file-x-icon-16.png"); //NON-NLS } else { this.setIconBaseWithExtension("org/sleuthkit/autopsy/images/file-icon-deleted.png"); //NON-NLS } diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/LayoutFileNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/LayoutFileNode.java index 8f7f753db2..c33fdc59c4 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/LayoutFileNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/LayoutFileNode.java @@ -73,7 +73,7 @@ public class LayoutFileNode extends AbstractAbstractFileNode { this.setDisplayName(nameForLayoutFile(lf)); if (lf.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.CARVED)) { - this.setIconBaseWithExtension("org/sleuthkit/autopsy/images/carved-file-icon-16.png"); //NON-NLS + this.setIconBaseWithExtension("org/sleuthkit/autopsy/images/carved-file-x-icon-16.png"); //NON-NLS } else if (lf.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.LAYOUT_FILE)) { if (lf.isDirNameFlagSet(TskData.TSK_FS_NAME_FLAG_ENUM.UNALLOC)) { this.setIconBaseWithExtension("org/sleuthkit/autopsy/images/file-icon-deleted.png"); //NON-NLS diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/SlackFileNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/SlackFileNode.java index 0e7bce56cc..791b586404 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/SlackFileNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/SlackFileNode.java @@ -63,7 +63,7 @@ public class SlackFileNode extends AbstractFsContentNode { // set name, display name, and icon if (file.isDirNameFlagSet(TSK_FS_NAME_FLAG_ENUM.UNALLOC)) { if (file.getType().equals(TSK_DB_FILES_TYPE_ENUM.CARVED)) { - this.setIconBaseWithExtension("org/sleuthkit/autopsy/images/carved-file-icon-16.png"); //NON-NLS + this.setIconBaseWithExtension("org/sleuthkit/autopsy/images/carved-file-x-icon-16.png"); //NON-NLS } else { this.setIconBaseWithExtension("org/sleuthkit/autopsy/images/file-icon-deleted.png"); //NON-NLS } diff --git a/Core/src/org/sleuthkit/autopsy/images/carved-file-x-icon-16.png b/Core/src/org/sleuthkit/autopsy/images/carved-file-x-icon-16.png new file mode 100644 index 0000000000000000000000000000000000000000..75a935bf9e30136c5e4f8707f6facbc5b75d822c GIT binary patch literal 6611 zcmeHLdpy(o|DXFM<<|XjTH}PY%V!(2WrPirMXc!FY@c1qE`2sObkS9gI65kqRC0== zq;jItjnMr>Nx6hXNs3A?Mf^S+r}X=F`hI&nzTf}CZ130m`Fg(J@8|3C-WKgU-($4l zWJ3f3F`D5?^MjvFR1bY!`2Y0IK{f)RQxnb#l>4z2NGT-Y34=kTJWL8AL8Xv~KqzaD z2dshKSsV42*>vmcYnWYYT%1R$YLaxGx zYMIr|+Z(9GE&Y@FQi~I{>yIS%SKHz@9Of9ynYNdjTdQm?RJ{*va-z}NyI&r#xXCSb zTIutC%I~cc*Oy`X1o1h_nITMYPx(pY{tE^4N1F#DbF=3(F4`y3Sv}exeRVv`jNBX^ zpXr{g+troBGEG<18Pwldc+u=oq-DCesvkS{=1l498Cxe>3Or|c z#5rxB_==-z;s9Jxp6bUzqT$+05qKXhM8y>3usgsZ+ZIL?K+yM<*|QThDBs z$+IWh``ScV7i(wUr5Wjen2#|kI2D2H`B|)6{n19;J`Liz_E^Oe%nPiVotnCJ^Y$h_ zdaEhOacv%Jcq4FwVE6tfWq4Z0*p2V(ou3y@o0M8=R(<#NCdYhpn@7I5u|#lx{<+GJ zZ%2owbkriQEH={dejXcuJdpleaO)@P?V{Nca-hPXD^bEZ^?qZe`{Y+a*=combh^*Z zEifPZTf+UsO3v{aInE|~$(P#BG;W=f3T-oQin|L~6^|;<7POstPqNOiO}8~tuq$UA zw4GX-atm^_NKMe~+8sLTL|$JuPj`2;Zq%Fu+6T@Y+|KRqiw`dFJMuJrS4NH~dD@jj z$5>ss+Bxl~g4;Q@VY7FvJ#o3_!|dkvbH_TUIZKWeyV&g)i;kFRfXH<-S2T^G&)aqHK+_{bzgqLG z(8G@se|q|2-^v>|I?sL_px%5QuqzZZidScv+qSgI3yrvb1`;mXRLw`%Dcy@uab(1TSI;M$M`juSJ0pR8NWW=E^fE{?T+1< zJBpv@_)Wg#wM;qju-)iI=W6ByanPa~_WZiGX|`i*&Low+VYMv!^Zk{KYyHvp8#*)- zoTtY=3ma_Qo39u7m#0O=e4no3hO=Y-DAILR&fS{ts$1cI<=kYwmcLA5sJNDjj}eO; z>z|o3#bqC8>C&tD7KQ$`F`?pY^n<#ax0iRLm^wmy+YCl=Hk~B7Y+T;I(y3wQ9LzJb zmf|33%{#xfCX|rFWo>&t*v76XjGR#x6R^4}B-QSuHtE+5mwLRrly+HnLr1kPkh$!z zXz~o`&RlYYX5--!o|&ape%h^@;kAwsLCU4d=DeT2z(dC=oSS2n^JZyI3hD5AyH@h; z#}75)R$p7{>cnXoZ#>TXwbo_p4~u`&$+~9T8*zW#1D|a-$LSYUYiVB5BgJ9zjDDFi zSkYO1KIvF72Fjo0nY7|&VxeZ)(px4WGq=y$&{WvpUecS}5es_b)HS4abOz} zfza3{baV4%xVa&H{e0XR-gLOPr>@^c^(>w{wRENbky&U|sEO$f-~F?9c&*(%YkZNb z(Dw9;RZ!~$qR`WC*;Ji}o3~7lS*SOGzkJ#<9hcWyC8d>&yL7MqmHqq=C zE^2KJPtM{!v`kyQch^pOsdwnYcNfzqXc(;;YZ{##?wFUfy{)$(D(ZxSecdn-O(J{&`4!OeC)Sl=q+155CEhXl<%j0`@wdEE|Qb#ezD|zLfRxQz6N`ek; zc{+wicl-Ikj4?Jx-QT-jWpC>o&(B*19*=lC;e)Kk>z6j7he@&76VJI@ch2&%E_O{? z6g$?OQS)m-Z`8g4?Q>jcPvP}bk%lZ!rgQJ~{`LpAeSFA_0qgRQLABF4k&+35`@wyE zRryn_B6}vCy;n2fH7)6*<;4dNTs`Iv+Dq5gJ!qyrhV&+e-}~8ZwbzOiS102O2k(-= zO+`!7B!9IY&-=AP56N7PZb0Zo6)HF7K3ra3S5aMinZM-YM?}FLsKmP3%NxFeQ9}6I z4P<&#I1&+t&6Na!7^O%GU&{!Dqq9=V=Bxqb$RLm}6jMLM{~*V8_HUrEZ`= z=ot=y{^9dkobWXqG8g6SWay}*zyu;t&PFOl!D1OjNkyr7De!BR8H+-yA@Vg;R3OtA z=_Y|dBoRZz;LvoXFcgn+GDJE;Tpq=b<~~dTkEketTrQ zJ@oT>U-gi&;CDY*KTswKfjA&N6co!XMlLFo`+=V|`41>y_!k7y;1Jo!O^3)pl}UZ@ zU@i{}pGv*;k&AjTn7*HURB_-7MN+kg3Oy3Z<$S_PL!e+ahReZ%!Jr5}i40~2zQDu& zpB%n=7&WukTO}1>XYW8D+v6Mn5Y zmqT%fIoWX12}Nu^h?R=@>IqmsiW>y7u9p#~HzR7-_Ks*9e&=BQMpqGaq)kgE;> ztmDuWN5B^ILAbjPSH|CV;a3vYp6dYOKpcQ}u;&rcL;xV7gLqs!G=adeBaz4iGMD7= zCA&<*lPlN|=)#9f2aYpbx#~D0t%vi>=1aUn0IHG*2awS?G8!PT00Mw|5(KU{hL(`s|EKm%GXpG>H+}m zAN_|DRn_Py3{@fche)NtLJ;~get+llKj0ic+lOoC-?@*34SBmsq+xIu5y*WM;%_eh z72ptqw~zyhWs+}geI#TkTqCYSc+W#N_?m@pZS1FOdpLVl1^z$$8Lo=|;R-PIk43&q z-yd@Qkn6h?_%85|?D`?scPa2);2+ubUz5x5^9O%W48I>$z#r?6(+u~*UkT_2&GVok z235al7c&!Ji-FX0u?&I0Sg0Nv$|KI9uu)IWVAA!P3`U!xY?h{oZQ(C|&NFB(EQecr z5{l+kuqR$~?rS{~v-vruzDI9|vZL9gB0^CYXC9eG~ zYSG=pX*F!xRU|H?VtPt`c!MPB;6Pz@>V`)0g{Y2}uuC0%7d)R_n7nUiTT#VU%SMFf z(ZHvZ&kfQZeh3GyYAPfDd=YASv@HPJX?J;y@^wvTlB9cOW8bgLUzSeM){MGn8!J9u zx+DhJo>k4Cbv7b=5M;*Kb`UA16L2pZT+YSiPA@WS$Gq=C)h?Z~u6H;1+Pwk&c+A{a z&u;IWT-=B#O5@{ogMh3f+dPuXrtThqvMg@LG;78+O(kfpTC@7_GoDT2dBO2gOEa_U zr_Z$Uiz=@zTk*R&pivwZvhLoK@_6msL-9M?f@*6leT70s^2?O*0rfBHn-(wJIm@87 zqRFM8;6nTH<2SjNTf?BmoP@P|6iyn5lZWCJ+I!L$TwK@njMZsS*dedcduf$0;J-hA ziqqrvVw*h~nd>7(f^sosc~?Sj#AAmscOFeWdaNm6*0#~Q=gO;Z-05G}b#P|%{kBTy z{7WnPnwkcxB6jTiq-nm8?g|MEP_FwPs?##wfX-5 D>kTha literal 0 HcmV?d00001 From b8e9845f67c53dab659ba7b31e4f515d30205688 Mon Sep 17 00:00:00 2001 From: Joe Ho Date: Fri, 13 Sep 2019 19:09:07 -0400 Subject: [PATCH 096/102] Fix everything mentioned during Friday's conference call --- .../dsp/AddLogicalImageTask.java | 106 +++++++----------- ...geTask.java => AddMultipleImagesTask.java} | 104 ++++++++++------- .../dsp/Bundle.properties-MERGED | 17 +-- .../dsp/LogicalImagerDSProcessor.java | 1 - .../logicalimager/dsp/LogicalImagerPanel.java | 2 +- 5 files changed, 114 insertions(+), 116 deletions(-) rename Core/src/org/sleuthkit/autopsy/logicalimager/dsp/{AddMultipleImageTask.java => AddMultipleImagesTask.java} (77%) diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java index 16481c26a8..982ede8493 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java @@ -32,6 +32,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.logging.Level; +import javax.annotation.concurrent.GuardedBy; import org.apache.commons.io.FileUtils; import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.casemodule.Case; @@ -72,13 +73,14 @@ final class AddLogicalImageTask implements Runnable { private final Case currentCase; private volatile boolean cancelled; - private volatile boolean addingInterestingFiles; - private AddMultipleImageTask addMultipleImageTask; - private Thread multipleImageThread; private volatile boolean createVHD; private long totalFiles; private Map imagePathToObjIdMap; + private final Object addMultipleImagesLock; + @GuardedBy("addMultipleImagesLock") + private AddMultipleImagesTask addMultipleImagesTask = null; + AddLogicalImageTask(String deviceId, String timeZone, File src, File dest, @@ -93,6 +95,7 @@ final class AddLogicalImageTask implements Runnable { this.callback = callback; this.currentCase = Case.getCurrentCase(); this.blackboard = this.currentCase.getServices().getArtifactsBlackboard(); + this.addMultipleImagesLock = new Object(); } /** @@ -115,7 +118,8 @@ final class AddLogicalImageTask implements Runnable { "# {0} - reason", "AddLogicalImageTask.failedToAddInterestingFiles=Failed to add interesting files: {0}", "AddLogicalImageTask.addingExtractedFiles=Adding extracted files", "AddLogicalImageTask.doneAddingExtractedFiles=Done adding extracted files", - "# {0} - reason", "AddLogicalImageTask.failedToGetTotalFilesCount=Failed to get total files count: {0}" + "# {0} - reason", "AddLogicalImageTask.failedToGetTotalFilesCount=Failed to get total files count: {0}", + "AddLogicalImageTask.addImageCancelled=Add image cancelled" }) @Override public void run() { @@ -132,6 +136,15 @@ final class AddLogicalImageTask implements Runnable { errorList.add(msg); } + if (cancelled) { + // Don't delete destination directory once we started adding interesting files. + // At this point the database and destination directory are complete. + deleteDestinationDirectory(); + errorList.add("Add image cancelled"); + callback.done(DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS, errorList, emptyDataSources); + return; + } + // Add the SearchResults.txt and users.txt to the case report String resultsFilename; if (Paths.get(dest.toString(), SEARCH_RESULTS_TXT).toFile().exists()) { @@ -184,8 +197,6 @@ final class AddLogicalImageTask implements Runnable { return; } - addMultipleImageTask = null; - AddDataSourceCallback privateCallback = new AddDataSourceCallback(); List newDataSources = new ArrayList<>(); if (imagePaths.isEmpty()) { @@ -215,15 +226,14 @@ final class AddLogicalImageTask implements Runnable { createVHD = true; // ingest the VHDs try { - addMultipleImageTask = new AddMultipleImageTask(deviceId, imagePaths, timeZone , progressMonitor, privateCallback); - multipleImageThread = new Thread(addMultipleImageTask); - multipleImageThread.start(); - try { - multipleImageThread.join(); - } catch (InterruptedException ex) { - LOGGER.log(Level.SEVERE, "Add Image interrupted", ex); // NON-NLS - callback.done(DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS, errorList, emptyDataSources); - return; + synchronized (addMultipleImagesLock) { + addMultipleImagesTask = new AddMultipleImagesTask(deviceId, imagePaths, timeZone , progressMonitor); + } + addMultipleImagesTask.run(); + if (addMultipleImagesTask.getResult() == DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS) { + LOGGER.log(Level.SEVERE, "Failed to add VHD datasource"); // NON-NLS + callback.done(DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS, addMultipleImagesTask.getErrorMessages(), emptyDataSources); + return; } } catch (NoCurrentCaseException ex) { String msg = Bundle.AddLogicalImageTask_noCurrentCase(); @@ -233,13 +243,22 @@ final class AddLogicalImageTask implements Runnable { } } + if (cancelled) { + if (!createVHD) { + // TODO: When 5453 is fixed, we should be able to delete it when adding VHD. + deleteDestinationDirectory(); + } + errorList.add(Bundle.AddLogicalImageTask_addImageCancelled()); + callback.done(DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS, errorList, emptyDataSources); + return; + } + try { progressMonitor.setProgressText(Bundle.AddLogicalImageTask_addingInterestingFiles()); - addingInterestingFiles = true; addInterestingFiles(Paths.get(dest.toString(), resultsFilename), createVHD); progressMonitor.setProgressText(Bundle.AddLogicalImageTask_doneAddingInterestingFiles()); if (createVHD) { - callback.done(privateCallback.getResult(), privateCallback.getErrorMessages(), privateCallback.getNewDataSources()); + callback.done(addMultipleImagesTask.getResult(), addMultipleImagesTask.getErrorMessages(), addMultipleImagesTask.getNewDataSources()); } else { callback.done(DataSourceProcessorCallback.DataSourceProcessorResult.NO_ERRORS, errorList, newDataSources); } @@ -283,13 +302,10 @@ final class AddLogicalImageTask implements Runnable { void cancelTask() { LOGGER.log(Level.WARNING, "AddLogicalImageTask cancelled, processing may be incomplete"); // NON-NLS cancelled = true; - if (addMultipleImageTask != null) { - addMultipleImageTask.cancelTask(); - } - if (!createVHD && !addingInterestingFiles) { - // Don't delete destination directory once we started adding interesting files. - // At this point the database and destination directory are complete. - deleteDestinationDirectory(); + synchronized (addMultipleImagesLock) { + if (addMultipleImagesTask != null) { + addMultipleImagesTask.cancelTask(); + } } } @@ -322,6 +338,8 @@ final class AddLogicalImageTask implements Runnable { int lineNumber = 2; while ((line = br.readLine()) != null) { if (cancelled) { + // Don't delete destination directory once we started adding interesting files. + // At this point the database and destination directory are complete. return; } String[] fields = line.split("\t", -1); // NON-NLS @@ -465,46 +483,6 @@ final class AddLogicalImageTask implements Runnable { } } - /** - * AddDataSourceCallback private class for adding VHD source - */ - private class AddDataSourceCallback extends DataSourceProcessorCallback { - private List errorMessages; - private List newDataSources; - private DataSourceProcessorResult result; - private boolean inProgress = true; - - @Override - public void done(DataSourceProcessorCallback.DataSourceProcessorResult result, List errorMessages, List newDataSources) { - LOGGER.log(Level.INFO, "privateCallback done"); // NON-NLS - this.errorMessages = errorMessages; - this.newDataSources = newDataSources; - this.result = result; - this.inProgress = false; - } - - @Override - public void doneEDT(DataSourceProcessorResult result, List errorMessages, List newDataSources) { - done(result, errorMessages, newDataSources); - } - - public List getNewDataSources() { - return newDataSources; - } - - public List getErrorMessages() { - return errorMessages; - } - - public DataSourceProcessorResult getResult() { - return result; - } - - private boolean isInProgress() { - return inProgress; - } - } - String makeQuery(boolean createVHD, String vhdFilename, String fileMetaAddressStr, String parentPath, String filename) throws TskCoreException { String query; if (createVHD) { diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddMultipleImageTask.java b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddMultipleImagesTask.java similarity index 77% rename from Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddMultipleImageTask.java rename to Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddMultipleImagesTask.java index f19330015a..3ace5c002b 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddMultipleImageTask.java +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddMultipleImagesTask.java @@ -25,7 +25,6 @@ import javax.annotation.concurrent.GuardedBy; import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; -import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorCallback; import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorCallback.DataSourceProcessorResult; import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessorProgressMonitor; import org.sleuthkit.autopsy.coreutils.Logger; @@ -43,23 +42,26 @@ import org.sleuthkit.datamodel.TskFileRange; * */ @Messages({ - "AddMultipleImageTask.fsTypeUnknownErr=Cannot determine file system type" + "AddMultipleImagesTask.fsTypeUnknownErr=Cannot determine file system type" }) -class AddMultipleImageTask implements Runnable { +class AddMultipleImagesTask implements Runnable { - private static final Logger LOGGER = Logger.getLogger(AddMultipleImageTask.class.getName()); - public static final String TSK_FS_TYPE_UNKNOWN_ERR_MSG = Bundle.AddMultipleImageTask_fsTypeUnknownErr(); + private static final Logger LOGGER = Logger.getLogger(AddMultipleImagesTask.class.getName()); + public static final String TSK_FS_TYPE_UNKNOWN_ERR_MSG = Bundle.AddMultipleImagesTask_fsTypeUnknownErr(); private static final long TWO_GB = 2000000000L; private final String deviceId; private final List imageFilePaths; private final String timeZone; private final long chunkSize = TWO_GB; private final DataSourceProcessorProgressMonitor progressMonitor; - private final DataSourceProcessorCallback callback; private final Case currentCase; private boolean criticalErrorOccurred; private SleuthkitJNI.CaseDbHandle.AddImageProcess addImageProcess = null; - + private List errorMessages = new ArrayList<>(); + private DataSourceProcessorResult result; + private List newDataSources = new ArrayList<>(); + private List emptyDataSources = new ArrayList<>(); + /* * The cancellation requested flag and SleuthKit add image process are * guarded by a lock to synchronize cancelling the process (setting the flag @@ -84,20 +86,18 @@ class AddMultipleImageTask implements Runnable { * java.util.TimeZone.getID. * @param progressMonitor Progress monitor for reporting progress during * processing. - * @param callback Callback to call when processing is done. * * @throws NoCurrentCaseException The exception if there is no open case. */ @Messages({ - "# {0} - file", "AddMultipleImageTask.addingFileAsLogicalFile=Adding: {0} as an unallocated space file.", + "# {0} - file", "AddMultipleImagesTask.addingFileAsLogicalFile=Adding: {0} as an unallocated space file.", "# {0} - deviceId", "# {1} - exceptionMessage", - "AddMultipleImageTask.errorAddingImgWithoutFileSystem=Error adding images without file systems for device {0}: {1}",}) - AddMultipleImageTask(String deviceId, List imageFilePaths, String timeZone, - DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callback) throws NoCurrentCaseException { + "AddMultipleImagesTask.errorAddingImgWithoutFileSystem=Error adding images without file systems for device {0}: {1}",}) + AddMultipleImagesTask(String deviceId, List imageFilePaths, String timeZone, + DataSourceProcessorProgressMonitor progressMonitor) throws NoCurrentCaseException { this.deviceId = deviceId; this.imageFilePaths = imageFilePaths; this.timeZone = timeZone; - this.callback = callback; this.progressMonitor = progressMonitor; currentCase = Case.getCurrentCaseThrows(); this.criticalErrorOccurred = false; @@ -105,13 +105,13 @@ class AddMultipleImageTask implements Runnable { } @Messages({ - "AddMultipleImageTask.cancelled=Cancellation: Add image process reverted", + "AddMultipleImagesTask.cancelled=Cancellation: Add image process reverted", }) @Override public void run() { - List errorMessages = new ArrayList<>(); - List newDataSources = new ArrayList<>(); - List emptyDataSources = new ArrayList<>(); + errorMessages = new ArrayList<>(); + newDataSources = new ArrayList<>(); + emptyDataSources = new ArrayList<>(); /* * Try to add the input image files as images. @@ -132,8 +132,9 @@ class AddMultipleImageTask implements Runnable { commitOrRevertAddImageProcess(imageFilePath, errorMessages, newDataSources); synchronized (tskAddImageProcessLock) { if (tskAddImageProcessStopped) { - errorMessages.add(Bundle.AddMultipleImageTask_cancelled()); - callback.done(DataSourceProcessorResult.CRITICAL_ERRORS, errorMessages, emptyDataSources); + errorMessages.add(Bundle.AddMultipleImagesTask_cancelled()); + result = DataSourceProcessorResult.CRITICAL_ERRORS; + newDataSources = emptyDataSources; return; } } @@ -151,7 +152,7 @@ class AddMultipleImageTask implements Runnable { SleuthkitCase caseDatabase; caseDatabase = currentCase.getSleuthkitCase(); try { - progressMonitor.setProgressText(Bundle.AddMultipleImageTask_addingFileAsLogicalFile(corruptedImageFilePaths.toString())); + progressMonitor.setProgressText(Bundle.AddMultipleImagesTask_addingFileAsLogicalFile(corruptedImageFilePaths.toString())); caseDatabase.acquireSingleUserCaseWriteLock(); @@ -175,7 +176,7 @@ class AddMultipleImageTask implements Runnable { caseDatabase.addLayoutFiles(dataSource, fileRanges); } catch (TskCoreException ex) { - errorMessages.add(Bundle.AddMultipleImageTask_errorAddingImgWithoutFileSystem(deviceId, ex.getLocalizedMessage())); + errorMessages.add(Bundle.AddMultipleImagesTask_errorAddingImgWithoutFileSystem(deviceId, ex.getLocalizedMessage())); criticalErrorOccurred = true; } finally { caseDatabase.releaseSingleUserCaseWriteLock(); @@ -189,10 +190,6 @@ class AddMultipleImageTask implements Runnable { progressMonitor.setProgress(0); progressMonitor.setProgress(100); - /* - * Pass the results back via the callback. - */ - DataSourceProcessorCallback.DataSourceProcessorResult result; if (criticalErrorOccurred) { result = DataSourceProcessorResult.CRITICAL_ERRORS; } else if (!errorMessages.isEmpty()) { @@ -200,7 +197,6 @@ class AddMultipleImageTask implements Runnable { } else { result = DataSourceProcessorResult.NO_ERRORS; } - callback.done(result, errorMessages, newDataSources); } /** @@ -208,7 +204,7 @@ class AddMultipleImageTask implements Runnable { * partial processing of the input. */ void cancelTask() { - LOGGER.log(Level.WARNING, "AddMultipleImageTask cancelled, processing may be incomplete"); // NON-NLS + LOGGER.log(Level.WARNING, "AddMultipleImagesTask cancelled, processing may be incomplete"); // NON-NLS synchronized (tskAddImageProcessLock) { tskAddImageProcessStopped = true; if (addImageProcess != null) { @@ -239,19 +235,19 @@ class AddMultipleImageTask implements Runnable { * for later addition as an unallocated space file. * @param errorMessages If there are any error messages, the * error messages are added to this list for - * eventual return to the caller via the - * callback. + * eventual return to the caller via the getter + * method. */ @Messages({ - "# {0} - imageFilePath", "AddMultipleImageTask.adding=Adding: {0}", - "# {0} - imageFilePath", "# {1} - deviceId", "# {2} - exceptionMessage", "AddMultipleImageTask.criticalErrorAdding=Critical error adding {0} for device {1}: {2}", - "# {0} - imageFilePath", "# {1} - deviceId", "# {2} - exceptionMessage", "AddMultipleImageTask.criticalErrorReverting=Critical error reverting add image process for {0} for device {1}: {2}", - "# {0} - imageFilePath", "# {1} - deviceId", "# {2} - exceptionMessage", "AddMultipleImageTask.nonCriticalErrorAdding=Non-critical error adding {0} for device {1}: {2}",}) + "# {0} - imageFilePath", "AddMultipleImagesTask.adding=Adding: {0}", + "# {0} - imageFilePath", "# {1} - deviceId", "# {2} - exceptionMessage", "AddMultipleImagesTask.criticalErrorAdding=Critical error adding {0} for device {1}: {2}", + "# {0} - imageFilePath", "# {1} - deviceId", "# {2} - exceptionMessage", "AddMultipleImagesTask.criticalErrorReverting=Critical error reverting add image process for {0} for device {1}: {2}", + "# {0} - imageFilePath", "# {1} - deviceId", "# {2} - exceptionMessage", "AddMultipleImagesTask.nonCriticalErrorAdding=Non-critical error adding {0} for device {1}: {2}",}) private void run(String imageFilePath, List corruptedImageFilePaths, List errorMessages) { /* * Try to add the image to the case database as a data source. */ - progressMonitor.setProgressText(Bundle.AddMultipleImageTask_adding(imageFilePath)); + progressMonitor.setProgressText(Bundle.AddMultipleImagesTask_adding(imageFilePath)); try { addImageProcess.run(deviceId, new String[]{imageFilePath}); } catch (TskCoreException ex) { @@ -264,11 +260,11 @@ class AddMultipleImageTask implements Runnable { */ corruptedImageFilePaths.add(imageFilePath); } else { - errorMessages.add(Bundle.AddMultipleImageTask_criticalErrorAdding(imageFilePath, deviceId, ex.getLocalizedMessage())); + errorMessages.add(Bundle.AddMultipleImagesTask_criticalErrorAdding(imageFilePath, deviceId, ex.getLocalizedMessage())); criticalErrorOccurred = true; } } catch (TskDataException ex) { - errorMessages.add(Bundle.AddMultipleImageTask_nonCriticalErrorAdding(imageFilePath, deviceId, ex.getLocalizedMessage())); + errorMessages.add(Bundle.AddMultipleImagesTask_nonCriticalErrorAdding(imageFilePath, deviceId, ex.getLocalizedMessage())); } } @@ -279,10 +275,10 @@ class AddMultipleImageTask implements Runnable { * * @param imageFilePath The image file path. * @param errorMessages Error messages, if any, are added to this list for - * eventual return via the callback. + * eventual return via the getter method. * @param newDataSources If the new image is successfully committed, it is * added to this list for eventual return via the - * callback. + * getter method. */ private void commitOrRevertAddImageProcess(String imageFilePath, List errorMessages, List newDataSources) { synchronized (tskAddImageProcessLock) { @@ -290,7 +286,7 @@ class AddMultipleImageTask implements Runnable { try { addImageProcess.revert(); } catch (TskCoreException ex) { - errorMessages.add(Bundle.AddMultipleImageTask_criticalErrorReverting(imageFilePath, deviceId, ex.getLocalizedMessage())); + errorMessages.add(Bundle.AddMultipleImagesTask_criticalErrorReverting(imageFilePath, deviceId, ex.getLocalizedMessage())); criticalErrorOccurred = true; } return; @@ -299,7 +295,7 @@ class AddMultipleImageTask implements Runnable { /* * Try to commit the results of the add image process, retrieve the new * image from the case database, and add it to the list of new data - * sources to be returned via the callback. + * sources to be returned via the getter method. */ try { long imageId = addImageProcess.commit(); @@ -312,7 +308,7 @@ class AddMultipleImageTask implements Runnable { */ String verificationError = dataSource.verifyImageSize(); if (!verificationError.isEmpty()) { - errorMessages.add(Bundle.AddMultipleImageTask_nonCriticalErrorAdding(imageFilePath, deviceId, verificationError)); + errorMessages.add(Bundle.AddMultipleImagesTask_nonCriticalErrorAdding(imageFilePath, deviceId, verificationError)); } } catch (TskCoreException ex) { /* @@ -320,9 +316,33 @@ class AddMultipleImageTask implements Runnable { * for the newly added image failed. Either way, this is a critical * error. */ - errorMessages.add(Bundle.AddMultipleImageTask_criticalErrorAdding(imageFilePath, deviceId, ex.getLocalizedMessage())); + errorMessages.add(Bundle.AddMultipleImagesTask_criticalErrorAdding(imageFilePath, deviceId, ex.getLocalizedMessage())); criticalErrorOccurred = true; } } } + + /** + * Return the error messages from the AddMultipleImagesTask run + * @return List of error message + */ + public List getErrorMessages() { + return errorMessages; + } + + /** + * Return the result the AddMultipleImagesTask run + * @return The result of the run + */ + public DataSourceProcessorResult getResult() { + return result; + } + + /** + * Return the new data sources the AddMultipleImagesTask run + * @return The new data sources of the run + */ + public List getNewDataSources() { + return newDataSources; + } } diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/Bundle.properties-MERGED index ebe9de0684..3e44d06104 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/Bundle.properties-MERGED @@ -2,6 +2,7 @@ # To change this template file, choose Tools | Templates # and open the template in the editor. +AddLogicalImageTask.addImageCancelled=Add image cancelled # {0} - file number # {1} - total files AddLogicalImageTask.addingExtractedFile=Adding extracted files ({0}/{1}) @@ -46,26 +47,26 @@ AddLogicalImageTask.noCurrentCase=No current case # {2} - expected length AddLogicalImageTask.notEnoughFields=File does not contain enough fields at line {0}, got {1}, expecting {2} # {0} - imageFilePath -AddMultipleImageTask.adding=Adding: {0} +AddMultipleImagesTask.adding=Adding: {0} # {0} - file -AddMultipleImageTask.addingFileAsLogicalFile=Adding: {0} as an unallocated space file. -AddMultipleImageTask.cancelled=Cancellation: Add image process reverted +AddMultipleImagesTask.addingFileAsLogicalFile=Adding: {0} as an unallocated space file. +AddMultipleImagesTask.cancelled=Cancellation: Add image process reverted # {0} - imageFilePath # {1} - deviceId # {2} - exceptionMessage -AddMultipleImageTask.criticalErrorAdding=Critical error adding {0} for device {1}: {2} +AddMultipleImagesTask.criticalErrorAdding=Critical error adding {0} for device {1}: {2} # {0} - imageFilePath # {1} - deviceId # {2} - exceptionMessage -AddMultipleImageTask.criticalErrorReverting=Critical error reverting add image process for {0} for device {1}: {2} +AddMultipleImagesTask.criticalErrorReverting=Critical error reverting add image process for {0} for device {1}: {2} # {0} - deviceId # {1} - exceptionMessage -AddMultipleImageTask.errorAddingImgWithoutFileSystem=Error adding images without file systems for device {0}: {1} -AddMultipleImageTask.fsTypeUnknownErr=Cannot determine file system type +AddMultipleImagesTask.errorAddingImgWithoutFileSystem=Error adding images without file systems for device {0}: {1} +AddMultipleImagesTask.fsTypeUnknownErr=Cannot determine file system type # {0} - imageFilePath # {1} - deviceId # {2} - exceptionMessage -AddMultipleImageTask.nonCriticalErrorAdding=Non-critical error adding {0} for device {1}: {2} +AddMultipleImagesTask.nonCriticalErrorAdding=Non-critical error adding {0} for device {1}: {2} LogicalImagerDSProcessor.dataSourceType=Autopsy Logical Imager Results LogicalImagerDSProcessor.destinationDirectoryConfirmation=Destination directory confirmation # {0} - directory diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/LogicalImagerDSProcessor.java b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/LogicalImagerDSProcessor.java index a3a2632374..81f3dbf923 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/LogicalImagerDSProcessor.java +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/LogicalImagerDSProcessor.java @@ -229,7 +229,6 @@ public final class LogicalImagerDSProcessor implements DataSourceProcessor { @Override public void cancel() { if (addLogicalImageTask != null) { - thread.interrupt(); addLogicalImageTask.cancelTask(); } } diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/LogicalImagerPanel.java b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/LogicalImagerPanel.java index 1cdf09286b..6c5b374187 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/LogicalImagerPanel.java +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/LogicalImagerPanel.java @@ -518,7 +518,7 @@ final class LogicalImagerPanel extends JPanel implements DocumentListener { } } catch (IOException ignored) { //unable to get this removable drive for default selection will try and select next removable drive by default - logger.log(Level.INFO, "Unable to select first removable drive found", ignored); + logger.log(Level.INFO, String.format("Unable to select first removable drive found: %s", ignored.getMessage())); } } i++; From 17ce90d3a91ef13dacca3babb7caa922b647f947 Mon Sep 17 00:00:00 2001 From: Joe Ho Date: Fri, 13 Sep 2019 20:02:18 -0400 Subject: [PATCH 097/102] Fix codacy errors --- .../autopsy/logicalimager/dsp/AddMultipleImagesTask.java | 3 +-- .../autopsy/logicalimager/dsp/LogicalImagerDSProcessor.java | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddMultipleImagesTask.java b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddMultipleImagesTask.java index 3ace5c002b..54bf3590db 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddMultipleImagesTask.java +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddMultipleImagesTask.java @@ -60,7 +60,6 @@ class AddMultipleImagesTask implements Runnable { private List errorMessages = new ArrayList<>(); private DataSourceProcessorResult result; private List newDataSources = new ArrayList<>(); - private List emptyDataSources = new ArrayList<>(); /* * The cancellation requested flag and SleuthKit add image process are @@ -111,7 +110,7 @@ class AddMultipleImagesTask implements Runnable { public void run() { errorMessages = new ArrayList<>(); newDataSources = new ArrayList<>(); - emptyDataSources = new ArrayList<>(); + List emptyDataSources = new ArrayList<>(); /* * Try to add the input image files as images. diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/LogicalImagerDSProcessor.java b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/LogicalImagerDSProcessor.java index 81f3dbf923..5540c1ac80 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/LogicalImagerDSProcessor.java +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/LogicalImagerDSProcessor.java @@ -53,7 +53,6 @@ public final class LogicalImagerDSProcessor implements DataSourceProcessor { private static final String LOGICAL_IMAGER_DIR = "LogicalImager"; //NON-NLS private final LogicalImagerPanel configPanel; private AddLogicalImageTask addLogicalImageTask; - private Thread thread; /* * Constructs a Logical Imager data source processor that implements the @@ -222,7 +221,7 @@ public final class LogicalImagerDSProcessor implements DataSourceProcessor { ) throws NoCurrentCaseException { addLogicalImageTask = new AddLogicalImageTask(deviceId, timeZone, src, dest, progressMonitor, callback); - thread = new Thread(addLogicalImageTask); + Thread thread = new Thread(addLogicalImageTask); thread.start(); } From 52830ffad9c274995c877b1194f161abcdf4cfcc Mon Sep 17 00:00:00 2001 From: Joe Ho Date: Mon, 16 Sep 2019 10:13:50 -0400 Subject: [PATCH 098/102] Fix PR comment --- .../autopsy/logicalimager/dsp/AddLogicalImageTask.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java index 982ede8493..b53388c872 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java @@ -227,6 +227,11 @@ final class AddLogicalImageTask implements Runnable { // ingest the VHDs try { synchronized (addMultipleImagesLock) { + if (cancelled) { + LOGGER.log(Level.SEVERE, "Add VHD cancelled"); // NON-NLS + callback.done(DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS, errorList, emptyDataSources); + return; + } addMultipleImagesTask = new AddMultipleImagesTask(deviceId, imagePaths, timeZone , progressMonitor); } addMultipleImagesTask.run(); @@ -301,8 +306,8 @@ final class AddLogicalImageTask implements Runnable { */ void cancelTask() { LOGGER.log(Level.WARNING, "AddLogicalImageTask cancelled, processing may be incomplete"); // NON-NLS - cancelled = true; synchronized (addMultipleImagesLock) { + cancelled = true; if (addMultipleImagesTask != null) { addMultipleImagesTask.cancelTask(); } From 37adf19bbdd13dfe5f66583bde8e797d99a97c09 Mon Sep 17 00:00:00 2001 From: Joe Ho Date: Mon, 16 Sep 2019 12:06:24 -0400 Subject: [PATCH 099/102] postArtifact when we cancel interesting files --- .../autopsy/logicalimager/dsp/AddLogicalImageTask.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java index b53388c872..6c0bc5a0d6 100644 --- a/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java +++ b/Core/src/org/sleuthkit/autopsy/logicalimager/dsp/AddLogicalImageTask.java @@ -140,7 +140,7 @@ final class AddLogicalImageTask implements Runnable { // Don't delete destination directory once we started adding interesting files. // At this point the database and destination directory are complete. deleteDestinationDirectory(); - errorList.add("Add image cancelled"); + errorList.add(Bundle.AddLogicalImageTask_addImageCancelled()); callback.done(DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS, errorList, emptyDataSources); return; } @@ -229,6 +229,7 @@ final class AddLogicalImageTask implements Runnable { synchronized (addMultipleImagesLock) { if (cancelled) { LOGGER.log(Level.SEVERE, "Add VHD cancelled"); // NON-NLS + errorList.add(Bundle.AddLogicalImageTask_addImageCancelled()); callback.done(DataSourceProcessorCallback.DataSourceProcessorResult.CRITICAL_ERRORS, errorList, emptyDataSources); return; } @@ -345,7 +346,7 @@ final class AddLogicalImageTask implements Runnable { if (cancelled) { // Don't delete destination directory once we started adding interesting files. // At this point the database and destination directory are complete. - return; + break; } String[] fields = line.split("\t", -1); // NON-NLS if (fields.length != 14) { From 5ad7cbe07f23c6655caf28b62abef8d30c546f01 Mon Sep 17 00:00:00 2001 From: Kelly Kelly Date: Mon, 16 Sep 2019 15:09:38 -0400 Subject: [PATCH 100/102] Timeline-removed listener for backspace button --- .../sleuthkit/autopsy/timeline/TimeLineTopComponent.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/timeline/TimeLineTopComponent.java b/Core/src/org/sleuthkit/autopsy/timeline/TimeLineTopComponent.java index be65ad40a9..92ed7c6c8a 100755 --- a/Core/src/org/sleuthkit/autopsy/timeline/TimeLineTopComponent.java +++ b/Core/src/org/sleuthkit/autopsy/timeline/TimeLineTopComponent.java @@ -356,13 +356,9 @@ public final class TimeLineTopComponent extends TopComponent implements Explorer scene.addEventFilter(KeyEvent.KEY_PRESSED, keyEvent -> { if (new KeyCodeCombination(KeyCode.LEFT, KeyCodeCombination.ALT_DOWN).match(keyEvent)) { new Back(controller).handle(null); - } else if (new KeyCodeCombination(KeyCode.BACK_SPACE).match(keyEvent)) { - new Back(controller).handle(null); } else if (new KeyCodeCombination(KeyCode.RIGHT, KeyCodeCombination.ALT_DOWN).match(keyEvent)) { new Forward(controller).handle(null); - } else if (new KeyCodeCombination(KeyCode.BACK_SPACE, KeyCodeCombination.SHIFT_DOWN).match(keyEvent)) { - new Forward(controller).handle(null); - } + } }); //add ui componenets to JFXPanels From fbc2cc2eff2365bed0ff132d86fe3e56fe234549 Mon Sep 17 00:00:00 2001 From: Kelly Kelly Date: Mon, 16 Sep 2019 15:19:35 -0400 Subject: [PATCH 101/102] Removed unused import from EmlParser --- .../src/org/sleuthkit/autopsy/thunderbirdparser/EMLParser.java | 1 - 1 file changed, 1 deletion(-) diff --git a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/EMLParser.java b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/EMLParser.java index 14a2d73993..e0b50ffa8c 100755 --- a/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/EMLParser.java +++ b/thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/EMLParser.java @@ -22,7 +22,6 @@ import java.io.FileNotFoundException; import java.io.IOException; import org.apache.james.mime4j.MimeException; import org.apache.james.mime4j.dom.Message; -import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.ReadContentInputStream; From e8d81c4240532e04802bab51b70376c951fbe0ef Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Tue, 17 Sep 2019 11:57:57 -0400 Subject: [PATCH 102/102] 5423 prevent multi-select in instances list --- Core/src/org/sleuthkit/autopsy/filequery/ResultsPanel.form | 1 + Core/src/org/sleuthkit/autopsy/filequery/ResultsPanel.java | 1 + 2 files changed, 2 insertions(+) diff --git a/Core/src/org/sleuthkit/autopsy/filequery/ResultsPanel.form b/Core/src/org/sleuthkit/autopsy/filequery/ResultsPanel.form index c666c85413..97080a329a 100644 --- a/Core/src/org/sleuthkit/autopsy/filequery/ResultsPanel.form +++ b/Core/src/org/sleuthkit/autopsy/filequery/ResultsPanel.form @@ -296,6 +296,7 @@ + diff --git a/Core/src/org/sleuthkit/autopsy/filequery/ResultsPanel.java b/Core/src/org/sleuthkit/autopsy/filequery/ResultsPanel.java index d8798a42e4..5809eb6051 100644 --- a/Core/src/org/sleuthkit/autopsy/filequery/ResultsPanel.java +++ b/Core/src/org/sleuthkit/autopsy/filequery/ResultsPanel.java @@ -427,6 +427,7 @@ public class ResultsPanel extends javax.swing.JPanel { instancesList.setBorder(javax.swing.BorderFactory.createTitledBorder(org.openide.util.NbBundle.getMessage(ResultsPanel.class, "ResultsPanel.instancesList.border.title"))); // NOI18N instancesList.setModel(instancesListModel); + instancesList.setSelectionMode(javax.swing.ListSelectionModel.SINGLE_SELECTION); instancesList.setCellRenderer(new InstancesCellRenderer()); instancesScrollPane.setViewportView(instancesList);