mstoeck3/autopsy-flatpak
1
0
Fork 0
mirror of https://github.com/overcuriousity/autopsy-flatpak.git synced 2025-07-06 21:00:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge custom-release-may-2018 into develop

Browse Source
This commit is contained in:
Richard Cordovano 2018-05-21 18:42:23 -04:00
commit f9a93e0360
50 changed files with 2624 additions and 212 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Core/build.xml
View File

@ -100,7 +100,6 @@
<target name="get-deps" depends="init-ivy,getTSKJars,get-thirdparty-dependencies,get-InternalPythonModules, download-binlist,getTestDataFiles">
<mkdir dir="${ext.dir}"/>
<copy file="${thirdparty.dir}/LICENSE-2.0.txt" todir="${ext.dir}" />
<!-- fetch all the dependencies from Ivy and stick them in the right places -->
<ivy:resolve log="quiet"/>
<ivy:retrieve conf="core" pattern="${ext.dir}/[artifact]-[revision](-[classifier]).[ext]" />

6
Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java
View File

@ -39,6 +39,8 @@ import org.openide.util.NbBundle.Messages;
import org.sleuthkit.autopsy.casemodule.Case;
import static org.sleuthkit.autopsy.centralrepository.datamodel.EamDbUtil.updateSchemaVersion;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.healthmonitor.EnterpriseHealthMonitor;
import org.sleuthkit.autopsy.healthmonitor.TimingMetric;
import org.sleuthkit.datamodel.CaseDbSchemaVersionNumber;
import org.sleuthkit.datamodel.TskData;
@ -939,6 +941,8 @@ public abstract class AbstractSqlEamDb implements EamDb {
return;
}
TimingMetric timingMetric = EnterpriseHealthMonitor.getTimingMetric("Correlation Engine: Bulk insert");
for (CorrelationAttribute.Type type : artifactTypes) {
String tableName = EamDbUtil.correlationTypeToInstanceTableName(type);
@ -989,6 +993,8 @@ public abstract class AbstractSqlEamDb implements EamDb {
bulkArtifacts.get(type.getDbTableName()).clear();
}
EnterpriseHealthMonitor.submitTimingMetric(timingMetric);
// Reset state
bulkArtifactsCount = 0;
}

4
Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java
View File

@ -49,6 +49,8 @@ import org.sleuthkit.datamodel.HashUtility;
import org.sleuthkit.datamodel.TskCoreException;
import org.sleuthkit.datamodel.TskData;
import org.sleuthkit.autopsy.centralrepository.eventlisteners.IngestEventsListener;
import org.sleuthkit.autopsy.healthmonitor.EnterpriseHealthMonitor;
import org.sleuthkit.autopsy.healthmonitor.TimingMetric;
/**
* Ingest module for inserting entries into the Central Repository database on
@ -129,7 +131,9 @@ final class IngestModule implements FileIngestModule {
*/
if (abstractFile.getKnown() != TskData.FileKnown.KNOWN && flagTaggedNotableItems) {
try {
TimingMetric timingMetric = EnterpriseHealthMonitor.getTimingMetric("Correlation Engine: Notable artifact query");
List<String> caseDisplayNamesList = dbManager.getListCasesHavingArtifactInstancesKnownBad(filesType, md5);
EnterpriseHealthMonitor.submitTimingMetric(timingMetric);
if (!caseDisplayNamesList.isEmpty()) {
postCorrelatedBadFileToBlackboard(abstractFile, caseDisplayNamesList);
}

2
Core/src/org/sleuthkit/autopsy/corecomponentinterfaces/DataResult.java
View File

@ -63,7 +63,7 @@ public interface DataResult {
* Sets the descriptive text about the source of the nodes displayed in this
* result view component.
*
* @param description The text to display.
* @param pathText The text to display.
*/
public void setPath(String pathText);

2
Core/src/org/sleuthkit/autopsy/corecomponents/DataResultPanel.java
View File

@ -734,7 +734,7 @@ public class DataResultPanel extends javax.swing.JPanel implements DataResult, C
*
* @return True or false.
*
* @Deprecated This method has no valid use case.
* @deprecated This method has no valid use case.
*/
@Deprecated
@Override

2
Core/src/org/sleuthkit/autopsy/datamodel/AutopsyVisitableItem.java
View File

@ -28,7 +28,7 @@ public interface AutopsyVisitableItem {
/**
* visitor pattern support
*
* @param v visitor
* @param visitor visitor
*
* @return visitor return value
*/

2
Core/src/org/sleuthkit/autopsy/datamodel/ContentNode.java
View File

@ -40,7 +40,7 @@ abstract class ContentNode extends DisplayableItemNode {
/**
* Visitor pattern support.
*
* @param v visitor
* @param visitor visitor
*
* @return visitor's visit return value
*/

456
Core/src/org/sleuthkit/autopsy/healthmonitor/EnterpriseHealthMonitor.java
View File

@ -28,8 +28,11 @@ import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.Map;
import java.util.List;
import java.util.HashMap;
import java.util.List;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.GregorianCalendar;
import java.util.UUID;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@ -37,6 +40,7 @@ import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.logging.Level;
import java.util.Random;
import org.apache.commons.dbcp2.BasicDataSource;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException;
@ -44,7 +48,6 @@ import org.sleuthkit.autopsy.coordinationservice.CoordinationService;
import org.sleuthkit.autopsy.core.UserPreferences;
import org.sleuthkit.autopsy.core.UserPreferencesException;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.coreutils.ModuleSettings;
import org.sleuthkit.autopsy.coreutils.ThreadUtils;
import org.sleuthkit.datamodel.CaseDbConnectionInfo;
import org.sleuthkit.datamodel.CaseDbSchemaVersionNumber;
@ -64,8 +67,6 @@ public final class EnterpriseHealthMonitor implements PropertyChangeListener {
private final static Logger logger = Logger.getLogger(EnterpriseHealthMonitor.class.getName());
private final static String DATABASE_NAME = "EnterpriseHealthMonitor";
private final static String MODULE_NAME = "EnterpriseHealthMonitor";
private final static String IS_ENABLED_KEY = "is_enabled";
private final static long DATABASE_WRITE_INTERVAL = 60; // Minutes
public static final CaseDbSchemaVersionNumber CURRENT_DB_SCHEMA_VERSION
= new CaseDbSchemaVersionNumber(1, 0);
@ -80,6 +81,7 @@ public final class EnterpriseHealthMonitor implements PropertyChangeListener {
private final Map<String, TimingInfo> timingInfoMap;
private static final int CONN_POOL_SIZE = 10;
private BasicDataSource connectionPool = null;
private CaseDbConnectionInfo connectionSettingsInUse = null;
private String hostName;
private EnterpriseHealthMonitor() throws HealthMonitorException {
@ -100,22 +102,11 @@ public final class EnterpriseHealthMonitor implements PropertyChangeListener {
logger.log(Level.SEVERE, "Unable to look up host name - falling back to UUID " + hostName, ex);
}
// Read from module settings to determine if the module is enabled
if (ModuleSettings.settingExists(MODULE_NAME, IS_ENABLED_KEY)) {
if(ModuleSettings.getConfigSetting(MODULE_NAME, IS_ENABLED_KEY).equals("true")){
isEnabled.set(true);
try {
activateMonitor();
} catch (HealthMonitorException ex) {
// If we failed to activate it, then disable the monitor
logger.log(Level.SEVERE, "Health monitor activation failed - disabling health monitor");
setEnabled(false);
throw ex;
}
return;
}
}
isEnabled.set(false);
// Read from the database to determine if the module is enabled
updateFromGlobalEnabledStatus();
// Start the timer for database checks and writes
startTimer();
}
/**
@ -137,10 +128,13 @@ public final class EnterpriseHealthMonitor implements PropertyChangeListener {
* out of the maps, and sets up the timer for writing to the database.
* @throws HealthMonitorException
*/
private synchronized void activateMonitor() throws HealthMonitorException {
private synchronized void activateMonitorLocally() throws HealthMonitorException {
logger.log(Level.INFO, "Activating Servies Health Monitor");
// Make sure there are no left over connections to an old database
shutdownConnections();
if (!UserPreferences.getIsMultiUserModeEnabled()) {
throw new HealthMonitorException("Multi user mode is not enabled - can not activate health monitor");
}
@ -168,9 +162,6 @@ public final class EnterpriseHealthMonitor implements PropertyChangeListener {
// Clear out any old data
timingInfoMap.clear();
// Start the timer for database writes
startTimer();
}
/**
@ -180,16 +171,13 @@ public final class EnterpriseHealthMonitor implements PropertyChangeListener {
* and shuts down the connection pool.
* @throws HealthMonitorException
*/
private synchronized void deactivateMonitor() throws HealthMonitorException {
private synchronized void deactivateMonitorLocally() throws HealthMonitorException {
logger.log(Level.INFO, "Deactivating Servies Health Monitor");
// Clear out the collected data
timingInfoMap.clear();
// Stop the timer
stopTimer();
// Shut down the connection pool
shutdownConnections();
}
@ -202,7 +190,7 @@ public final class EnterpriseHealthMonitor implements PropertyChangeListener {
stopTimer();
healthMonitorOutputTimer = new ScheduledThreadPoolExecutor(1, new ThreadFactoryBuilder().setNameFormat("health_monitor_timer").build());
healthMonitorOutputTimer.scheduleWithFixedDelay(new DatabaseWriteTask(), DATABASE_WRITE_INTERVAL, DATABASE_WRITE_INTERVAL, TimeUnit.MINUTES);
healthMonitorOutputTimer.scheduleWithFixedDelay(new PeriodicHealthMonitorTask(), DATABASE_WRITE_INTERVAL, DATABASE_WRITE_INTERVAL, TimeUnit.MINUTES);
}
/**
@ -234,15 +222,18 @@ public final class EnterpriseHealthMonitor implements PropertyChangeListener {
}
if(enabled) {
getInstance().activateMonitor();
getInstance().activateMonitorLocally();
// If activateMonitor fails, we won't update either of these
ModuleSettings.setConfigSetting(MODULE_NAME, IS_ENABLED_KEY, "true");
// If activateMonitor fails, we won't update this
getInstance().setGlobalEnabledStatusInDB(true);
isEnabled.set(true);
} else {
ModuleSettings.setConfigSetting(MODULE_NAME, IS_ENABLED_KEY, "false");
if(isEnabled.get()) {
// If we were enabled before, set the global state to disabled
getInstance().setGlobalEnabledStatusInDB(false);
}
isEnabled.set(false);
getInstance().deactivateMonitor();
getInstance().deactivateMonitorLocally();
}
}
@ -460,7 +451,6 @@ public final class EnterpriseHealthMonitor implements PropertyChangeListener {
String createCommand = "SELECT 1 AS result FROM pg_database WHERE datname='" + DATABASE_NAME + "'";
rs = statement.executeQuery(createCommand);
if(rs.next()) {
logger.log(Level.INFO, "Existing Enterprise Health Monitor database found");
return true;
}
} finally {
@ -501,6 +491,7 @@ public final class EnterpriseHealthMonitor implements PropertyChangeListener {
private void setupConnectionPool() throws HealthMonitorException {
try {
CaseDbConnectionInfo db = UserPreferences.getDatabaseConnectionInfo();
connectionSettingsInUse = db;
connectionPool = new BasicDataSource();
connectionPool.setDriverClassName("org.postgresql.Driver");
@ -598,6 +589,108 @@ public final class EnterpriseHealthMonitor implements PropertyChangeListener {
}
}
/**
* Return whether the health monitor is locally enabled.
* This does not query the database.
* @return true if it is enabled, false otherwise
*/
static boolean monitorIsEnabled() {
return isEnabled.get();
}
/**
* Check whether monitoring should be enabled from the monitor database
* and enable/disable as needed.
* @throws HealthMonitorException
*/
synchronized void updateFromGlobalEnabledStatus() throws HealthMonitorException {
boolean previouslyEnabled = monitorIsEnabled();
// We can't even check the database if multi user settings aren't enabled.
if (!UserPreferences.getIsMultiUserModeEnabled()) {
isEnabled.set(false);
if(previouslyEnabled) {
deactivateMonitorLocally();
}
return;
}
// If the health monitor database doesn't exist or if it is not initialized,
// then monitoring isn't enabled
if ((! databaseExists()) || (! databaseIsInitialized())) {
isEnabled.set(false);
if(previouslyEnabled) {
deactivateMonitorLocally();
}
return;
}
// If we're currently enabled, check whether the multiuser settings have changed.
// If they have, force a reset on the connection pool.
if(previouslyEnabled && (connectionSettingsInUse != null)) {
try {
CaseDbConnectionInfo currentSettings = UserPreferences.getDatabaseConnectionInfo();
if(! (connectionSettingsInUse.getUserName().equals(currentSettings.getUserName())
&& connectionSettingsInUse.getPassword().equals(currentSettings.getPassword())
&& connectionSettingsInUse.getPort().equals(currentSettings.getPort())
&& connectionSettingsInUse.getHost().equals(currentSettings.getHost()) )) {
shutdownConnections();
}
} catch (UserPreferencesException ex) {
throw new HealthMonitorException("Error reading database connection info", ex);
}
}
boolean currentlyEnabled = getGlobalEnabledStatusFromDB();
if( currentlyEnabled != previouslyEnabled) {
if( ! currentlyEnabled ) {
isEnabled.set(false);
deactivateMonitorLocally();
} else {
isEnabled.set(true);
activateMonitorLocally();
}
}
}
/**
* Read the enabled status from the database.
* Check that the health monitor database exists before calling this.
* @return true if the database is enabled, false otherwise
* @throws HealthMonitorException
*/
private boolean getGlobalEnabledStatusFromDB() throws HealthMonitorException {
try (Connection conn = connect();
Statement statement = conn.createStatement();
ResultSet resultSet = statement.executeQuery("SELECT value FROM db_info WHERE name='MONITOR_ENABLED'")) {
if (resultSet.next()) {
return(resultSet.getBoolean("value"));
}
throw new HealthMonitorException("No enabled status found in database");
} catch (SQLException ex) {
throw new HealthMonitorException("Error initializing database", ex);
}
}
/**
* Set the global enabled status in the database.
* @throws HealthMonitorException
*/
private void setGlobalEnabledStatusInDB(boolean status) throws HealthMonitorException {
try (Connection conn = connect();
Statement statement = conn.createStatement();) {
statement.execute("UPDATE db_info SET value='" + status + "' WHERE name='MONITOR_ENABLED'");
} catch (SQLException ex) {
throw new HealthMonitorException("Error setting enabled status", ex);
}
}
/**
* Get the current schema version
* @return the current schema version
@ -688,6 +781,7 @@ public final class EnterpriseHealthMonitor implements PropertyChangeListener {
statement.execute("INSERT INTO db_info (name, value) VALUES ('SCHEMA_VERSION', '" + CURRENT_DB_SCHEMA_VERSION.getMajor() + "')");
statement.execute("INSERT INTO db_info (name, value) VALUES ('SCHEMA_MINOR_VERSION', '" + CURRENT_DB_SCHEMA_VERSION.getMinor() + "')");
statement.execute("INSERT INTO db_info (name, value) VALUES ('MONITOR_ENABLED', 'true')");
conn.commit();
} catch (SQLException ex) {
@ -708,20 +802,24 @@ public final class EnterpriseHealthMonitor implements PropertyChangeListener {
/**
* The task called by the ScheduledThreadPoolExecutor to handle
* the database writes.
* the database checks/writes.
*/
static final class DatabaseWriteTask implements Runnable {
static final class PeriodicHealthMonitorTask implements Runnable {
/**
* Write current metric data to the database
* Perform all periodic tasks:
* - Check if monitoring has been enabled / disabled in the database
* - Gather any additional metrics
* - Write current metric data to the database
*/
@Override
public void run() {
try {
getInstance().updateFromGlobalEnabledStatus();
getInstance().gatherTimerBasedMetrics();
getInstance().writeCurrentStateToDatabase();
} catch (HealthMonitorException ex) {
logger.log(Level.SEVERE, "Error writing current metrics to database", ex); //NON-NLS
logger.log(Level.SEVERE, "Error performing periodic task", ex); //NON-NLS
}
}
}
@ -734,12 +832,220 @@ public final class EnterpriseHealthMonitor implements PropertyChangeListener {
case CURRENT_CASE:
if ((null == evt.getNewValue()) && (evt.getOldValue() instanceof Case)) {
// When a case is closed, write the current metrics to the database
healthMonitorExecutor.submit(new EnterpriseHealthMonitor.DatabaseWriteTask());
healthMonitorExecutor.submit(new EnterpriseHealthMonitor.PeriodicHealthMonitorTask());
}
break;
}
}
/**
* Debugging method to generate sample data for the database.
* It will delete all current timing data and replace it with randomly generated values.
* If there is more than one node, the second node's times will trend upwards.
*/
void populateDatabaseWithSampleData(int nDays, int nNodes, boolean createVerificationData) throws HealthMonitorException {
if(! isEnabled.get()) {
throw new HealthMonitorException("Can't populate database - monitor not enabled");
}
// Get the database lock
CoordinationService.Lock lock = getSharedDbLock();
if(lock == null) {
throw new HealthMonitorException("Error getting database lock");
}
String[] metricNames = {"Disk Reads: Hash calculation", "Database: getImages query", "Solr: Index chunk", "Solr: Connectivity check"}; // NON-NLS
Random rand = new Random();
long maxTimestamp = System.currentTimeMillis();
long millisPerHour = 1000 * 60 * 60;
long minTimestamp = maxTimestamp - (nDays * (millisPerHour * 24));
Connection conn = null;
try {
conn = connect();
if(conn == null) {
throw new HealthMonitorException("Error getting database connection");
}
try (Statement statement = conn.createStatement()) {
statement.execute("DELETE FROM timing_data"); // NON-NLS
} catch (SQLException ex) {
logger.log(Level.SEVERE, "Error clearing timing data", ex);
return;
}
// Add timing metrics to the database
String addTimingInfoSql = "INSERT INTO timing_data (name, host, timestamp, count, average, max, min) VALUES (?, ?, ?, ?, ?, ?, ?)";
try (PreparedStatement statement = conn.prepareStatement(addTimingInfoSql)) {
for(String metricName:metricNames) {
long baseIndex = rand.nextInt(900) + 100;
int multiplier = rand.nextInt(5);
long minIndexTimeNanos;
switch(multiplier) {
case 0:
minIndexTimeNanos = baseIndex;
break;
case 1:
minIndexTimeNanos = baseIndex * 1000;
break;
default:
minIndexTimeNanos = baseIndex * 1000 * 1000;
break;
}
long maxIndexTimeOverMin = minIndexTimeNanos * 3;
for(int node = 0;node < nNodes; node++) {
String host = "testHost" + node; // NON-NLS
double count = 0;
double maxCount = nDays * 24 + 1;
// Record data every hour, with a small amount of randomness about when it starts
for(long timestamp = minTimestamp + rand.nextInt(1000 * 60 * 55);timestamp < maxTimestamp;timestamp += millisPerHour) {
double aveTime;
// This creates data that increases in the last couple of days of the simulated
// collection
count++;
double slowNodeMultiplier = 1.0;
if((maxCount - count) <= 3 * 24) {
slowNodeMultiplier += (3 - (maxCount - count) / 24) * 0.33;
}
if( ! createVerificationData ) {
// Try to make a reasonable sample data set, with most points in a small range
// but some higher and lower
int outlierVal = rand.nextInt(30);
long randVal = rand.nextLong();
if(randVal < 0) {
randVal *= -1;
}
if(outlierVal < 2){
aveTime = minIndexTimeNanos + maxIndexTimeOverMin + randVal % maxIndexTimeOverMin;
} else if(outlierVal == 2){
aveTime = (minIndexTimeNanos / 2) + randVal % (minIndexTimeNanos / 2);
} else if(outlierVal < 17) {
aveTime = minIndexTimeNanos + randVal % (maxIndexTimeOverMin / 2);
} else {
aveTime = minIndexTimeNanos + randVal % maxIndexTimeOverMin;
}
if(node == 1) {
aveTime = aveTime * slowNodeMultiplier;
}
} else {
// Create a data set strictly for testing that the display is working
// correctly. The average time will equal the day of the month from
// the timestamp (in milliseconds)
Calendar thisDate = new GregorianCalendar();
thisDate.setTimeInMillis(timestamp);
int day = thisDate.get(Calendar.DAY_OF_MONTH);
aveTime = day * 1000000;
}
statement.setString(1, metricName);
statement.setString(2, host);
statement.setLong(3, timestamp);
statement.setLong(4, 0);
statement.setDouble(5, aveTime / 1000000);
statement.setDouble(6, 0);
statement.setDouble(7, 0);
statement.execute();
}
}
}
} catch (SQLException ex) {
throw new HealthMonitorException("Error saving metric data to database", ex);
}
} finally {
try {
if(conn != null) {
conn.close();
}
} catch (SQLException ex) {
logger.log(Level.SEVERE, "Error closing Connection.", ex);
}
try {
lock.release();
} catch (CoordinationService.CoordinationServiceException ex) {
throw new HealthMonitorException("Error releasing database lock", ex);
}
}
}
/**
* Get timing metrics currently stored in the database.
* @param timeRange Maximum age for returned metrics (in milliseconds)
* @return A map with metric name mapped to a list of data
* @throws HealthMonitorException
*/
Map<String, List<DatabaseTimingResult>> getTimingMetricsFromDatabase(long timeRange) throws HealthMonitorException {
// Make sure the monitor is enabled. It could theoretically get disabled after this
// check but it doesn't seem worth holding a lock to ensure that it doesn't since that
// may slow down ingest.
if(! isEnabled.get()) {
throw new HealthMonitorException("Health Monitor is not enabled");
}
// Calculate the smallest timestamp we should return
long minimumTimestamp = System.currentTimeMillis() - timeRange;
try (CoordinationService.Lock lock = getSharedDbLock()) {
if(lock == null) {
throw new HealthMonitorException("Error getting database lock");
}
Connection conn = connect();
if(conn == null) {
throw new HealthMonitorException("Error getting database connection");
}
Map<String, List<DatabaseTimingResult>> resultMap = new HashMap<>();
try (Statement statement = conn.createStatement();
ResultSet resultSet = statement.executeQuery("SELECT * FROM timing_data WHERE timestamp > " + minimumTimestamp)) {
while (resultSet.next()) {
String name = resultSet.getString("name");
DatabaseTimingResult timingResult = new DatabaseTimingResult(resultSet);
if(resultMap.containsKey(name)) {
resultMap.get(name).add(timingResult);
} else {
List<DatabaseTimingResult> resultList = new ArrayList<>();
resultList.add(timingResult);
resultMap.put(name, resultList);
}
}
return resultMap;
} catch (SQLException ex) {
throw new HealthMonitorException("Error reading timing metrics from database", ex);
} finally {
try {
conn.close();
} catch (SQLException ex) {
logger.log(Level.SEVERE, "Error closing Connection.", ex);
}
}
} catch (CoordinationService.CoordinationServiceException ex) {
throw new HealthMonitorException("Error getting database lock", ex);
}
}
/**
* Get an exclusive lock for the health monitor database.
* Acquire this before creating, initializing, or updating the database schema.
@ -856,4 +1162,74 @@ public final class EnterpriseHealthMonitor implements PropertyChangeListener {
return count;
}
}
/**
* Class for retrieving timing metrics from the database to display to the user.
* All times will be in milliseconds.
*/
static class DatabaseTimingResult {
private final long timestamp; // Time the metric was recorded
private final String hostname; // Host that recorded the metric
private final long count; // Number of metrics collected
private final double average; // Average of the durations collected (milliseconds)
private final double max; // Maximum value found (milliseconds)
private final double min; // Minimum value found (milliseconds)
DatabaseTimingResult(ResultSet resultSet) throws SQLException {
this.timestamp = resultSet.getLong("timestamp");
this.hostname = resultSet.getString("host");
this.count = resultSet.getLong("count");
this.average = resultSet.getDouble("average");
this.max = resultSet.getDouble("max");
this.min = resultSet.getDouble("min");
}
/**
* Get the timestamp for when the metric was recorded
* @return
*/
long getTimestamp() {
return timestamp;
}
/**
* Get the average duration
* @return average duration (milliseconds)
*/
double getAverage() {
return average;
}
/**
* Get the maximum duration
* @return maximum duration (milliseconds)
*/
double getMax() {
return max;
}
/**
* Get the minimum duration
* @return minimum duration (milliseconds)
*/
double getMin() {
return min;
}
/**
* Get the total number of metrics collected
* @return number of metrics collected
*/
long getCount() {
return count;
}
/**
* Get the name of the host that recorded this metric
* @return the host
*/
String getHostName() {
return hostname;
}
}
}

513
Core/src/org/sleuthkit/autopsy/healthmonitor/HealthMonitorDashboard.java Normal file
View File

@ -0,0 +1,513 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2018 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.healthmonitor;
import java.awt.Container;
import java.awt.Cursor;
import java.awt.Dimension;
import java.util.Set;
import java.util.HashSet;
import java.util.HashMap;
import java.util.Arrays;
import java.util.List;
import java.awt.event.ActionEvent;
import java.awt.event.ActionListener;
import java.io.File;
import javax.swing.Box;
import javax.swing.JButton;
import javax.swing.JDialog;
import javax.swing.JComboBox;
import javax.swing.JSeparator;
import javax.swing.JCheckBox;
import javax.swing.JLabel;
import javax.swing.JPanel;
import javax.swing.JScrollPane;
import javax.swing.BorderFactory;
import java.util.Map;
import javax.swing.BoxLayout;
import java.awt.GridLayout;
import java.nio.file.Paths;
import java.util.logging.Level;
import java.util.stream.Collectors;
import org.openide.modules.Places;
import org.openide.util.NbBundle;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil;
/**
* Dashboard for viewing metrics and controlling the health monitor.
*/
public class HealthMonitorDashboard {
private final static Logger logger = Logger.getLogger(HealthMonitorDashboard.class.getName());
private final static String ADMIN_ACCESS_FILE_NAME = "adminAccess"; // NON-NLS
private final static String ADMIN_ACCESS_FILE_PATH = Paths.get(Places.getUserDirectory().getAbsolutePath(), ADMIN_ACCESS_FILE_NAME).toString();
Map<String, List<EnterpriseHealthMonitor.DatabaseTimingResult>> timingData;
private JComboBox<String> dateComboBox = null;
private JComboBox<String> hostComboBox = null;
private JCheckBox hostCheckBox = null;
private JCheckBox showTrendLineCheckBox = null;
private JCheckBox skipOutliersCheckBox = null;
private JPanel graphPanel = null;
private JDialog dialog = null;
private final Container parentWindow;
/**
* Create an instance of the dashboard.
* Call display() after creation to show the dashboard.
* @param parent The parent container (for centering the UI)
*/
public HealthMonitorDashboard(Container parent) {
timingData = new HashMap<>();
parentWindow = parent;
}
/**
* Display the dashboard.
*/
@NbBundle.Messages({"HealthMonitorDashboard.display.errorCreatingDashboard=Error creating health monitor dashboard",
"HealthMonitorDashboard.display.dashboardTitle=Enterprise Health Monitor"})
public void display() {
// Update the enabled status and get the timing data, then create all
// the sub panels.
JPanel timingPanel;
JPanel adminPanel;
try {
updateData();
timingPanel = createTimingPanel();
adminPanel = createAdminPanel();
} catch (HealthMonitorException ex) {
logger.log(Level.SEVERE, "Error creating panels for health monitor dashboard", ex);
MessageNotifyUtil.Message.error(Bundle.HealthMonitorDashboard_display_errorCreatingDashboard());
return;
}
// Create the main panel for the dialog
JPanel mainPanel = new JPanel();
mainPanel.setLayout(new BoxLayout(mainPanel, BoxLayout.Y_AXIS));
// Add the timing panel
mainPanel.add(timingPanel);
// Add the admin panel if the admin file is present
File adminFile = new File(ADMIN_ACCESS_FILE_PATH);
if(adminFile.exists()) {
mainPanel.add(adminPanel);
}
// Create and show the dialog
dialog = new JDialog();
dialog.setTitle(Bundle.HealthMonitorDashboard_display_dashboardTitle());
dialog.add(mainPanel);
dialog.pack();
dialog.setLocationRelativeTo(parentWindow);
dialog.setVisible(true);
}
/**
* Delete the current dialog and create a new one. This should only be
* called after enabling or disabling the health monitor.
*/
private void redisplay() {
if (dialog != null) {
dialog.setVisible(false);
dialog.dispose();
}
display();
}
/**
* Check the monitor enabled status and, if enabled, get the timing data.
* @throws HealthMonitorException
*/
private void updateData() throws HealthMonitorException {
// Update the monitor status
EnterpriseHealthMonitor.getInstance().updateFromGlobalEnabledStatus();
if(EnterpriseHealthMonitor.monitorIsEnabled()) {
// Get a copy of the timing data from the database
timingData = EnterpriseHealthMonitor.getInstance().getTimingMetricsFromDatabase(DateRange.getMaximumTimestampRange());
}
}
/**
* Create the panel holding the timing graphs and the controls for them.
* @return The timing panel
* @throws HealthMonitorException
*/
@NbBundle.Messages({"HealthMonitorDashboard.createTimingPanel.noData=No data to display - monitor is not enabled",
"HealthMonitorDashboard.createTimingPanel.timingMetricsTitle=Timing Metrics"})
private JPanel createTimingPanel() throws HealthMonitorException {
// If the monitor isn't enabled, just add a message
if(! EnterpriseHealthMonitor.monitorIsEnabled()) {
//timingMetricPanel.setPreferredSize(new Dimension(400,100));
JPanel emptyTimingMetricPanel = new JPanel();
emptyTimingMetricPanel.add(new JLabel(Bundle.HealthMonitorDashboard_createTimingPanel_timingMetricsTitle()));
emptyTimingMetricPanel.add(new JLabel(" "));
emptyTimingMetricPanel.add(new JLabel(Bundle.HealthMonitorDashboard_createTimingPanel_noData()));
return emptyTimingMetricPanel;
}
JPanel timingMetricPanel = new JPanel();
timingMetricPanel.setLayout(new BoxLayout(timingMetricPanel, BoxLayout.PAGE_AXIS));
timingMetricPanel.setBorder(BorderFactory.createEtchedBorder());
// Add title
JLabel timingMetricTitle = new JLabel(Bundle.HealthMonitorDashboard_createTimingPanel_timingMetricsTitle());
timingMetricPanel.add(timingMetricTitle);
timingMetricPanel.add(new JSeparator());
// Add the controls
timingMetricPanel.add(createTimingControlPanel());
timingMetricPanel.add(new JSeparator());
// Create panel to hold graphs
graphPanel = new JPanel();
graphPanel.setLayout(new GridLayout(0,2));
// Update the graph panel, put it in a scroll pane, and add to the timing metric panel
updateTimingMetricGraphs();
JScrollPane scrollPane = new JScrollPane(graphPanel, JScrollPane.VERTICAL_SCROLLBAR_AS_NEEDED, JScrollPane.HORIZONTAL_SCROLLBAR_NEVER);
timingMetricPanel.add(scrollPane);
timingMetricPanel.revalidate();
timingMetricPanel.repaint();
return timingMetricPanel;
}
/**
* Create the panel with combo boxes for date range and host.
* @return the control panel
*/
@NbBundle.Messages({"HealthMonitorDashboard.createTimingControlPanel.filterByHost=Filter by host",
"HealthMonitorDashboard.createTimingControlPanel.maxDays=Max days to display",
"HealthMonitorDashboard.createTimingControlPanel.skipOutliers=Do not plot outliers",
"HealthMonitorDashboard.createTimingControlPanel.showTrendLine=Show trend line"})
private JPanel createTimingControlPanel() {
JPanel timingControlPanel = new JPanel();
// If the monitor is not enabled, don't add any components
if(! EnterpriseHealthMonitor.monitorIsEnabled()) {
return timingControlPanel;
}
// Create the combo box for selecting how much data to display
String[] dateOptionStrings = Arrays.stream(DateRange.values()).map(e -> e.getLabel()).toArray(String[]::new);
dateComboBox = new JComboBox<>(dateOptionStrings);
dateComboBox.setSelectedItem(DateRange.ONE_DAY.getLabel());
// Set up the listener on the date combo box
dateComboBox.addActionListener(new ActionListener() {
@Override
public void actionPerformed(ActionEvent arg0) {
try {
updateTimingMetricGraphs();
} catch (HealthMonitorException ex) {
logger.log(Level.SEVERE, "Error updating timing metric panel", ex);
}
}
});
// Create an array of host names
Set<String> hostNameSet = new HashSet<>();
for(String metricType:timingData.keySet()) {
for(EnterpriseHealthMonitor.DatabaseTimingResult result: timingData.get(metricType)) {
hostNameSet.add(result.getHostName());
}
}
// Load the host names into the combo box
hostComboBox = new JComboBox<>(hostNameSet.toArray(new String[hostNameSet.size()]));
// Set up the listener on the combo box
hostComboBox.addActionListener(new ActionListener() {
@Override
public void actionPerformed(ActionEvent arg0) {
try {
if((hostCheckBox != null) && hostCheckBox.isSelected()) {
updateTimingMetricGraphs();
}
} catch (HealthMonitorException ex) {
logger.log(Level.SEVERE, "Error populating timing metric panel", ex);
}
}
});
// Create the host checkbox
hostCheckBox = new JCheckBox(Bundle.HealthMonitorDashboard_createTimingControlPanel_filterByHost());
hostCheckBox.setSelected(false);
hostComboBox.setEnabled(false);
// Set up the listener on the checkbox
hostCheckBox.addActionListener(new ActionListener() {
@Override
public void actionPerformed(ActionEvent arg0) {
try {
hostComboBox.setEnabled(hostCheckBox.isSelected());
updateTimingMetricGraphs();
} catch (HealthMonitorException ex) {
logger.log(Level.SEVERE, "Error populating timing metric panel", ex);
}
}
});
// Create the checkbox for showing the trend line
showTrendLineCheckBox = new JCheckBox(Bundle.HealthMonitorDashboard_createTimingControlPanel_showTrendLine());
showTrendLineCheckBox.setSelected(true);
// Set up the listener on the checkbox
showTrendLineCheckBox.addActionListener(new ActionListener() {
@Override
public void actionPerformed(ActionEvent arg0) {
try {
updateTimingMetricGraphs();
} catch (HealthMonitorException ex) {
logger.log(Level.SEVERE, "Error populating timing metric panel", ex);
}
}
});
// Create the checkbox for omitting outliers
skipOutliersCheckBox = new JCheckBox(Bundle.HealthMonitorDashboard_createTimingControlPanel_skipOutliers());
skipOutliersCheckBox.setSelected(false);
// Set up the listener on the checkbox
skipOutliersCheckBox.addActionListener(new ActionListener() {
@Override
public void actionPerformed(ActionEvent arg0) {
try {
updateTimingMetricGraphs();
} catch (HealthMonitorException ex) {
logger.log(Level.SEVERE, "Error populating timing metric panel", ex);
}
}
});
// Add the date range combo box and label to the panel
timingControlPanel.add(new JLabel(Bundle.HealthMonitorDashboard_createTimingControlPanel_maxDays()));
timingControlPanel.add(dateComboBox);
// Put some space between the elements
timingControlPanel.add(Box.createHorizontalStrut(100));
// Add the host combo box and checkbox to the panel
timingControlPanel.add(hostCheckBox);
timingControlPanel.add(hostComboBox);
// Put some space between the elements
timingControlPanel.add(Box.createHorizontalStrut(100));
// Add the skip outliers checkbox
timingControlPanel.add(this.showTrendLineCheckBox);
// Put some space between the elements
timingControlPanel.add(Box.createHorizontalStrut(100));
// Add the skip outliers checkbox
timingControlPanel.add(this.skipOutliersCheckBox);
return timingControlPanel;
}
/**
* Update the timing graphs.
* @throws HealthMonitorException
*/
@NbBundle.Messages({"HealthMonitorDashboard.updateTimingMetricGraphs.noData=No data to display"})
private void updateTimingMetricGraphs() throws HealthMonitorException {
// Clear out any old graphs
graphPanel.removeAll();
if(timingData.keySet().isEmpty()) {
// There are no timing metrics in the database
graphPanel.add(new JLabel(Bundle.HealthMonitorDashboard_updateTimingMetricGraphs_noData()));
return;
}
for(String metricName:timingData.keySet()) {
// If necessary, trim down the list of results to fit the selected time range
List<EnterpriseHealthMonitor.DatabaseTimingResult> intermediateTimingDataForDisplay;
if(dateComboBox.getSelectedItem() != null) {
DateRange selectedDateRange = DateRange.fromLabel(dateComboBox.getSelectedItem().toString());
long threshold = System.currentTimeMillis() - selectedDateRange.getTimestampRange();
intermediateTimingDataForDisplay = timingData.get(metricName).stream()
.filter(t -> t.getTimestamp() > threshold)
.collect(Collectors.toList());
} else {
intermediateTimingDataForDisplay = timingData.get(metricName);
}
// Get the name of the selected host, if there is one.
// The graph always uses the data from all hosts to generate the x and y scales
// so we don't filter anything out here.
String hostToDisplay = null;
if(hostCheckBox.isSelected() && (hostComboBox.getSelectedItem() != null)) {
hostToDisplay = hostComboBox.getSelectedItem().toString();
}
// Generate the graph
TimingMetricGraphPanel singleTimingGraphPanel = new TimingMetricGraphPanel(intermediateTimingDataForDisplay,
hostToDisplay, true, metricName, skipOutliersCheckBox.isSelected(), showTrendLineCheckBox.isSelected());
singleTimingGraphPanel.setPreferredSize(new Dimension(700,200));
graphPanel.add(singleTimingGraphPanel);
}
graphPanel.revalidate();
graphPanel.repaint();
}
/**
* Create the admin panel.
* This allows the health monitor to be enabled and disabled.
* @return
*/
@NbBundle.Messages({"HealthMonitorDashboard.createAdminPanel.enableButton=Enable monitor",
"HealthMonitorDashboard.createAdminPanel.disableButton=Disable monitor"})
private JPanel createAdminPanel() {
JPanel adminPanel = new JPanel();
adminPanel.setBorder(BorderFactory.createEtchedBorder());
// Create the buttons for enabling/disabling the monitor
JButton enableButton = new JButton(Bundle.HealthMonitorDashboard_createAdminPanel_enableButton());
JButton disableButton = new JButton(Bundle.HealthMonitorDashboard_createAdminPanel_disableButton());
boolean isEnabled = EnterpriseHealthMonitor.monitorIsEnabled();
enableButton.setEnabled(! isEnabled);
disableButton.setEnabled(isEnabled);
// Set up a listener on the enable button
enableButton.addActionListener(new ActionListener() {
@Override
public void actionPerformed(ActionEvent arg0) {
try {
dialog.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR));
EnterpriseHealthMonitor.setEnabled(true);
redisplay();
} catch (HealthMonitorException ex) {
logger.log(Level.SEVERE, "Error enabling monitoring", ex);
} finally {
dialog.setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR));
}
}
});
// Set up a listener on the disable button
disableButton.addActionListener(new ActionListener() {
@Override
public void actionPerformed(ActionEvent arg0) {
try {
dialog.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR));
EnterpriseHealthMonitor.setEnabled(false);
redisplay();
} catch (HealthMonitorException ex) {
logger.log(Level.SEVERE, "Error disabling monitoring", ex);
} finally {
dialog.setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR));
}
}
});
// Add the buttons
adminPanel.add(enableButton);
adminPanel.add(Box.createHorizontalStrut(25));
adminPanel.add(disableButton);
return adminPanel;
}
/**
* Possible date ranges for the metrics in the UI
*/
@NbBundle.Messages({"HealthMonitorDashboard.DateRange.oneMonth=One month",
"HealthMonitorDashboard.DateRange.twoWeeks=Two weeks",
"HealthMonitorDashboard.DateRange.oneWeek=One week",
"HealthMonitorDashboard.DateRange.oneDay=One day"})
private enum DateRange {
ONE_DAY(Bundle.HealthMonitorDashboard_DateRange_oneDay(), 1),
ONE_WEEK(Bundle.HealthMonitorDashboard_DateRange_oneWeek(), 7),
TWO_WEEKS(Bundle.HealthMonitorDashboard_DateRange_twoWeeks(), 14),
ONE_MONTH(Bundle.HealthMonitorDashboard_DateRange_oneMonth(), 31);
private final String label;
private final long numberOfDays;
private static final long MILLISECONDS_PER_DAY = 1000 * 60 * 60 * 24;
DateRange(String label, long numberOfDays) {
this.label = label;
this.numberOfDays = numberOfDays;
}
/**
* Get the name for display in the UI
* @return the name
*/
String getLabel() {
return label;
}
/**
* Get the number of milliseconds represented by this date range.
* Compare the timestamps to ((current time in millis) - (this value)) to
* determine if they are in the range
* @return the time range in milliseconds
*/
long getTimestampRange() {
if (numberOfDays > 0) {
return numberOfDays * MILLISECONDS_PER_DAY;
} else {
return Long.MAX_VALUE;
}
}
/**
* Get the maximum range for this enum.
* This should be used for querying the database for the timing metrics to display.
* @return the maximum range in milliseconds
*/
static long getMaximumTimestampRange() {
long maxRange = Long.MIN_VALUE;
for (DateRange dateRange : DateRange.values()) {
if (dateRange.getTimestampRange() > maxRange) {
maxRange = dateRange.getTimestampRange();
}
}
return maxRange;
}
static DateRange fromLabel(String text) {
for (DateRange dateRange : DateRange.values()) {
if (dateRange.label.equalsIgnoreCase(text)) {
return dateRange;
}
}
return ONE_DAY; // If the comparison failed, return a default
}
}
}

510
Core/src/org/sleuthkit/autopsy/healthmonitor/TimingMetricGraphPanel.java Normal file
View File

@ -0,0 +1,510 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2018 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.healthmonitor;
import java.awt.BasicStroke;
import java.awt.Color;
import java.awt.FontMetrics;
import java.awt.Graphics;
import java.awt.Graphics2D;
import java.awt.Point;
import java.awt.RenderingHints;
import java.awt.Stroke;
import java.util.Collections;
import java.util.stream.Collectors;
import java.util.Comparator;
import java.util.ArrayList;
import java.util.List;
import java.util.Calendar;
import java.util.GregorianCalendar;
import javax.swing.JPanel;
import org.sleuthkit.autopsy.coreutils.Logger;
import java.util.logging.Level;
import java.util.TimeZone;
import java.util.concurrent.TimeUnit;
import org.openide.util.NbBundle;
import org.sleuthkit.autopsy.healthmonitor.EnterpriseHealthMonitor.DatabaseTimingResult;
/**
* Creates a graph of the given timing metric data
*/
class TimingMetricGraphPanel extends JPanel {
private final static Logger logger = Logger.getLogger(TimingMetricGraphPanel.class.getName());
private final int padding = 25;
private final int labelPadding = 25;
private final Color lineColor = new Color(0x12, 0x20, 0xdb, 180);
private final Color gridColor = new Color(200, 200, 200, 200);
private final Color trendLineColor = new Color(150, 10, 10, 200);
private static final Stroke GRAPH_STROKE = new BasicStroke(2f);
private static final Stroke NARROW_STROKE = new BasicStroke(1f);
private final int pointWidth = 4;
private final int numberYDivisions = 10;
private List<DatabaseTimingResult> timingResults;
private final String metricName;
private final boolean doLineGraph;
private final boolean skipOutliers;
private final boolean showTrendLine;
private String yUnitString;
private TrendLine trendLine;
private final long MILLISECONDS_PER_DAY = 1000 * 60 * 60 * 24;
private final long NANOSECONDS_PER_MILLISECOND = 1000 * 1000;
private long maxTimestamp;
private long minTimestamp;
private double maxMetricTime;
private double minMetricTime;
TimingMetricGraphPanel(List<DatabaseTimingResult> timingResultsFull,
String hostName, boolean doLineGraph, String metricName, boolean skipOutliers, boolean showTrendLine) {
this.doLineGraph = doLineGraph;
this.skipOutliers = skipOutliers;
this.showTrendLine = showTrendLine;
this.metricName = metricName;
if(hostName == null || hostName.isEmpty()) {
timingResults = timingResultsFull;
} else {
timingResults = timingResultsFull.stream()
.filter(t -> t.getHostName().equals(hostName))
.collect(Collectors.toList());
}
if(showTrendLine) {
try {
trendLine = new TrendLine(timingResults);
} catch (HealthMonitorException ex) {
// Log it, set trendLine to null and continue on
logger.log(Level.WARNING, "Can not generate a trend line on empty data set");
trendLine = null;
}
}
// Calculate these using the full data set, to make it easier to compare the results for
// individual hosts. Calculate the average at the same time.
maxMetricTime = Double.MIN_VALUE;
minMetricTime = Double.MAX_VALUE;
maxTimestamp = Long.MIN_VALUE;
minTimestamp = Long.MAX_VALUE;
double averageMetricTime = 0.0;
for (DatabaseTimingResult result : timingResultsFull) {
maxMetricTime = Math.max(maxMetricTime, result.getAverage());
minMetricTime = Math.min(minMetricTime, result.getAverage());
maxTimestamp = Math.max(maxTimestamp, result.getTimestamp());
minTimestamp = Math.min(minTimestamp, result.getTimestamp());
averageMetricTime += result.getAverage();
}
averageMetricTime = averageMetricTime / timingResultsFull.size();
// If we're omitting outliers, we may use a different maxMetricTime.
// If the max time is reasonably close to the average, do nothing
if (this.skipOutliers && (maxMetricTime > (averageMetricTime * 5))) {
// Calculate the standard deviation
double intermediateValue = 0.0;
for (DatabaseTimingResult result : timingResultsFull) {
double diff = result.getAverage() - averageMetricTime;
intermediateValue += diff * diff;
}
double standardDeviation = Math.sqrt(intermediateValue / timingResultsFull.size());
maxMetricTime = averageMetricTime + standardDeviation;
}
}
/**
* Setup of the graphics panel:
* Origin (0,0) is at the top left corner
*
* Horizontally (from the left): (padding)(label padding)(the graph)(padding)
* For plotting data on the x-axis, we scale it to the size of the graph and then add the padding and label padding
*
* Vertically (from the top): (padding)(the graph)(label padding)(padding)
* For plotting data on the y-axis, we subtract from the max value in the graph and then scale to the size of the graph
* @param g
*/
@NbBundle.Messages({"TimingMetricGraphPanel.paintComponent.nanoseconds=nanoseconds",
"TimingMetricGraphPanel.paintComponent.microseconds=microseconds",
"TimingMetricGraphPanel.paintComponent.milliseconds=milliseconds",
"TimingMetricGraphPanel.paintComponent.seconds=seconds",
"TimingMetricGraphPanel.paintComponent.minutes=minutes",
"TimingMetricGraphPanel.paintComponent.hours=hours",
"TimingMetricGraphPanel.paintComponent.displayingTime=displaying time in "})
@Override
protected void paintComponent(Graphics g) {
super.paintComponent(g);
Graphics2D g2 = (Graphics2D) g;
g2.setRenderingHint(RenderingHints.KEY_ANTIALIASING, RenderingHints.VALUE_ANTIALIAS_ON);
// Get the max and min timestamps to create the x-axis.
// We add a small buffer to each side so the data won't overwrite the axes.
double maxValueOnXAxis = maxTimestamp + TimeUnit.HOURS.toMillis(2); // Two hour buffer
double minValueOnXAxis = minTimestamp - TimeUnit.HOURS.toMillis(2); // Two hour buffer
// Get the max and min times to create the y-axis
// We add a small buffer to each side so the data won't overwrite the axes.
double maxValueOnYAxis = maxMetricTime;
double minValueOnYAxis = minMetricTime;
minValueOnYAxis = Math.max(0, minValueOnYAxis - (maxValueOnYAxis * 0.1));
maxValueOnYAxis = maxValueOnYAxis * 1.1;
// The graph itself has the following corners:
// (padding + label padding, padding + font height) -> top left
// (padding + label padding, getHeight() - label padding - padding) -> bottom left
// (getWidth() - padding, padding + font height) -> top right
// (padding + label padding, getHeight() - label padding - padding) -> bottom right
int leftGraphPadding = padding + labelPadding;
int rightGraphPadding = padding;
int topGraphPadding = padding + g2.getFontMetrics().getHeight();
int bottomGraphPadding = labelPadding;
// Calculate the scale for each axis.
// The size of the graph area is the width/height of the panel minus any padding.
// The scale is calculated based on this size of the graph compared to the data range.
// For example:
// getWidth() = 575 => graph width = 500
// If our max x value to plot is 10000 and our min is 0, then the xScale would be 0.05 - i.e.,
// our original x values will be multipled by 0.05 to translate them to an x-coordinate in the
// graph (plus the padding)
int graphWidth = getWidth() - leftGraphPadding - rightGraphPadding;
int graphHeight = getHeight() - topGraphPadding - bottomGraphPadding;
double xScale = ((double) graphWidth) / (maxValueOnXAxis - minValueOnXAxis);
double yScale = ((double) graphHeight) / (maxValueOnYAxis - minValueOnYAxis);
// Check if we should use a scale other than milliseconds
// The idea here is to pick the scale that would most commonly be used to
// represent the middle of our data. For example, if the middle of the graph
// would be 45,000,000 nanoseconds, then we would use milliseconds for the
// y-axis.
long middleOfGraphNano = (long)((minValueOnYAxis + (maxValueOnYAxis - minValueOnYAxis) / 2.0) * NANOSECONDS_PER_MILLISECOND);
double yLabelScale;
if(middleOfGraphNano < TimeUnit.MICROSECONDS.toNanos(1)) {
yUnitString = Bundle.TimingMetricGraphPanel_paintComponent_nanoseconds();
yLabelScale = TimeUnit.MILLISECONDS.toNanos(1);
} else if (TimeUnit.NANOSECONDS.toMicros(middleOfGraphNano) < TimeUnit.MILLISECONDS.toMicros(1)) {
yUnitString = Bundle.TimingMetricGraphPanel_paintComponent_microseconds();
yLabelScale = TimeUnit.MILLISECONDS.toMicros(1);
} else if (TimeUnit.NANOSECONDS.toMillis(middleOfGraphNano) < TimeUnit.SECONDS.toMillis(1)) {
yUnitString = Bundle.TimingMetricGraphPanel_paintComponent_milliseconds();
yLabelScale = 1;
} else if (TimeUnit.NANOSECONDS.toSeconds(middleOfGraphNano) < TimeUnit.MINUTES.toSeconds(1)) {
yUnitString = Bundle.TimingMetricGraphPanel_paintComponent_seconds();
yLabelScale = 1.0 / TimeUnit.SECONDS.toMillis(1);
} else if (TimeUnit.NANOSECONDS.toMinutes(middleOfGraphNano) < TimeUnit.HOURS.toMinutes(1)) {
yUnitString = Bundle.TimingMetricGraphPanel_paintComponent_minutes();
yLabelScale = 1.0 / (TimeUnit.MINUTES.toMillis(1));
} else {
yUnitString = Bundle.TimingMetricGraphPanel_paintComponent_hours();
yLabelScale = 1.0 / (TimeUnit.HOURS.toMillis(1));
}
// Draw white background
g2.setColor(Color.WHITE);
g2.fillRect(leftGraphPadding, topGraphPadding, graphWidth, graphHeight);
// Create hatch marks and grid lines for y axis.
int labelWidth;
int positionForMetricNameLabel = 0;
for (int i = 0; i < numberYDivisions + 1; i++) {
int x0 = leftGraphPadding;
int x1 = pointWidth + leftGraphPadding;
int y0 = getHeight() - ((i * graphHeight) / numberYDivisions + bottomGraphPadding);
int y1 = y0;
if ( ! timingResults.isEmpty()) {
// Draw the grid line
g2.setColor(gridColor);
g2.drawLine(leftGraphPadding + 1 + pointWidth, y0, getWidth() - rightGraphPadding, y1);
// Create the label
g2.setColor(Color.BLACK);
double yValue = minValueOnYAxis + ((maxValueOnYAxis - minValueOnYAxis) * ((i * 1.0) / numberYDivisions));
String yLabel = Double.toString(((int) (yValue * 100 * yLabelScale)) / 100.0);
FontMetrics fontMetrics = g2.getFontMetrics();
labelWidth = fontMetrics.stringWidth(yLabel);
g2.drawString(yLabel, x0 - labelWidth - 5, y0 + (fontMetrics.getHeight() / 2) - 3);
// The nicest looking alignment for this label seems to be left-aligned with the top
// y-axis label. Save this position to be used to write the label later.
if (i == numberYDivisions) {
positionForMetricNameLabel = x0 - labelWidth - 5;
}
}
// Draw the small hatch mark
g2.setColor(Color.BLACK);
g2.drawLine(x0, y0, x1, y1);
}
// On the x-axis, the farthest right grid line should represent midnight preceding the last recorded value
Calendar maxDate = new GregorianCalendar();
maxDate.setTimeInMillis(maxTimestamp);
maxDate.set(Calendar.HOUR_OF_DAY, 0);
maxDate.set(Calendar.MINUTE, 0);
maxDate.set(Calendar.SECOND, 0);
maxDate.set(Calendar.MILLISECOND, 0);
long maxMidnightInMillis = maxDate.getTimeInMillis();
// We don't want to display more than 20 grid lines. If we have more
// data then that, put multiple days within one division
long totalDays = (maxMidnightInMillis - (long)minValueOnXAxis) / MILLISECONDS_PER_DAY;
long daysPerDivision;
if(totalDays <= 20) {
daysPerDivision = 1;
} else {
daysPerDivision = (totalDays / 20);
if((totalDays % 20) != 0) {
daysPerDivision++;
}
}
// Draw the vertical grid lines and labels
// The vertical grid lines will be at midnight, and display the date underneath them
// At present we use GMT because of some complications with daylight savings time.
for (long currentDivision = maxMidnightInMillis; currentDivision >= minValueOnXAxis; currentDivision -= MILLISECONDS_PER_DAY * daysPerDivision) {
int x0 = (int) ((currentDivision - minValueOnXAxis) * xScale + leftGraphPadding);
int x1 = x0;
int y0 = getHeight() - bottomGraphPadding;
int y1 = y0 - pointWidth;
// Draw the light grey grid line
g2.setColor(gridColor);
g2.drawLine(x0, getHeight() - bottomGraphPadding - 1 - pointWidth, x1, topGraphPadding);
// Draw the hatch mark
g2.setColor(Color.BLACK);
g2.drawLine(x0, y0, x1, y1);
// Draw the label
Calendar thisDate = new GregorianCalendar();
thisDate.setTimeZone(TimeZone.getTimeZone("GMT")); // Stick with GMT to avoid daylight savings issues
thisDate.setTimeInMillis(currentDivision);
int month = thisDate.get(Calendar.MONTH) + 1;
int day = thisDate.get(Calendar.DAY_OF_MONTH);
String xLabel = month + "/" + day;
FontMetrics metrics = g2.getFontMetrics();
labelWidth = metrics.stringWidth(xLabel);
g2.drawString(xLabel, x0 - labelWidth / 2, y0 + metrics.getHeight() + 3);
}
// Create x and y axes
g2.setColor(Color.BLACK);
g2.drawLine(leftGraphPadding, getHeight() - bottomGraphPadding, leftGraphPadding, topGraphPadding);
g2.drawLine(leftGraphPadding, getHeight() - bottomGraphPadding, getWidth() - rightGraphPadding, getHeight() - bottomGraphPadding);
// Create the points to plot
List<Point> graphPoints = new ArrayList<>();
for (int i = 0; i < timingResults.size(); i++) {
double metricTime = timingResults.get(i).getAverage();
int x1 = (int) ((timingResults.get(i).getTimestamp() - minValueOnXAxis) * xScale + leftGraphPadding);
int y1 = (int) ((maxValueOnYAxis - metricTime) * yScale + topGraphPadding);
graphPoints.add(new Point(x1, y1));
}
// Sort the points
Collections.sort(graphPoints, new Comparator<Point>() {
@Override
public int compare(Point o1, Point o2) {
if(o1.getX() > o2.getX()) {
return 1;
} else if (o1.getX() < o2.getX()) {
return -1;
}
return 0;
}
});
// Draw the selected type of graph. If there's only one data point,
// draw that single point.
g2.setStroke(NARROW_STROKE);
g2.setColor(lineColor);
if(doLineGraph && graphPoints.size() > 1) {
for (int i = 0; i < graphPoints.size() - 1; i++) {
int x1 = graphPoints.get(i).x;
int y1 = graphPoints.get(i).y;
int x2 = graphPoints.get(i + 1).x;
int y2 = graphPoints.get(i + 1).y;
g2.drawLine(x1, y1, x2, y2);
}
} else {
for (int i = 0; i < graphPoints.size(); i++) {
int x = graphPoints.get(i).x - pointWidth / 2;
int y = graphPoints.get(i).y - pointWidth / 2;
int ovalW = pointWidth;
int ovalH = pointWidth;
g2.fillOval(x, y, ovalW, ovalH);
}
}
// Draw the trend line.
// Don't draw anything if we don't have at least two data points.
if(showTrendLine && (trendLine != null) && (timingResults.size() > 1)) {
double x0value = minValueOnXAxis;
double y0value = trendLine.getExpectedValueAt(x0value);
if (y0value < minValueOnYAxis) {
try {
y0value = minValueOnYAxis;
x0value = trendLine.getXGivenY(y0value);
} catch (HealthMonitorException ex) {
// The exception is caused by a slope of zero on the trend line, which
// shouldn't be able to happen at the same time as having a trend line that dips below the y-axis.
// If it does, log a warning but continue on with the original values.
logger.log(Level.WARNING, "Error plotting trend line", ex);
}
} else if (y0value > maxValueOnYAxis) {
try {
y0value = minValueOnYAxis;
x0value = trendLine.getXGivenY(y0value);
} catch (HealthMonitorException ex) {
// The exception is caused by a slope of zero on the trend line, which
// shouldn't be able to happen at the same time as having a trend line that dips below the y-axis.
// If it does, log a warning but continue on with the original values.
logger.log(Level.WARNING, "Error plotting trend line", ex);
}
}
int x0 = (int) ((x0value - minValueOnXAxis) * xScale) + leftGraphPadding;
int y0 = (int) ((maxValueOnYAxis - y0value) * yScale + topGraphPadding);
double x1value = maxValueOnXAxis;
double y1value = trendLine.getExpectedValueAt(maxValueOnXAxis);
if (y1value < minValueOnYAxis) {
try {
y1value = minValueOnYAxis;
x1value = trendLine.getXGivenY(y1value);
} catch (HealthMonitorException ex) {
// The exception is caused by a slope of zero on the trend line, which
// shouldn't be able to happen at the same time as having a trend line that dips below the y-axis.
// If it does, log a warning but continue on with the original values.
logger.log(Level.WARNING, "Error plotting trend line", ex);
}
} else if (y1value > maxValueOnYAxis) {
try {
y1value = maxValueOnYAxis;
x1value = trendLine.getXGivenY(y1value);
} catch (HealthMonitorException ex) {
// The exception is caused by a slope of zero on the trend line, which
// shouldn't be able to happen at the same time as having a trend line that dips below the y-axis.
// If it does, log a warning but continue on with the original values.
logger.log(Level.WARNING, "Error plotting trend line", ex);
}
}
int x1 = (int) ((x1value - minValueOnXAxis) * xScale) + leftGraphPadding;
int y1 = (int) ((maxValueOnYAxis - y1value) * yScale + topGraphPadding);
g2.setStroke(GRAPH_STROKE);
g2.setColor(trendLineColor);
g2.drawLine(x0, y0, x1, y1);
}
// The graph lines may have extended up past the bounds of the graph. Overwrite that
// area with the original background color.
g2.setColor(this.getBackground());
g2.fillRect(leftGraphPadding, 0, graphWidth, topGraphPadding);
// Write the scale. Do this after we erase the top block of the graph.
g2.setColor(Color.BLACK);
String scaleStr = Bundle.TimingMetricGraphPanel_paintComponent_displayingTime() + yUnitString;
String titleStr = metricName + " - " + scaleStr;
g2.drawString(titleStr, positionForMetricNameLabel, padding);
}
/**
* Class to generate a linear trend line from timing metric data.
*
* Formula for the linear trend line:
* (x,y) = (timestamp, metric time)
* n = total number of metrics
*
* slope = ( n * Σ(xy) - Σx * Σy ) / ( n * Σ(x^2) - (Σx)^2 )
*
* y intercept = ( Σy - (slope) * Σx ) / n
*/
private class TrendLine {
double slope;
double yInt;
TrendLine(List<DatabaseTimingResult> timingResults) throws HealthMonitorException {
if((timingResults == null) || timingResults.isEmpty()) {
throw new HealthMonitorException("Can not generate trend line for empty/null data set");
}
// Calculate intermediate values
int n = timingResults.size();
double sumX = 0;
double sumY = 0;
double sumXY = 0;
double sumXsquared = 0;
for(int i = 0;i < n;i++) {
double x = timingResults.get(i).getTimestamp();
double y = timingResults.get(i).getAverage();
sumX += x;
sumY += y;
sumXY += x * y;
sumXsquared += x * x;
}
// Calculate slope
// With only one measurement, the denominator will end being zero in the formula.
// Use a horizontal line in this case (or any case where the denominator is zero)
double denominator = n * sumXsquared - sumX * sumX;
if (denominator != 0) {
slope = (n * sumXY - sumX * sumY) / denominator;
} else {
slope = 0;
}
// Calculate y intercept
yInt = (sumY - slope * sumX) / n;
}
/**
* Get the expected y value for a given x
* @param x x coordinate of the point on the trend line
* @return expected y coordinate of this point on the trend line
*/
double getExpectedValueAt(double x) {
return (slope * x + yInt);
}
/**
* Get the x-coordinate for a given Y-coordinate.
* Should only be necessary when the trend line does not fit on the graph
* @param y the y coordinate
* @return expected x coordinate for the given y
* @throws HealthMonitorException
*/
double getXGivenY(double y) throws HealthMonitorException {
if (slope != 0.0) {
return ((y - yInt) / slope);
} else {
throw new HealthMonitorException("Attempted division by zero in trend line calculation");
}
}
}
}

119
Core/src/org/sleuthkit/autopsy/ingest/DataSourceIngestJob.java
View File

@ -18,6 +18,7 @@
*/
package org.sleuthkit.autopsy.ingest;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
@ -37,6 +38,9 @@ import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.coreutils.NetworkUtils;
import org.sleuthkit.autopsy.ingest.DataSourceIngestPipeline.PipelineModule;
import org.sleuthkit.autopsy.ingest.IngestJob.CancellationReason;
import org.sleuthkit.autopsy.ingest.IngestTasksScheduler.IngestJobTasksSnapshot;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.Content;
import org.sleuthkit.datamodel.IngestJobInfo;
@ -51,7 +55,7 @@ import org.sleuthkit.autopsy.modules.interestingitems.FilesSet;
* Encapsulates a data source and the ingest module pipelines used to process
* it.
*/
final class DataSourceIngestJob {
public final class DataSourceIngestJob {
private static final Logger logger = Logger.getLogger(DataSourceIngestJob.class.getName());
@ -1079,71 +1083,90 @@ final class DataSourceIngestJob {
* @return An ingest job statistics object.
*/
Snapshot getSnapshot(boolean getIngestTasksSnapshot) {
return new Snapshot(getIngestTasksSnapshot);
/**
* Determine whether file ingest is running at the time of this snapshot
* and determine the earliest file ingest level pipeline start time, if
* file ingest was started at all.
*/
boolean fileIngestRunning = false;
Date fileIngestStartTime = null;
for (FileIngestPipeline pipeline : this.fileIngestPipelines) {
if (pipeline.isRunning()) {
fileIngestRunning = true;
}
Date pipelineStartTime = pipeline.getStartTime();
if (null != pipelineStartTime && (null == fileIngestStartTime || pipelineStartTime.before(fileIngestStartTime))) {
fileIngestStartTime = pipelineStartTime;
}
}
long processedFilesCount = 0;
long estimatedFilesToProcessCount = 0;
long snapShotTime = new Date().getTime();
IngestJobTasksSnapshot tasksSnapshot = null;
if (getIngestTasksSnapshot) {
synchronized (fileIngestProgressLock) {
processedFilesCount = this.processedFiles;
estimatedFilesToProcessCount = this.estimatedFilesToProcess;
snapShotTime = new Date().getTime();
}
tasksSnapshot = DataSourceIngestJob.taskScheduler.getTasksSnapshotForJob(id);
}
return new Snapshot(this.dataSource.getName(), id, createTime,
getCurrentDataSourceIngestModule(), fileIngestRunning, fileIngestStartTime,
cancelled, cancellationReason, cancelledDataSourceIngestModules,
processedFilesCount, estimatedFilesToProcessCount, snapShotTime, tasksSnapshot);
}
/**
* Stores basic diagnostic statistics for a data source ingest job.
*/
final class Snapshot {
public static final class Snapshot implements Serializable {
private static final long serialVersionUID = 1L;
private final String dataSource;
private final long jobId;
private final long jobStartTime;
private final long snapShotTime;
private final DataSourceIngestPipeline.PipelineModule dataSourceLevelIngestModule;
private boolean fileIngestRunning;
private Date fileIngestStartTime;
transient private final PipelineModule dataSourceLevelIngestModule;
private final boolean fileIngestRunning;
private final Date fileIngestStartTime;
private final long processedFiles;
private final long estimatedFilesToProcess;
private final IngestTasksScheduler.IngestJobTasksSnapshot tasksSnapshot;
private final boolean jobCancelled;
private final IngestJob.CancellationReason jobCancellationReason;
private final List<String> cancelledDataSourceModules;
private final IngestJobTasksSnapshot tasksSnapshot;
transient private final boolean jobCancelled;
transient private final CancellationReason jobCancellationReason;
transient private final List<String> cancelledDataSourceModules;
/**
* Constructs an object to store basic diagnostic statistics for a data
* source ingest job.
*/
Snapshot(boolean getIngestTasksSnapshot) {
this.dataSource = DataSourceIngestJob.this.dataSource.getName();
this.jobId = DataSourceIngestJob.this.id;
this.jobStartTime = DataSourceIngestJob.this.createTime;
this.dataSourceLevelIngestModule = DataSourceIngestJob.this.getCurrentDataSourceIngestModule();
Snapshot(String dataSourceName, long jobId, long jobStartTime, PipelineModule dataSourceIngestModule,
boolean fileIngestRunning, Date fileIngestStartTime,
boolean jobCancelled, CancellationReason cancellationReason, List<String> cancelledModules,
long processedFiles, long estimatedFilesToProcess,
long snapshotTime, IngestJobTasksSnapshot tasksSnapshot) {
this.dataSource = dataSourceName;
this.jobId = jobId;
this.jobStartTime = jobStartTime;
this.dataSourceLevelIngestModule = dataSourceIngestModule;
/**
* Determine whether file ingest is running at the time of this
* snapshot and determine the earliest file ingest level pipeline
* start time, if file ingest was started at all.
*/
for (FileIngestPipeline pipeline : DataSourceIngestJob.this.fileIngestPipelines) {
if (pipeline.isRunning()) {
this.fileIngestRunning = true;
}
Date pipelineStartTime = pipeline.getStartTime();
if (null != pipelineStartTime && (null == this.fileIngestStartTime || pipelineStartTime.before(this.fileIngestStartTime))) {
this.fileIngestStartTime = pipelineStartTime;
}
}
this.jobCancelled = cancelled;
this.fileIngestRunning = fileIngestRunning;
this.fileIngestStartTime = fileIngestStartTime;
this.jobCancelled = jobCancelled;
this.jobCancellationReason = cancellationReason;
this.cancelledDataSourceModules = new ArrayList<>(DataSourceIngestJob.this.cancelledDataSourceIngestModules);
this.cancelledDataSourceModules = cancelledModules;
if (getIngestTasksSnapshot) {
synchronized (DataSourceIngestJob.this.fileIngestProgressLock) {
this.processedFiles = DataSourceIngestJob.this.processedFiles;
this.estimatedFilesToProcess = DataSourceIngestJob.this.estimatedFilesToProcess;
this.snapShotTime = new Date().getTime();
}
this.tasksSnapshot = DataSourceIngestJob.taskScheduler.getTasksSnapshotForJob(this.jobId);
} else {
this.processedFiles = 0;
this.estimatedFilesToProcess = 0;
this.snapShotTime = new Date().getTime();
this.tasksSnapshot = null;
}
this.processedFiles = processedFiles;
this.estimatedFilesToProcess = estimatedFilesToProcess;
this.snapShotTime = snapshotTime;
this.tasksSnapshot = tasksSnapshot;
}
/**
@ -1190,11 +1213,11 @@ final class DataSourceIngestJob {
return this.dataSourceLevelIngestModule;
}
boolean fileIngestIsRunning() {
boolean getFileIngestIsRunning() {
return this.fileIngestRunning;
}
Date fileIngestStartTime() {
Date getFileIngestStartTime() {
return this.fileIngestStartTime;
}

4
Core/src/org/sleuthkit/autopsy/ingest/IngestJob.java
View File

@ -355,10 +355,10 @@ public final class IngestJob {
dataSourceModule = new DataSourceIngestModuleHandle(dataSourceJobs.get(snapshot.getJobId()), module);
}
}
if (snapshot.fileIngestIsRunning()) {
if (snapshot.getFileIngestIsRunning()) {
fileIngestRunning = true;
}
Date childFileIngestStartTime = snapshot.fileIngestStartTime();
Date childFileIngestStartTime = snapshot.getFileIngestStartTime();
if (null != childFileIngestStartTime && (null == fileIngestStartTime || childFileIngestStartTime.before(fileIngestStartTime))) {
fileIngestStartTime = childFileIngestStartTime;
}

16
Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java
View File

@ -22,6 +22,7 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder;
import java.awt.EventQueue;
import java.beans.PropertyChangeEvent;
import java.beans.PropertyChangeListener;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@ -107,7 +108,7 @@ import org.sleuthkit.datamodel.Content;
* job progress, and ingest module run times.
*/
@ThreadSafe
public class IngestManager {
public class IngestManager implements IngestProgressSnapshotProvider {
private final static Logger logger = Logger.getLogger(IngestManager.class.getName());
private final static String INGEST_JOB_EVENT_CHANNEL_NAME = "%s-Ingest-Job-Events"; //NON-NLS
@ -756,7 +757,8 @@ public class IngestManager {
*
* @return Map of module name to run time (in milliseconds)
*/
Map<String, Long> getModuleRunTimes() {
@Override
public Map<String, Long> getModuleRunTimes() {
synchronized (ingestModuleRunTimes) {
Map<String, Long> times = new HashMap<>(ingestModuleRunTimes);
return times;
@ -769,7 +771,8 @@ public class IngestManager {
*
* @return A collection of ingest manager ingest task snapshots.
*/
List<IngestThreadActivitySnapshot> getIngestThreadActivitySnapshots() {
@Override
public List<IngestThreadActivitySnapshot> getIngestThreadActivitySnapshots() {
return new ArrayList<>(ingestThreadActivitySnapshots.values());
}
@ -778,7 +781,8 @@ public class IngestManager {
*
* @return A list of ingest job state snapshots.
*/
List<DataSourceIngestJob.Snapshot> getIngestJobSnapshots() {
@Override
public List<DataSourceIngestJob.Snapshot> getIngestJobSnapshots() {
List<DataSourceIngestJob.Snapshot> snapShots = new ArrayList<>();
synchronized (ingestJobsById) {
ingestJobsById.values().forEach((job) -> {
@ -916,7 +920,9 @@ public class IngestManager {
* running in an ingest thread.
*/
@Immutable
static final class IngestThreadActivitySnapshot {
public static final class IngestThreadActivitySnapshot implements Serializable {
private static final long serialVersionUID = 1L;
private final long threadId;
private final Date startTime;

19
Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotDialog.java
View File

@ -50,12 +50,14 @@ public final class IngestProgressSnapshotDialog extends JDialog {
/**
* Constructs an instance of the dialog with its own frame. Could be modal.
* Uses the given provider as the source of data for the dialog.
*
* @param owner - the owner of this dialog. If this dialog should be
* modal, the owner gets set to non modal.
* @param shouldBeModal - true if this should be modal, false otherwise.
* @param provider - the provider to use as the source of data.
*/
public IngestProgressSnapshotDialog(Container owner, Boolean shouldBeModal) {
public IngestProgressSnapshotDialog(Container owner, Boolean shouldBeModal, IngestProgressSnapshotProvider provider) {
super((Window) owner, TITLE, ModalityType.MODELESS);
if (shouldBeModal && owner instanceof JDialog) { // if called from a modal dialog, manipulate the parent be just under this in z order, and not modal.
final JDialog pseudoOwner = (JDialog) owner;
@ -82,7 +84,7 @@ public final class IngestProgressSnapshotDialog extends JDialog {
this.getRootPane().registerKeyboardAction(e -> {
this.dispose();
}, KeyStroke.getKeyStroke(KeyEvent.VK_ESCAPE, 0), JComponent.WHEN_IN_FOCUSED_WINDOW);
add(new IngestProgressSnapshotPanel(this));
add(new IngestProgressSnapshotPanel(this, provider));
pack();
setResizable(false);
if (shouldBeModal) { // if called from a modal dialog, become modal, otherwise don't.
@ -90,4 +92,17 @@ public final class IngestProgressSnapshotDialog extends JDialog {
}
setVisible(true);
}
/**
* Constructs an instance of the dialog with its own frame. Could be modal.
* Uses the internal IngestManager instance as the source of data for the
* dialog
*
* @param owner - the owner of this dialog. If this dialog should be
* modal, the owner gets set to non modal.
* @param shouldBeModal - true if this should be modal, false otherwise.
*/
public IngestProgressSnapshotDialog(Container owner, Boolean shouldBeModal) {
this(owner, shouldBeModal, IngestManager.getInstance());
}
}

12
Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotPanel.java
View File

@ -33,15 +33,17 @@ import org.openide.util.NbBundle;
/**
* A panel that displays ingest task progress snapshots.
*/
public class IngestProgressSnapshotPanel extends javax.swing.JPanel {
class IngestProgressSnapshotPanel extends javax.swing.JPanel {
private final JDialog parent;
private final IngestProgressSnapshotProvider snapshotProvider;
private final IngestThreadActivitySnapshotsTableModel threadActivityTableModel;
private final IngestJobTableModel jobTableModel;
private final ModuleTableModel moduleTableModel;
IngestProgressSnapshotPanel(JDialog parent) {
IngestProgressSnapshotPanel(JDialog parent, IngestProgressSnapshotProvider snapshotProvider) {
this.parent = parent;
this.snapshotProvider = snapshotProvider;
threadActivityTableModel = new IngestThreadActivitySnapshotsTableModel();
jobTableModel = new IngestJobTableModel();
moduleTableModel = new ModuleTableModel();
@ -105,7 +107,7 @@ public class IngestProgressSnapshotPanel extends javax.swing.JPanel {
}
private void refresh() {
snapshots = IngestManager.getInstance().getIngestThreadActivitySnapshots();
snapshots = snapshotProvider.getIngestThreadActivitySnapshots();
fireTableDataChanged();
}
@ -187,7 +189,7 @@ public class IngestProgressSnapshotPanel extends javax.swing.JPanel {
}
private void refresh() {
jobSnapshots = IngestManager.getInstance().getIngestJobSnapshots();
jobSnapshots = snapshotProvider.getIngestJobSnapshots();
fireTableDataChanged();
}
@ -299,7 +301,7 @@ public class IngestProgressSnapshotPanel extends javax.swing.JPanel {
}
private void refresh() {
Map<String, Long> moduleStatMap = IngestManager.getInstance().getModuleRunTimes();
Map<String, Long> moduleStatMap = snapshotProvider.getModuleRunTimes();
moduleStats.clear();
totalTime = 0;
for (String k : moduleStatMap.keySet()) {

49
Core/src/org/sleuthkit/autopsy/ingest/IngestProgressSnapshotProvider.java Normal file
View File

@ -0,0 +1,49 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2018 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.ingest;
import java.util.List;
import java.util.Map;
/**
* Interface that provides a snapshot of ingest progress.
*/
public interface IngestProgressSnapshotProvider {
/**
* Get a snapshot of the state of ingest threads.
*
* @return A list of IngestThreadActivitySnapshot
*/
List<IngestManager.IngestThreadActivitySnapshot> getIngestThreadActivitySnapshots();
/**
* Get a snapshot of the state of ingest jobs.
*
* @return A list of ingest job snapshots.
*/
List<DataSourceIngestJob.Snapshot> getIngestJobSnapshots();
/**
* Gets the cumulative run times for the ingest module.
*
* @return Map of module name to run time (in milliseconds)
*/
Map<String, Long> getModuleRunTimes();
}

22
Core/src/org/sleuthkit/autopsy/ingest/IngestTasksScheduler.java
View File

@ -18,6 +18,7 @@
*/
package org.sleuthkit.autopsy.ingest;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@ -555,7 +556,11 @@ final class IngestTasksScheduler {
* @return
*/
synchronized IngestJobTasksSnapshot getTasksSnapshotForJob(long jobId) {
return new IngestJobTasksSnapshot(jobId);
return new IngestJobTasksSnapshot(jobId, this.dataSourceIngestThreadQueue.countQueuedTasksForJob(jobId),
countTasksForJob(this.rootFileTaskQueue, jobId),
countTasksForJob(this.pendingFileTaskQueue, jobId),
this.fileIngestThreadsQueue.countQueuedTasksForJob(jobId),
this.dataSourceIngestThreadQueue.countRunningTasksForJob(jobId) + this.fileIngestThreadsQueue.countRunningTasksForJob(jobId));
}
/**
@ -825,8 +830,9 @@ final class IngestTasksScheduler {
/**
* A snapshot of ingest tasks data for an ingest job.
*/
class IngestJobTasksSnapshot {
static final class IngestJobTasksSnapshot implements Serializable {
private static final long serialVersionUID = 1L;
private final long jobId;
private final long dsQueueSize;
private final long rootQueueSize;
@ -839,13 +845,13 @@ final class IngestTasksScheduler {
*
* @param jobId The identifier associated with the job.
*/
IngestJobTasksSnapshot(long jobId) {
IngestJobTasksSnapshot(long jobId, long dsQueueSize, long rootQueueSize, long dirQueueSize, long fileQueueSize, long runningListSize) {
this.jobId = jobId;
this.dsQueueSize = IngestTasksScheduler.this.dataSourceIngestThreadQueue.countQueuedTasksForJob(jobId);
this.rootQueueSize = countTasksForJob(IngestTasksScheduler.this.rootFileTaskQueue, jobId);
this.dirQueueSize = countTasksForJob(IngestTasksScheduler.this.pendingFileTaskQueue, jobId);
this.fileQueueSize = IngestTasksScheduler.this.fileIngestThreadsQueue.countQueuedTasksForJob(jobId);;
this.runningListSize = IngestTasksScheduler.this.dataSourceIngestThreadQueue.countRunningTasksForJob(jobId) + IngestTasksScheduler.this.fileIngestThreadsQueue.countRunningTasksForJob(jobId);
this.dsQueueSize = dsQueueSize;
this.rootQueueSize = rootQueueSize;
this.dirQueueSize = dirQueueSize;
this.fileQueueSize = fileQueueSize;
this.runningListSize = runningListSize;
}
/**

25
Core/src/org/sleuthkit/autopsy/modules/encryptiondetection/EncryptionDetectionFileIngestModule.java
View File

@ -65,6 +65,9 @@ final class EncryptionDetectionFileIngestModule extends FileIngestModuleAdapter
private static final int FILE_SIZE_MODULUS = 512;
private static final String DATABASE_FILE_EXTENSION = "db";
private static final int MINIMUM_DATABASE_FILE_SIZE = 65536; //64 KB
private static final String MIME_TYPE_OOXML_PROTECTED = "application/x-ooxml-protected";
private static final String MIME_TYPE_MSWORD = "application/msword";
private static final String MIME_TYPE_MSEXCEL = "application/vnd.ms-excel";
@ -141,8 +144,7 @@ final class EncryptionDetectionFileIngestModule extends FileIngestModuleAdapter
}
} else {
if (isFilePasswordProtected(file)) {
return flagFile(file, BlackboardArtifact.ARTIFACT_TYPE.TSK_ENCRYPTION_DETECTED,
Bundle.EncryptionDetectionFileIngestModule_artifactComment_password());
return flagFile(file, BlackboardArtifact.ARTIFACT_TYPE.TSK_ENCRYPTION_DETECTED, Bundle.EncryptionDetectionFileIngestModule_artifactComment_password());
}
}
}
@ -182,6 +184,9 @@ final class EncryptionDetectionFileIngestModule extends FileIngestModuleAdapter
private IngestModule.ProcessResult flagFile(AbstractFile file, BlackboardArtifact.ARTIFACT_TYPE artifactType, String comment) {
try {
BlackboardArtifact artifact = file.newArtifact(artifactType);
artifact.addAttribute(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_COMMENT,
EncryptionDetectionModuleFactory.getModuleName(), comment)
);
artifact.addAttribute(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_COMMENT,
EncryptionDetectionModuleFactory.getModuleName(), comment));
@ -362,9 +367,21 @@ final class EncryptionDetectionFileIngestModule extends FileIngestModuleAdapter
/*
* Qualify the size.
*/
boolean fileSizeQualified = false;
String fileExtension = file.getNameExtension();
long contentSize = file.getSize();
if (contentSize >= minimumFileSize) {
// Database files qualify at 64 KB minimum for SQLCipher detection.
if (fileExtension.equalsIgnoreCase(DATABASE_FILE_EXTENSION)) {
if (contentSize >= MINIMUM_DATABASE_FILE_SIZE) {
fileSizeQualified = true;
}
} else if (contentSize >= minimumFileSize) {
if (!fileSizeMultipleEnforced || (contentSize % FILE_SIZE_MODULUS) == 0) {
fileSizeQualified = true;
}
}
if (fileSizeQualified) {
/*
* Qualify the entropy.
*/
@ -373,7 +390,7 @@ final class EncryptionDetectionFileIngestModule extends FileIngestModuleAdapter
possiblyEncrypted = true;
}
}
}
return possiblyEncrypted;
}
}

0
Core/test/qa-functional/src/org/sleuthkit/autopsy/centralrepository/datamodel/CentralRepoDatamodelTest.java Normal file → Executable file
View File

0
Core/test/qa-functional/src/org/sleuthkit/autopsy/ingest/IngestFileFiltersTest.java Normal file → Executable file
View File

68
Core/test/qa-functional/src/org/sleuthkit/autopsy/modules/encryptiondetection/EncryptionDetectionTest.java
View File

@ -52,10 +52,12 @@ public class EncryptionDetectionTest extends NbTestCase {
private static final String BITLOCKER_DETECTION_CASE_NAME = "testBitlockerEncryption";
private static final String PASSWORD_DETECTION_CASE_NAME = "PasswordDetectionTest";
private static final String VERACRYPT_DETECTION_CASE_NAME = "VeraCryptDetectionTest";
private static final String SQLCIPHER_DETECTION_CASE_NAME = "SQLCipherDetectionTest";
private final Path BITLOCKER_DETECTION_IMAGE_PATH = Paths.get(this.getDataDir().toString(), "encryption_detection_bitlocker_test.vhd");
private final Path PASSWORD_DETECTION_IMAGE_PATH = Paths.get(this.getDataDir().toString(), "password_detection_test.img");
private final Path VERACRYPT_DETECTION_IMAGE_PATH = Paths.get(this.getDataDir().toString(), "veracrypt_detection_test.vhd");
private final Path SQLCIPHER_DETECTION_IMAGE_PATH = Paths.get(this.getDataDir().toString(), "encryption_detection_sqlcipher_test.vhd");
private boolean testSucceeded;
@ -90,7 +92,7 @@ public class EncryptionDetectionTest extends NbTestCase {
IngestUtils.addDataSource(dataSourceProcessor, BITLOCKER_DETECTION_IMAGE_PATH);
/*
* Create ingest job settings.
* Create ingest job settings and run ingest job.
*/
IngestModuleFactory ingestModuleFactory = new EncryptionDetectionModuleFactory();
IngestModuleIngestJobSettings settings = ingestModuleFactory.getDefaultIngestJobSettings();
@ -164,7 +166,6 @@ public class EncryptionDetectionTest extends NbTestCase {
/*
* Create ingest job settings.
*/
ArrayList<IngestModuleTemplate> templates = new ArrayList<>();
templates.add(IngestUtils.getIngestModuleTemplate(new EncryptionDetectionModuleFactory()));
IngestJobSettings ingestJobSettings = new IngestJobSettings(PASSWORD_DETECTION_CASE_NAME, IngestType.FILES_ONLY, templates);
@ -295,4 +296,67 @@ public class EncryptionDetectionTest extends NbTestCase {
testSucceeded = true;
}
/**
* Test the Encryption Detection module's SQLCipher encryption detection.
*/
public void testSqlCipherEncryption() {
try {
Case openCase = CaseUtils.createAsCurrentCase(BITLOCKER_DETECTION_CASE_NAME);
ImageDSProcessor dataSourceProcessor = new ImageDSProcessor();
IngestUtils.addDataSource(dataSourceProcessor, BITLOCKER_DETECTION_IMAGE_PATH);
/*
* Create ingest job settings.
*/
ArrayList<IngestModuleTemplate> templates = new ArrayList<>();
templates.add(IngestUtils.getIngestModuleTemplate(new EncryptionDetectionModuleFactory()));
IngestJobSettings ingestJobSettings = new IngestJobSettings(PASSWORD_DETECTION_CASE_NAME, IngestType.FILES_ONLY, templates);
IngestUtils.runIngestJob(openCase.getDataSources(), ingestJobSettings);
/*
* Purge specific files to be tested.
*/
FileManager fileManager = openCase.getServices().getFileManager();
List<AbstractFile> results = fileManager.findFiles("%%", "sqlcipher");
assertEquals("Unexpected number of SQLCipher results.", 15, results.size());
for (AbstractFile file : results) {
/*
* Process only non-slack files.
*/
if (file.isFile() && !file.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.SLACK)) {
/*
* Determine which assertions to use for the file based on
* its name.
*/
List<BlackboardArtifact> artifactsList = file.getAllArtifacts();
String[] splitNameArray = file.getName().split("\\.");
if (splitNameArray[0].startsWith("sqlcipher-") && splitNameArray[splitNameArray.length - 1].equals("db")) {
/*
* Check that the SQLCipher database file has one
* TSK_ENCRYPTION_SUSPECTED artifact.
*/
int artifactsListSize = artifactsList.size();
String errorMessage = String.format("File '%s' (objId=%d) has %d artifacts, but 1 was expected.", file.getName(), file.getId(), artifactsListSize);
assertEquals(errorMessage, 1, artifactsListSize);
String artifactTypeName = artifactsList.get(0).getArtifactTypeName();
errorMessage = String.format("File '%s' (objId=%d) has an unexpected '%s' artifact.", file.getName(), file.getId(), artifactTypeName);
assertEquals(errorMessage, BlackboardArtifact.ARTIFACT_TYPE.TSK_ENCRYPTION_SUSPECTED.toString(), artifactTypeName);
} else {
/*
* Check that the file has no artifacts.
*/
int artifactsListSize = artifactsList.size();
String errorMessage = String.format("File '%s' (objId=%d) has %d artifacts, but none were expected.", file.getName(), file.getId(), artifactsListSize);
assertEquals(errorMessage, 0, artifactsListSize);
}
}
}
} catch (TskCoreException ex) {
Exceptions.printStackTrace(ex);
Assert.fail(ex);
}
}
}

2
Core/test/qa-functional/src/org/sleuthkit/autopsy/testutils/CaseUtils.java
View File

@ -38,8 +38,6 @@ import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException;
*/
public final class CaseUtils {
private static final String PRESERVE_CASE_DATA_LIST_FILE_NAME = ".preserve";
/**
* Create a case case directory and case for the given case name.
*

1
Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AinStatusNode.java
View File

@ -146,6 +146,7 @@ final class AinStatusNode extends AbstractNode {
List<Action> actions = new ArrayList<>();
if (AutoIngestDashboard.isAdminAutoIngestDashboard()) {
if (nodeState.getState() == AutoIngestNodeState.State.PAUSED_BY_REQUEST
|| nodeState.getState() == AutoIngestNodeState.State.PAUSE_REQUESTED
|| nodeState.getState() == AutoIngestNodeState.State.PAUSED_DUE_TO_SYSTEM_ERROR
|| nodeState.getState() == AutoIngestNodeState.State.RUNNING) {
actions.add(new AutoIngestAdminActions.AutoIngestNodeControlAction.PauseResumeAction(nodeState));

194
Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestAdminActions.java
View File

@ -19,23 +19,28 @@
package org.sleuthkit.autopsy.experimental.autoingest;
import java.awt.Cursor;
import java.awt.Desktop;
import java.awt.EventQueue;
import java.awt.event.ActionEvent;
import java.io.IOException;
import java.nio.file.Path;
import java.util.logging.Level;
import javax.swing.AbstractAction;
import javax.swing.JOptionPane;
import org.openide.util.NbBundle;
import org.openide.windows.WindowManager;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil;
import org.sleuthkit.autopsy.experimental.autoingest.AutoIngestMonitor.AutoIngestNodeState;
import org.sleuthkit.autopsy.ingest.IngestProgressSnapshotDialog;
final class AutoIngestAdminActions {
private static final Logger logger = Logger.getLogger(AutoIngestAdminActions.class.getName());
static abstract class AutoIngestNodeControlAction extends AbstractAction {
private final AutoIngestNodeState nodeState;
private final Logger logger = Logger.getLogger(AutoIngestNodeControlAction.class.getName());
AutoIngestNodeControlAction(AutoIngestNodeState nodeState, String title) {
super(title);
@ -138,19 +143,20 @@ final class AutoIngestAdminActions {
static final class ProgressDialogAction extends AbstractAction {
private static final long serialVersionUID = 1L;
private final AutoIngestJob job;
ProgressDialogAction() {
ProgressDialogAction(AutoIngestJob job) {
super(Bundle.AutoIngestAdminActions_progressDialogAction_title());
this.job = job;
}
@Override
public void actionPerformed(ActionEvent e) {
//TODO JIRA-3734
final AutoIngestDashboardTopComponent tc = (AutoIngestDashboardTopComponent) WindowManager.getDefault().findTopComponent(AutoIngestDashboardTopComponent.PREFERRED_ID);
if (tc != null) {
AutoIngestDashboard dashboard = tc.getAutoIngestDashboard();
if (dashboard != null) {
new IngestProgressSnapshotDialog(dashboard.getTopLevelAncestor(), true);
new IngestProgressSnapshotDialog(dashboard.getTopLevelAncestor(), true, job);
}
}
}
@ -165,14 +171,49 @@ final class AutoIngestAdminActions {
static final class CancelJobAction extends AbstractAction {
private static final long serialVersionUID = 1L;
private final AutoIngestJob job;
CancelJobAction() {
CancelJobAction(AutoIngestJob job) {
super(Bundle.AutoIngestAdminActions_cancelJobAction_title());
this.job = job;
}
@Override
public void actionPerformed(ActionEvent e) {
//TODO JIRA-3738
if (job == null) {
return;
}
final AutoIngestDashboardTopComponent tc = (AutoIngestDashboardTopComponent) WindowManager.getDefault().findTopComponent(AutoIngestDashboardTopComponent.PREFERRED_ID);
if (tc == null) {
return;
}
AutoIngestDashboard dashboard = tc.getAutoIngestDashboard();
if (dashboard != null) {
Object[] options = {
org.openide.util.NbBundle.getMessage(AutoIngestControlPanel.class, "ConfirmationDialog.CancelJob"),
org.openide.util.NbBundle.getMessage(AutoIngestControlPanel.class, "ConfirmationDialog.DoNotCancelJob")};
int reply = JOptionPane.showOptionDialog(dashboard.getRunningJobsPanel(),
NbBundle.getMessage(AutoIngestControlPanel.class, "ConfirmationDialog.CancelJobAreYouSure"),
NbBundle.getMessage(AutoIngestControlPanel.class, "ConfirmationDialog.ConfirmCancellationHeader"),
JOptionPane.DEFAULT_OPTION,
JOptionPane.WARNING_MESSAGE,
null,
options,
options[1]);
if (reply == 0) {
/*
* Call setCursor on this to ensure it appears (if there is
* time to see it).
*/
dashboard.getRunningJobsPanel().setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR));
EventQueue.invokeLater(() -> {
dashboard.getMonitor().cancelJob(job);
dashboard.getRunningJobsPanel().setCursor(Cursor.getDefaultCursor());
});
}
}
}
@Override
@ -201,18 +242,49 @@ final class AutoIngestAdminActions {
}
}
@NbBundle.Messages({"AutoIngestAdminActions.reprocessJobAction.title=Reprocess Job"})
@NbBundle.Messages({"AutoIngestAdminActions.reprocessJobAction.title=Reprocess Job",
"AutoIngestAdminActions.reprocessJobAction.error=Failed to reprocess job"})
static final class ReprocessJobAction extends AbstractAction {
private static final long serialVersionUID = 1L;
private final AutoIngestJob job;
ReprocessJobAction() {
ReprocessJobAction(AutoIngestJob job) {
super(Bundle.AutoIngestAdminActions_reprocessJobAction_title());
this.job = job;
}
@Override
public void actionPerformed(ActionEvent e) {
//TODO JIRA-3739
if (job == null) {
return;
}
final AutoIngestDashboardTopComponent tc = (AutoIngestDashboardTopComponent) WindowManager.getDefault().findTopComponent(AutoIngestDashboardTopComponent.PREFERRED_ID);
if (tc == null) {
return;
}
AutoIngestDashboard dashboard = tc.getAutoIngestDashboard();
if (dashboard != null) {
/*
* Call setCursor on this to ensure it appears (if there is time
* to see it).
*/
dashboard.getCompletedJobsPanel().setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR));
EventQueue.invokeLater(() -> {
try {
dashboard.getMonitor().reprocessJob(job);
dashboard.refreshTables();
dashboard.getCompletedJobsPanel().setCursor(Cursor.getDefaultCursor());
} catch (AutoIngestMonitor.AutoIngestMonitorException ex) {
logger.log(Level.SEVERE, Bundle.AutoIngestAdminActions_reprocessJobAction_error(), ex);
MessageNotifyUtil.Message.error(Bundle.AutoIngestAdminActions_reprocessJobAction_error());
} finally {
dashboard.getCompletedJobsPanel().setCursor(Cursor.getDefaultCursor());
}
});
}
}
@Override
@ -221,18 +293,67 @@ final class AutoIngestAdminActions {
}
}
@NbBundle.Messages({"AutoIngestAdminActions.deleteCaseAction.title=Delete Case"})
@NbBundle.Messages({"AutoIngestAdminActions.deleteCaseAction.title=Delete Case",
"AutoIngestAdminActions.deleteCaseAction.error=Failed to delete case."})
static final class DeleteCaseAction extends AbstractAction {
private static final long serialVersionUID = 1L;
private final AutoIngestJob job;
DeleteCaseAction() {
DeleteCaseAction(AutoIngestJob selectedJob) {
super(Bundle.AutoIngestAdminActions_deleteCaseAction_title());
this.job = selectedJob;
}
@Override
public void actionPerformed(ActionEvent e) {
//TODO JIRA-3740
if (job == null) {
return;
}
final AutoIngestDashboardTopComponent tc = (AutoIngestDashboardTopComponent) WindowManager.getDefault().findTopComponent(AutoIngestDashboardTopComponent.PREFERRED_ID);
if (tc == null) {
return;
}
AutoIngestDashboard dashboard = tc.getAutoIngestDashboard();
if (dashboard != null) {
String caseName = job.getManifest().getCaseName();
Object[] options = {
org.openide.util.NbBundle.getMessage(AutoIngestControlPanel.class, "ConfirmationDialog.Delete"),
org.openide.util.NbBundle.getMessage(AutoIngestControlPanel.class, "ConfirmationDialog.DoNotDelete")
};
Object[] msgContent = {org.openide.util.NbBundle.getMessage(AutoIngestControlPanel.class, "ConfirmationDialog.DeleteAreYouSure") + "\"" + caseName + "\"?"};
int reply = JOptionPane.showOptionDialog(dashboard,
msgContent,
org.openide.util.NbBundle.getMessage(AutoIngestControlPanel.class, "ConfirmationDialog.ConfirmDeletionHeader"),
JOptionPane.DEFAULT_OPTION,
JOptionPane.WARNING_MESSAGE,
null,
options,
options[JOptionPane.NO_OPTION]);
if (reply == JOptionPane.YES_OPTION) {
EventQueue.invokeLater(() -> {
dashboard.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR));
AutoIngestManager.CaseDeletionResult result = dashboard.getMonitor().deleteCase(job);
dashboard.getCompletedJobsPanel().refresh(dashboard.getMonitor().getJobsSnapshot(), new AutoIngestNodeRefreshEvents.RefreshChildrenEvent());
dashboard.setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR));
if (AutoIngestManager.CaseDeletionResult.FAILED == result) {
JOptionPane.showMessageDialog(dashboard,
String.format("Could not delete case %s. It may be in use.", caseName),
org.openide.util.NbBundle.getMessage(AutoIngestControlPanel.class, "AutoIngestControlPanel.DeletionFailed"),
JOptionPane.INFORMATION_MESSAGE);
} else if (AutoIngestManager.CaseDeletionResult.PARTIALLY_DELETED == result) {
JOptionPane.showMessageDialog(dashboard,
String.format("Could not fully delete case %s. See log for details.", caseName),
org.openide.util.NbBundle.getMessage(AutoIngestControlPanel.class, "AutoIngestControlPanel.DeletionFailed"),
JOptionPane.INFORMATION_MESSAGE);
}
});
}
}
}
@Override
@ -241,18 +362,61 @@ final class AutoIngestAdminActions {
}
}
@NbBundle.Messages({"AutoIngestAdminActions.showCaseLogAction.title=Show Case Log"})
@NbBundle.Messages({"AutoIngestAdminActions.showCaseLogAction.title=Show Case Log",
"AutoIngestAdminActions.showCaseLogActionFailed.title=Unable to display case log",
"AutoIngestAdminActions.showCaseLogActionFailed.message=Case log file does not exist",
"AutoIngestAdminActions.showCaseLogActionDialog.ok=Okay",
"AutoIngestAdminActions.showCaseLogActionDialog.cannotFindLog=Unable to find the selected case log file",
"AutoIngestAdminActions.showCaseLogActionDialog.unableToShowLogFile=Unable to show log file"})
static final class ShowCaseLogAction extends AbstractAction {
private static final long serialVersionUID = 1L;
private final AutoIngestJob job;
ShowCaseLogAction() {
ShowCaseLogAction(AutoIngestJob job) {
super(Bundle.AutoIngestAdminActions_showCaseLogAction_title());
this.job = job;
}
@Override
public void actionPerformed(ActionEvent e) {
//TODO JIRA-
if (job == null) {
return;
}
final AutoIngestDashboardTopComponent tc = (AutoIngestDashboardTopComponent) WindowManager.getDefault().findTopComponent(AutoIngestDashboardTopComponent.PREFERRED_ID);
if (tc == null) {
return;
}
AutoIngestDashboard dashboard = tc.getAutoIngestDashboard();
if (dashboard != null) {
try {
Path caseDirectoryPath = job.getCaseDirectoryPath();
if (null != caseDirectoryPath) {
Path pathToLog = AutoIngestJobLogger.getLogPath(caseDirectoryPath);
if (pathToLog.toFile().exists()) {
Desktop.getDesktop().edit(pathToLog.toFile());
} else {
JOptionPane.showMessageDialog(dashboard, Bundle.AutoIngestAdminActions_showCaseLogActionFailed_message(),
Bundle.AutoIngestAdminActions_showCaseLogAction_title(), JOptionPane.ERROR_MESSAGE);
}
} else {
MessageNotifyUtil.Message.warn("The case directory for this job has been deleted.");
}
} catch (IOException ex) {
logger.log(Level.SEVERE, "Dashboard error attempting to display case auto ingest log", ex);
Object[] options = {Bundle.AutoIngestAdminActions_showCaseLogActionDialog_ok()};
JOptionPane.showOptionDialog(dashboard,
Bundle.AutoIngestAdminActions_showCaseLogActionDialog_cannotFindLog(),
Bundle.AutoIngestAdminActions_showCaseLogActionDialog_unableToShowLogFile(),
JOptionPane.DEFAULT_OPTION,
JOptionPane.PLAIN_MESSAGE,
null,
options,
options[0]);
}
}
}
@Override

3
Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestControlPanel.java
View File

@ -850,6 +850,7 @@ public final class AutoIngestControlPanel extends JPanel implements Observer {
case JOB_STARTED:
case JOB_COMPLETED:
case CASE_DELETED:
case REPROCESS_JOB:
updateExecutor.submit(new UpdateAllJobsTablesTask());
break;
case PAUSED_BY_USER_REQUEST:
@ -1681,7 +1682,7 @@ public final class AutoIngestControlPanel extends JPanel implements Observer {
this.setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR));
if (CaseDeletionResult.FAILED == result) {
JOptionPane.showMessageDialog(this,
String.format("Could not delete case %s. It may be in in use.", caseName),
String.format("Could not delete case %s. It may be in use.", caseName),
org.openide.util.NbBundle.getMessage(AutoIngestControlPanel.class, "AutoIngestControlPanel.DeletionFailed"),
JOptionPane.INFORMATION_MESSAGE);
} else if (CaseDeletionResult.PARTIALLY_DELETED == result) {

22
Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestDashboard.form
View File

@ -47,6 +47,8 @@
<Group type="102" alignment="0" attributes="0">
<Component id="refreshButton" linkSize="1" min="-2" pref="100" max="-2" attributes="0"/>
<EmptySpace max="32767" attributes="0"/>
<Component id="healthMonitorButton" min="-2" max="-2" attributes="0"/>
<EmptySpace type="unrelated" max="-2" attributes="0"/>
<Component id="clusterMetricsButton" linkSize="1" min="-2" max="-2" attributes="0"/>
</Group>
</Group>
@ -78,6 +80,7 @@
<Group type="103" groupAlignment="3" attributes="0">
<Component id="refreshButton" alignment="3" min="-2" max="-2" attributes="0"/>
<Component id="clusterMetricsButton" alignment="3" min="-2" max="-2" attributes="0"/>
<Component id="healthMonitorButton" alignment="3" min="-2" max="-2" attributes="0"/>
</Group>
<EmptySpace min="-2" max="-2" attributes="0"/>
</Group>
@ -191,5 +194,24 @@
<EventHandler event="actionPerformed" listener="java.awt.event.ActionListener" parameters="java.awt.event.ActionEvent" handler="clusterMetricsButtonActionPerformed"/>
</Events>
</Component>
<Component class="javax.swing.JButton" name="healthMonitorButton">
<Properties>
<Property name="text" type="java.lang.String" editor="org.netbeans.modules.i18n.form.FormI18nStringEditor">
<ResourceString bundle="org/sleuthkit/autopsy/experimental/autoingest/Bundle.properties" key="AutoIngestDashboard.healthMonitorButton.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
</Property>
<Property name="maximumSize" type="java.awt.Dimension" editor="org.netbeans.beaninfo.editors.DimensionEditor">
<Dimension value="[133, 23]"/>
</Property>
<Property name="minimumSize" type="java.awt.Dimension" editor="org.netbeans.beaninfo.editors.DimensionEditor">
<Dimension value="[133, 23]"/>
</Property>
<Property name="preferredSize" type="java.awt.Dimension" editor="org.netbeans.beaninfo.editors.DimensionEditor">
<Dimension value="[133, 23]"/>
</Property>
</Properties>
<Events>
<EventHandler event="actionPerformed" listener="java.awt.event.ActionListener" parameters="java.awt.event.ActionEvent" handler="healthMonitorButtonActionPerformed"/>
</Events>
</Component>
</SubComponents>
</Form>

37
Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestDashboard.java
View File

@ -38,7 +38,9 @@ import org.openide.util.NbBundle;
import org.openide.util.NbBundle.Messages;
import org.sleuthkit.autopsy.core.ServicesMonitor;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.healthmonitor.HealthMonitorDashboard;
import org.sleuthkit.autopsy.experimental.autoingest.AutoIngestMonitor.JobsSnapshot;
import org.sleuthkit.autopsy.experimental.autoingest.AutoIngestNodeRefreshEvents.RefreshChildrenEvent;
/**
* A dashboard for monitoring an automated ingest cluster.
@ -122,6 +124,14 @@ final class AutoIngestDashboard extends JPanel implements Observer {
return pendingJobsPanel;
}
AutoIngestJobsPanel getRunningJobsPanel() {
return runningJobsPanel;
}
AutoIngestJobsPanel getCompletedJobsPanel() {
return completedJobsPanel;
}
/**
* Update status of the services on the dashboard
*/
@ -261,9 +271,9 @@ final class AutoIngestDashboard extends JPanel implements Observer {
* @param nodeStateSnapshot The jobs snapshot.
*/
void refreshTables() {
pendingJobsPanel.refresh(autoIngestMonitor.getJobsSnapshot());
runningJobsPanel.refresh(autoIngestMonitor.getJobsSnapshot());
completedJobsPanel.refresh(autoIngestMonitor.getJobsSnapshot());
pendingJobsPanel.refresh(autoIngestMonitor.getJobsSnapshot(), new RefreshChildrenEvent());
runningJobsPanel.refresh(autoIngestMonitor.getJobsSnapshot(), new RefreshChildrenEvent());
completedJobsPanel.refresh(autoIngestMonitor.getJobsSnapshot(), new RefreshChildrenEvent());
}
/**
@ -322,6 +332,7 @@ final class AutoIngestDashboard extends JPanel implements Observer {
lbServicesStatus = new javax.swing.JLabel();
tbServicesStatusMessage = new javax.swing.JTextField();
clusterMetricsButton = new javax.swing.JButton();
healthMonitorButton = new javax.swing.JButton();
org.openide.awt.Mnemonics.setLocalizedText(jButton1, org.openide.util.NbBundle.getMessage(AutoIngestDashboard.class, "AutoIngestDashboard.jButton1.text")); // NOI18N
@ -362,6 +373,16 @@ final class AutoIngestDashboard extends JPanel implements Observer {
}
});
org.openide.awt.Mnemonics.setLocalizedText(healthMonitorButton, org.openide.util.NbBundle.getMessage(AutoIngestDashboard.class, "AutoIngestDashboard.healthMonitorButton.text")); // NOI18N
healthMonitorButton.setMaximumSize(new java.awt.Dimension(133, 23));
healthMonitorButton.setMinimumSize(new java.awt.Dimension(133, 23));
healthMonitorButton.setPreferredSize(new java.awt.Dimension(133, 23));
healthMonitorButton.addActionListener(new java.awt.event.ActionListener() {
public void actionPerformed(java.awt.event.ActionEvent evt) {
healthMonitorButtonActionPerformed(evt);
}
});
javax.swing.GroupLayout layout = new javax.swing.GroupLayout(this);
this.setLayout(layout);
layout.setHorizontalGroup(
@ -385,6 +406,8 @@ final class AutoIngestDashboard extends JPanel implements Observer {
.addGroup(javax.swing.GroupLayout.Alignment.LEADING, layout.createSequentialGroup()
.addComponent(refreshButton, javax.swing.GroupLayout.PREFERRED_SIZE, 100, javax.swing.GroupLayout.PREFERRED_SIZE)
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)
.addComponent(healthMonitorButton, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED)
.addComponent(clusterMetricsButton)))
.addContainerGap())
);
@ -413,7 +436,8 @@ final class AutoIngestDashboard extends JPanel implements Observer {
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
.addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE)
.addComponent(refreshButton)
.addComponent(clusterMetricsButton))
.addComponent(clusterMetricsButton)
.addComponent(healthMonitorButton, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE))
.addContainerGap())
);
}// </editor-fold>//GEN-END:initComponents
@ -435,9 +459,14 @@ final class AutoIngestDashboard extends JPanel implements Observer {
new AutoIngestMetricsDialog(this.getTopLevelAncestor());
}//GEN-LAST:event_clusterMetricsButtonActionPerformed
private void healthMonitorButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_healthMonitorButtonActionPerformed
new HealthMonitorDashboard(this.getTopLevelAncestor()).display();
}//GEN-LAST:event_healthMonitorButtonActionPerformed
// Variables declaration - do not modify//GEN-BEGIN:variables
private javax.swing.JButton clusterMetricsButton;
private javax.swing.JScrollPane completedScrollPane;
private javax.swing.JButton healthMonitorButton;
private javax.swing.JButton jButton1;
private javax.swing.JLabel lbCompleted;
private javax.swing.JLabel lbPending;

72
Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJob.java
View File

@ -1,7 +1,7 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2011-2017 Basis Technology Corp.
* Copyright 2011-2018 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
@ -25,23 +25,28 @@ import java.time.Instant;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import javax.annotation.concurrent.GuardedBy;
import javax.annotation.concurrent.Immutable;
import javax.annotation.concurrent.ThreadSafe;
import org.sleuthkit.autopsy.corecomponentinterfaces.DataSourceProcessor;
import org.sleuthkit.autopsy.coreutils.NetworkUtils;
import org.sleuthkit.autopsy.ingest.DataSourceIngestJob.Snapshot;
import org.sleuthkit.autopsy.ingest.IngestJob;
import org.sleuthkit.autopsy.ingest.IngestManager.IngestThreadActivitySnapshot;
import org.sleuthkit.autopsy.ingest.IngestProgressSnapshotProvider;
/**
* An automated ingest job, which is an ingest job performed by the automated
* ingest service.
*/
@ThreadSafe
final class AutoIngestJob implements Comparable<AutoIngestJob>, Serializable {
final class AutoIngestJob implements Comparable<AutoIngestJob>, IngestProgressSnapshotProvider, Serializable {
private static final long serialVersionUID = 1L;
private static final int CURRENT_VERSION = 2;
private static final int CURRENT_VERSION = 3;
private static final int DEFAULT_PRIORITY = 0;
private static final String LOCAL_HOST_NAME = NetworkUtils.getLocalHostName();
@ -89,6 +94,13 @@ final class AutoIngestJob implements Comparable<AutoIngestJob>, Serializable {
@GuardedBy("this")
private long dataSourceSize;
/*
* Version 3 fields.
*/
private List<IngestThreadActivitySnapshot> ingestThreadsSnapshot;
private List<Snapshot> ingestJobsSnapshot;
private Map<String, Long> moduleRunTimesSnapshot;
/**
* Constructs a new automated ingest job. All job state not specified in the
* job manifest is set to the default state for a new job.
@ -125,6 +137,13 @@ final class AutoIngestJob implements Comparable<AutoIngestJob>, Serializable {
* Version 2 fields.
*/
this.dataSourceSize = 0;
/*
* Version 3 fields.
*/
this.ingestThreadsSnapshot = Collections.emptyList();
this.ingestJobsSnapshot = Collections.emptyList();
this.moduleRunTimesSnapshot = Collections.emptyMap();
} catch (Exception ex) {
throw new AutoIngestJobException(String.format("Error creating automated ingest job"), ex);
}
@ -167,6 +186,13 @@ final class AutoIngestJob implements Comparable<AutoIngestJob>, Serializable {
* Version 2 fields.
*/
this.dataSourceSize = nodeData.getDataSourceSize();
/*
* Version 3 fields
*/
this.ingestThreadsSnapshot = Collections.emptyList();
this.ingestJobsSnapshot = Collections.emptyList();
this.moduleRunTimesSnapshot = Collections.emptyMap();
} catch (Exception ex) {
throw new AutoIngestJobException(String.format("Error creating automated ingest job"), ex);
}
@ -340,6 +366,31 @@ final class AutoIngestJob implements Comparable<AutoIngestJob>, Serializable {
return this.ingestJob;
}
/**
* Sets the ingest thread snapshot for the auto ingest job.
*
* @param snapshot
*/
synchronized void setIngestThreadSnapshot(List<IngestThreadActivitySnapshot> snapshot) {
this.ingestThreadsSnapshot = snapshot;
}
/**
* Sets the ingest job snapshot for the auto ingest job.
* @param snapshot
*/
synchronized void setIngestJobsSnapshot(List<Snapshot> snapshot) {
this.ingestJobsSnapshot = snapshot;
}
/**
* Sets the module run times snapshot for the auto ingest job.
* @param snapshot
*/
synchronized void setModuleRuntimesSnapshot(Map<String, Long> snapshot) {
this.moduleRunTimesSnapshot = snapshot;
}
/**
* Cancels the job.
*/
@ -541,6 +592,21 @@ final class AutoIngestJob implements Comparable<AutoIngestJob>, Serializable {
return -this.getManifest().getDateFileCreated().compareTo(otherJob.getManifest().getDateFileCreated());
}
@Override
public List<IngestThreadActivitySnapshot> getIngestThreadActivitySnapshots() {
return this.ingestThreadsSnapshot;
}
@Override
public List<Snapshot> getIngestJobSnapshots() {
return this.ingestJobsSnapshot;
}
@Override
public Map<String, Long> getModuleRunTimes() {
return this.moduleRunTimesSnapshot;
}
/**
* Comparator that supports doing a descending sort of jobs based on job
* completion date.

32
Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJobCancelEvent.java Normal file
View File

@ -0,0 +1,32 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2018 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.experimental.autoingest;
import java.io.Serializable;
/**
* Event published to remotely cancel an AutoIngestJob.
*/
public final class AutoIngestJobCancelEvent extends AutoIngestJobEvent implements Serializable{
private static final long serialVersionUID = 1L;
public AutoIngestJobCancelEvent(AutoIngestJob job) {
super(AutoIngestManager.Event.CANCEL_JOB, job);
}
}

32
Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJobReprocessEvent.java Normal file
View File

@ -0,0 +1,32 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2018 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.experimental.autoingest;
import java.io.Serializable;
/**
* Event published to reprocess an AutoIngestJob.
*/
public final class AutoIngestJobReprocessEvent extends AutoIngestJobEvent implements Serializable{
private static final long serialVersionUID = 1L;
public AutoIngestJobReprocessEvent(AutoIngestJob job) {
super(AutoIngestManager.Event.REPROCESS_JOB, job);
}
}

156
Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJobsNode.java
View File

@ -18,6 +18,8 @@
*/
package org.sleuthkit.autopsy.experimental.autoingest;
import com.google.common.eventbus.EventBus;
import com.google.common.eventbus.Subscribe;
import javax.swing.Action;
import java.time.Instant;
import java.util.ArrayList;
@ -40,6 +42,9 @@ import org.sleuthkit.autopsy.guiutils.StatusIconCellRenderer;
*/
final class AutoIngestJobsNode extends AbstractNode {
//Event bus is non static so that each instance of this will only listen to events sent to that instance
private final EventBus refreshChildrenEventBus;
@Messages({
"AutoIngestJobsNode.caseName.text=Case Name",
"AutoIngestJobsNode.dataSource.text=Data Source",
@ -55,17 +60,27 @@ final class AutoIngestJobsNode extends AbstractNode {
/**
* Construct a new AutoIngestJobsNode.
*/
AutoIngestJobsNode(JobsSnapshot jobsSnapshot, AutoIngestJobStatus status) {
super(Children.create(new AutoIngestNodeChildren(jobsSnapshot, status), false));
AutoIngestJobsNode(JobsSnapshot jobsSnapshot, AutoIngestJobStatus status, EventBus eventBus) {
super(Children.create(new AutoIngestNodeChildren(jobsSnapshot, status, eventBus), false));
refreshChildrenEventBus = eventBus;
}
/**
* Refresh the contents of the AutoIngestJobsNode and all of its children.
*/
void refresh(AutoIngestNodeRefreshEvents.AutoIngestRefreshEvent refreshEvent) {
refreshChildrenEventBus.post(refreshEvent);
}
/**
* A ChildFactory for generating JobNodes.
*/
static class AutoIngestNodeChildren extends ChildFactory<AutoIngestJob> {
static final class AutoIngestNodeChildren extends ChildFactory<AutoIngestJob> {
private final AutoIngestJobStatus autoIngestJobStatus;
private final JobsSnapshot jobsSnapshot;
private final RefreshChildrenSubscriber refreshChildrenSubscriber = new RefreshChildrenSubscriber();
private final EventBus refreshEventBus;
/**
* Create children nodes for the AutoIngestJobsNode which will each
@ -74,9 +89,11 @@ final class AutoIngestJobsNode extends AbstractNode {
* @param snapshot the snapshot which contains the AutoIngestJobs
* @param status the status of the jobs being displayed
*/
AutoIngestNodeChildren(JobsSnapshot snapshot, AutoIngestJobStatus status) {
AutoIngestNodeChildren(JobsSnapshot snapshot, AutoIngestJobStatus status, EventBus eventBus) {
jobsSnapshot = snapshot;
autoIngestJobStatus = status;
refreshEventBus = eventBus;
refreshChildrenSubscriber.register(refreshEventBus);
}
@Override
@ -104,7 +121,47 @@ final class AutoIngestJobsNode extends AbstractNode {
@Override
protected Node createNodeForKey(AutoIngestJob key) {
return new JobNode(key, autoIngestJobStatus);
return new JobNode(key, autoIngestJobStatus, refreshEventBus);
}
/**
* Class which registers with EventBus and causes child nodes which
* exist to be refreshed.
*/
private class RefreshChildrenSubscriber {
/**
* Construct a RefreshChildrenSubscriber
*/
private RefreshChildrenSubscriber() {
}
/**
* Registers this subscriber with the specified EventBus to receive
* events posted to it.
*
* @param eventBus - the EventBus to register this subscriber to
*/
private void register(EventBus eventBus) {
eventBus.register(this);
}
/**
* Receive events which implement the AutoIngestRefreshEvent
* interface from the EventBus which this class is registered to,
* and refresh the children created by this factory.
*
*
* @param refreshEvent the AutoIngestRefreshEvent which was received
*/
@Subscribe
private void subscribeToRefresh(AutoIngestNodeRefreshEvents.AutoIngestRefreshEvent refreshEvent) {
//Ignore netbeans suggesting this isn't being used, it is used behind the scenes by the EventBus
//RefreshChildrenEvents can change which children are present however
//RefreshJobEvents and RefreshCaseEvents can still change the order we want to display them in
refresh(true);
}
}
}
@ -116,6 +173,7 @@ final class AutoIngestJobsNode extends AbstractNode {
private final AutoIngestJob autoIngestJob;
private final AutoIngestJobStatus jobStatus;
private final RefreshNodeSubscriber refreshNodeSubscriber = new RefreshNodeSubscriber();
/**
* Construct a new JobNode to represent an AutoIngestJob and its status.
@ -124,12 +182,13 @@ final class AutoIngestJobsNode extends AbstractNode {
* @param status - the current status of the AutoIngestJob being
* represented
*/
JobNode(AutoIngestJob job, AutoIngestJobStatus status) {
JobNode(AutoIngestJob job, AutoIngestJobStatus status, EventBus eventBus) {
super(Children.LEAF);
jobStatus = status;
autoIngestJob = job;
setName(autoIngestJob.getManifest().getCaseName());
setDisplayName(autoIngestJob.getManifest().getCaseName());
setName(autoIngestJob.toString()); //alows job to be uniquely found by name since it will involve a hash of the AutoIngestJob
setDisplayName(autoIngestJob.getManifest().getCaseName()); //displays user friendly case name as name
refreshNodeSubscriber.register(eventBus);
}
/**
@ -201,20 +260,89 @@ final class AutoIngestJobsNode extends AbstractNode {
actions.add(deprioritizeCaseAction);
break;
case RUNNING_JOB:
actions.add(new AutoIngestAdminActions.ProgressDialogAction());
actions.add(new AutoIngestAdminActions.CancelJobAction());
actions.add(new AutoIngestAdminActions.CancelModuleAction());
actions.add(new AutoIngestAdminActions.ProgressDialogAction(autoIngestJob));
actions.add(new AutoIngestAdminActions.CancelJobAction(autoIngestJob));
// actions.add(new AutoIngestAdminActions.CancelModuleAction());
break;
case COMPLETED_JOB:
actions.add(new AutoIngestAdminActions.ReprocessJobAction());
actions.add(new AutoIngestAdminActions.DeleteCaseAction());
actions.add(new AutoIngestAdminActions.ShowCaseLogAction());
actions.add(new AutoIngestAdminActions.ReprocessJobAction(autoIngestJob));
actions.add(new AutoIngestAdminActions.DeleteCaseAction(autoIngestJob));
actions.add(new AutoIngestAdminActions.ShowCaseLogAction(autoIngestJob));
break;
default:
}
}
return actions.toArray(new Action[actions.size()]);
}
/**
* Class which registers with EventBus and causes specific nodes to have
* their properties to be refreshed.
*/
private class RefreshNodeSubscriber {
/**
* Constructs a RefreshNodeSubscriber
*/
private RefreshNodeSubscriber() {
}
/**
* Registers this subscriber with the specified EventBus to receive
* events posted to it.
*
* @param eventBus - the EventBus to register this subscriber to
*/
private void register(EventBus eventBus) {
eventBus.register(this);
}
/**
* Receive events of type RefreshJobEvent from the EventBus which
* this class is registered to and refresh the nodes properties if
* it is the node for the job specified in the event.
*
* @param refreshEvent the RefreshJobEvent which was received
*/
@Subscribe
private void subscribeToRefreshJob(AutoIngestNodeRefreshEvents.RefreshJobEvent refreshEvent) {
//Ignore netbeans suggesting this isn't being used, it is used behind the scenes by the EventBus
if (getAutoIngestJob().equals(refreshEvent.getJobToRefresh())) {
setSheet(createSheet());
}
}
/**
* Receive events of type RefreshCaseEvent from the EventBus which
* this class is registered to and refresh the nodes which have jobs
* which are members of case specified in the event.
*
* @param refreshEvent the RefreshCaseEvent which was received
*/
@Subscribe
private void subscribeToRefreshCase(AutoIngestNodeRefreshEvents.RefreshCaseEvent refreshEvent) {
//Ignore netbeans suggesting this isn't being used, it is used behind the scenes by the EventBus
if (getAutoIngestJob().getManifest().getCaseName().equals(refreshEvent.getCaseToRefresh())) {
setSheet(createSheet());
}
}
/**
* Refresh the properties of all running jobs anytime a
* RefreshChildrenEvent is received so that stages and times stay up
* to date.
*
* @param refreshEvent - the RefreshChildrenEvent which was received
*/
@Subscribe
private void subscribeToRefreshChildren(AutoIngestNodeRefreshEvents.RefreshChildrenEvent refreshEvent) {
//Ignore netbeans suggesting this isn't being used, it is used behind the scenes by the EventBus
if (jobStatus == AutoIngestJobStatus.RUNNING_JOB) {
setSheet(createSheet());
}
}
}
}
/**

22
Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJobsPanel.java
View File

@ -18,8 +18,8 @@
*/
package org.sleuthkit.autopsy.experimental.autoingest;
import com.google.common.eventbus.EventBus;
import java.awt.Dimension;
import java.beans.PropertyVetoException;
import javax.swing.ListSelectionModel;
import javax.swing.event.ListSelectionListener;
import org.netbeans.swing.outline.DefaultOutlineModel;
@ -31,6 +31,7 @@ import org.sleuthkit.autopsy.datamodel.EmptyNode;
import org.sleuthkit.autopsy.experimental.autoingest.AutoIngestJobsNode.AutoIngestJobStatus;
import org.sleuthkit.autopsy.experimental.autoingest.AutoIngestJobsNode.JobNode;
import org.sleuthkit.autopsy.experimental.autoingest.AutoIngestMonitor.JobsSnapshot;
import org.sleuthkit.autopsy.experimental.autoingest.AutoIngestNodeRefreshEvents.AutoIngestRefreshEvent;
/**
* A panel which displays an outline view with all jobs for a specified status.
@ -163,21 +164,18 @@ final class AutoIngestJobsPanel extends javax.swing.JPanel implements ExplorerMa
* @param jobsSnapshot - the JobsSnapshot which will provide the new
* contents
*/
void refresh(JobsSnapshot jobsSnapshot) {
void refresh(JobsSnapshot jobsSnapshot, AutoIngestRefreshEvent refreshEvent) {
synchronized (this) {
outline.setRowSelectionAllowed(false);
Node[] selectedNodes = explorerManager.getSelectedNodes();
AutoIngestJobsNode autoIngestNode = new AutoIngestJobsNode(jobsSnapshot, status);
explorerManager.setRootContext(autoIngestNode);
if (explorerManager.getRootContext() instanceof AutoIngestJobsNode) {
((AutoIngestJobsNode) explorerManager.getRootContext()).refresh(refreshEvent);
} else {
//Make a new AutoIngestJobsNode with it's own EventBus and set it as the root context
explorerManager.setRootContext(new AutoIngestJobsNode(jobsSnapshot, status, new EventBus("AutoIngestJobsNodeEventBus")));
}
outline.setRowSelectionAllowed(true);
if (selectedNodes.length > 0 && autoIngestNode.getChildren().findChild(selectedNodes[0].getName()) != null && outline.isFocusable()) { //don't allow saved selections of empty nodes to be restored
try {
explorerManager.setSelectedNodes(new Node[]{autoIngestNode.getChildren().findChild(selectedNodes[0].getName())});
} catch (PropertyVetoException ignore) {
//Unable to select previously selected node
}
}
outline.setFocusable(true);
}
}

52
Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java
View File

@ -63,7 +63,6 @@ import org.sleuthkit.autopsy.casemodule.Case.CaseType;
import org.sleuthkit.autopsy.casemodule.CaseActionException;
import org.sleuthkit.autopsy.casemodule.CaseDetails;
import org.sleuthkit.autopsy.casemodule.CaseMetadata;
import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException;
import org.sleuthkit.autopsy.coordinationservice.CaseNodeData;
import org.sleuthkit.autopsy.coordinationservice.CoordinationService;
import org.sleuthkit.autopsy.coordinationservice.CoordinationService.CoordinationServiceException;
@ -134,11 +133,14 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
Event.JOB_STATUS_UPDATED.toString(),
Event.JOB_COMPLETED.toString(),
Event.CASE_PRIORITIZED.toString(),
Event.CASE_DELETED.toString(),
Event.JOB_STARTED.toString(),
Event.REPORT_STATE.toString(),
ControlEventType.PAUSE.toString(),
ControlEventType.RESUME.toString(),
ControlEventType.SHUTDOWN.toString()}));
ControlEventType.SHUTDOWN.toString(),
Event.CANCEL_JOB.toString(),
Event.REPROCESS_JOB.toString()}));
private static final long JOB_STATUS_EVENT_INTERVAL_SECONDS = 10;
private static final String JOB_STATUS_PUBLISHING_THREAD_NAME = "AIM-job-status-event-publisher-%d";
private static final long MAX_MISSED_JOB_STATUS_UPDATES = 10;
@ -288,6 +290,10 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
handleRemoteRequestNodeStateEvent();
} else if (event instanceof AutoIngestNodeControlEvent) {
handleRemoteNodeControlEvent((AutoIngestNodeControlEvent) event);
} else if (event instanceof AutoIngestJobCancelEvent) {
handleRemoteJobCancelledEvent((AutoIngestJobCancelEvent) event);
} else if (event instanceof AutoIngestJobReprocessEvent) {
handleRemoteJobReprocessEvent((AutoIngestJobReprocessEvent) event);
}
}
}
@ -375,6 +381,41 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
notifyObservers(Event.JOB_COMPLETED);
}
/**
* Processes a job cancellation request from the dashboard.
*
* @param event
*/
private void handleRemoteJobCancelledEvent(AutoIngestJobCancelEvent event) {
AutoIngestJob job = event.getJob();
if (job != null && job.getProcessingHostName().compareToIgnoreCase(LOCAL_HOST_NAME) == 0) {
if (event.getJob().equals(currentJob)) {
cancelCurrentJob();
}
}
}
/**
* Process a job reprocess event from a remote host.
*
* @param event
*/
private void handleRemoteJobReprocessEvent(AutoIngestJobReprocessEvent event) {
synchronized (jobsLock) {
AutoIngestJob job = event.getJob();
if (completedJobs.contains(job)) {
// Remove from completed jobs table.
completedJobs.remove(job);
// Add to pending jobs table and re-sort.
pendingJobs.add(job);
Collections.sort(pendingJobs, new AutoIngestJob.PriorityComparator());
setChanged();
notifyObservers(Event.REPROCESS_JOB);
}
}
}
/**
* Processes a job/case prioritization event from another node by triggering
* an immediate input directory scan.
@ -3011,6 +3052,9 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
synchronized (jobsLock) {
if (currentJob != null) {
currentJob.getProcessingStageDetails();
currentJob.setIngestThreadSnapshot(IngestManager.getInstance().getIngestThreadActivitySnapshots());
currentJob.setIngestJobsSnapshot(IngestManager.getInstance().getIngestJobSnapshots());
currentJob.setModuleRuntimesSnapshot(IngestManager.getInstance().getModuleRunTimes());
setChanged();
notifyObservers(Event.JOB_STATUS_UPDATED);
updateCoordinationServiceManifestNode(currentJob);
@ -3106,7 +3150,9 @@ final class AutoIngestManager extends Observable implements PropertyChangeListen
RUNNING,
SHUTTING_DOWN,
SHUTDOWN,
REPORT_STATE
REPORT_STATE,
CANCEL_JOB,
REPROCESS_JOB
}
/**

146
Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestMonitor.java
View File

@ -21,8 +21,11 @@ package org.sleuthkit.autopsy.experimental.autoingest;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import java.beans.PropertyChangeEvent;
import java.beans.PropertyChangeListener;
import java.nio.file.Path;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@ -34,6 +37,9 @@ import java.util.concurrent.TimeUnit;
import java.util.logging.Level;
import java.util.stream.Collectors;
import javax.annotation.concurrent.GuardedBy;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.casemodule.CaseActionException;
import org.sleuthkit.autopsy.casemodule.CaseMetadata;
import org.sleuthkit.autopsy.coordinationservice.CoordinationService;
import org.sleuthkit.autopsy.coordinationservice.CoordinationService.CoordinationServiceException;
import org.sleuthkit.autopsy.coreutils.Logger;
@ -41,8 +47,12 @@ import org.sleuthkit.autopsy.coreutils.NetworkUtils;
import org.sleuthkit.autopsy.events.AutopsyEventException;
import org.sleuthkit.autopsy.events.AutopsyEventPublisher;
import org.sleuthkit.autopsy.experimental.autoingest.AutoIngestJob.ProcessingStatus;
import static org.sleuthkit.autopsy.experimental.autoingest.AutoIngestJob.ProcessingStatus.DELETED;
import static org.sleuthkit.autopsy.experimental.autoingest.AutoIngestJob.ProcessingStatus.PENDING;
import org.sleuthkit.autopsy.experimental.autoingest.AutoIngestManager.CaseDeletionResult;
import org.sleuthkit.autopsy.experimental.autoingest.AutoIngestManager.Event;
import org.sleuthkit.autopsy.experimental.autoingest.AutoIngestNodeControlEvent.ControlEventType;
/**
* An auto ingest monitor responsible for monitoring and reporting the
* processing of auto ingest jobs.
@ -178,7 +188,17 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen
*/
AutoIngestJob job = event.getJob();
jobsSnapshot.removePendingJob(job);
jobsSnapshot.addOrReplaceRunningJob(job);
// Update the state of the existing job in the running jobs table
for (AutoIngestJob runningJob : jobsSnapshot.getRunningJobs()) {
if (runningJob.equals(job)) {
runningJob.setIngestJobsSnapshot(job.getIngestJobSnapshots());
runningJob.setIngestThreadSnapshot(job.getIngestThreadActivitySnapshots());
runningJob.setModuleRuntimesSnapshot(job.getModuleRunTimes());
runningJob.setProcessingStage(job.getProcessingStage(), job.getProcessingStageStartDate());
runningJob.setProcessingStatus(job.getProcessingStatus());
}
}
setChanged();
notifyObservers(jobsSnapshot);
}
@ -253,6 +273,7 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen
/**
* Gets the current state of known AIN's in the system.
*
* @return
*/
List<AutoIngestNodeState> getNodeStates() {
@ -526,6 +547,117 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen
}
}
/**
* Send an event to tell a remote node to cancel the given job.
*
* @param job
*/
void cancelJob(AutoIngestJob job) {
new Thread(() -> {
eventPublisher.publishRemotely(new AutoIngestJobCancelEvent(job));
}).start();
}
/**
* Reprocess the given job.
*
* @param job
*/
void reprocessJob(AutoIngestJob job) throws AutoIngestMonitorException {
synchronized (jobsLock) {
if (!jobsSnapshot.getCompletedJobs().contains(job)) {
return;
}
jobsSnapshot.removeCompletedJob(job);
/*
* Add the job to the pending jobs queue and update the coordination
* service manifest node data for the job.
*/
if (null != job && !job.getCaseDirectoryPath().toString().isEmpty()) {
/**
* We reset the status, completion date and processing stage but
* we keep the original priority.
*/
job.setErrorsOccurred(false);
job.setCompletedDate(new Date(0));
job.setProcessingStatus(PENDING);
job.setProcessingStage(AutoIngestJob.Stage.PENDING, Date.from(Instant.now()));
String manifestNodePath = job.getManifest().getFilePath().toString();
try {
AutoIngestJobNodeData nodeData = new AutoIngestJobNodeData(job);
coordinationService.setNodeData(CoordinationService.CategoryNode.MANIFESTS, manifestNodePath, nodeData.toArray());
} catch (CoordinationServiceException | InterruptedException ex) {
throw new AutoIngestMonitorException("Error reprocessing job " + job.toString(), ex);
}
// Add to pending jobs collection.
jobsSnapshot.addOrReplacePendingJob(job);
/*
* Publish a reprocess event.
*/
new Thread(() -> {
eventPublisher.publishRemotely(new AutoIngestJobReprocessEvent(job));
}).start();
}
}
}
/**
* Deletes a case. This includes deleting the case directory, the text
* index, and the case database. This does not include the directories
* containing the data sources and their manifests.
*
* @param job The job whose case you want to delete
*
* @return A result code indicating success, partial success, or failure.
*/
CaseDeletionResult deleteCase(AutoIngestJob job) {
synchronized (jobsLock) {
String caseName = job.getManifest().getCaseName();
Path metadataFilePath = job.getCaseDirectoryPath().resolve(caseName + CaseMetadata.getFileExtension());
try {
CaseMetadata metadata = new CaseMetadata(metadataFilePath);
Case.deleteCase(metadata);
} catch (CaseMetadata.CaseMetadataException ex) {
LOGGER.log(Level.SEVERE, String.format("Failed to get case metadata file %s for case %s at %s", metadataFilePath.toString(), caseName, job.getCaseDirectoryPath().toString()), ex);
return CaseDeletionResult.FAILED;
} catch (CaseActionException ex) {
LOGGER.log(Level.SEVERE, String.format("Failed to physically delete case %s at %s", caseName, job.getCaseDirectoryPath().toString()), ex);
return CaseDeletionResult.FAILED;
}
// Update the state of completed jobs associated with this case to indicate
// that the case has been deleted
for (AutoIngestJob completedJob : jobsSnapshot.getCompletedJobs()) {
if (caseName.equals(completedJob.getManifest().getCaseName())) {
try {
completedJob.setProcessingStatus(DELETED);
AutoIngestJobNodeData nodeData = new AutoIngestJobNodeData(completedJob);
coordinationService.setNodeData(CoordinationService.CategoryNode.MANIFESTS, completedJob.getManifest().getFilePath().toString(), nodeData.toArray());
} catch (CoordinationServiceException | InterruptedException ex) {
LOGGER.log(Level.SEVERE, String.format("Failed to update completed job node data for %s when deleting case %s", completedJob.getManifest().getFilePath().toString(), caseName), ex);
return CaseDeletionResult.PARTIALLY_DELETED;
}
}
}
// Remove jobs associated with this case from the completed jobs collection.
jobsSnapshot.completedJobs.removeIf((AutoIngestJob completedJob) ->
completedJob.getManifest().getCaseName().equals(caseName));
// Publish a message to update auto ingest nodes.
eventPublisher.publishRemotely(new AutoIngestCaseDeletedEvent(caseName, LOCAL_HOST_NAME));
}
return CaseDeletionResult.FULLY_DELETED;
}
/**
* Send the given control event to the given node.
*
@ -591,8 +723,8 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen
}
/**
* A snapshot of the pending jobs queue, running jobs list and completed jobs
* list for an auto ingest cluster.
* A snapshot of the pending jobs queue, running jobs list and completed
* jobs list for an auto ingest cluster.
*/
static final class JobsSnapshot {
@ -645,7 +777,7 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen
* Removes a job, if present, in the snapshot of the pending jobs queue
* for an auto ingest cluster.
*
* @param job The auot ingest job.
* @param job The auto ingest job.
*/
private void removePendingJob(AutoIngestJob job) {
this.pendingJobs.remove(job);
@ -665,7 +797,7 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen
* Removes a job, if present, in the snapshot of the running jobs list
* for an auto ingest cluster.
*
* @param job The auot ingest job.
* @param job The auto ingest job.
*/
private void removeRunningJob(AutoIngestJob job) {
this.runningJobs.remove(job);
@ -686,10 +818,10 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen
* Removes a job, if present, in the snapshot of the completed jobs list
* for an auto ingest cluster.
*
* @param job The auot ingest job.
* @param job The auto ingest job.
*/
private void removeCompletedJob(AutoIngestJob job) {
this.pendingJobs.remove(job);
this.completedJobs.remove(job);
}
/**

101
Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestNodeRefreshEvents.java Normal file
View File

@ -0,0 +1,101 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2018 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.experimental.autoingest;
/**
* Class which contains events to identify what should be refreshed in the
* AutoIngestJobsNode
*/
class AutoIngestNodeRefreshEvents {
/**
* An empty interface for all refresh events to implement.
*/
interface AutoIngestRefreshEvent {
}
/**
* An event to denote that the children of the AutoIngestJobsNode should be
* refreshed but no specific nodes need their properties refreshed.
*/
static final class RefreshChildrenEvent implements AutoIngestRefreshEvent {
/**
* Constructs a RefreshChildrenEvent.
*/
RefreshChildrenEvent() {
}
}
/**
* An event to denote that all nodes which represent jobs which are part of
* the specified case should be refreshed.
*/
static final class RefreshCaseEvent implements AutoIngestRefreshEvent {
private final String caseName;
/**
* Constructs a RefreshCaseEvent.
*/
RefreshCaseEvent(String name) {
caseName = name;
}
/**
* Get the case name which should have all it's jobs have their node
* refreshed.
*
* @return caseName - the case which contains the jobs which should have
* their nodes refreshed
*/
String getCaseToRefresh() {
return caseName;
}
}
/**
* An event to denote that a node for a specific job should be refreshed.
*/
static final class RefreshJobEvent implements AutoIngestRefreshEvent {
private final AutoIngestJob autoIngestJob;
/**
* Constructs a RefreshJobEvent.
*/
RefreshJobEvent(AutoIngestJob job) {
autoIngestJob = job;
}
/**
* Get the AutoIngestJob which should have it's node refresheds.
*
* @return autoIngestJob - the AutoIngestJob which should have it's node
* refreshed
*/
AutoIngestJob getJobToRefresh() {
return autoIngestJob;
}
}
}

1
Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/Bundle.properties
View File

@ -258,6 +258,7 @@ AutoIngestControlPanel.bnPrioritizeJob.toolTipText=Move this folder to the top o
AutoIngestControlPanel.bnPrioritizeCase.toolTipText=Move all images associated with a case to top of Pending queue.
AutoIngestControlPanel.bnPrioritizeJob.actionCommand=Prioritize Job
AutoIngestControlPanel.bnDeprioritizeJob.actionCommand=Deprioritize Job
AutoIngestDashboard.healthMonitorButton.text=Health Monitor
AinStatusDashboard.refreshButton.toolTipText=Refresh displayed tables
AinStatusDashboard.refreshButton.text=&Refresh
AinStatusDashboard.clusterMetricsButton.text=Auto Ingest &Metrics

26
Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/PrioritizationAction.java
View File

@ -27,6 +27,7 @@ import org.openide.util.NbBundle.Messages;
import org.openide.windows.WindowManager;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil;
import org.sleuthkit.autopsy.experimental.autoingest.AutoIngestNodeRefreshEvents;
/**
* Abstract actions which are for the modification of AutoIngestJob or Case
@ -38,6 +39,7 @@ abstract class PrioritizationAction extends AbstractAction {
private static final Logger logger = Logger.getLogger(PrioritizationAction.class.getName());
private final AutoIngestJob job;
/**
* Construct a new Prioritization action for the selected job
*
@ -88,7 +90,7 @@ abstract class PrioritizationAction extends AbstractAction {
EventQueue.invokeLater(() -> {
try {
modifyPriority(dashboard.getMonitor());
dashboard.getPendingJobsPanel().refresh(dashboard.getMonitor().getJobsSnapshot());
dashboard.getPendingJobsPanel().refresh(dashboard.getMonitor().getJobsSnapshot(), getRefreshEvent());
} catch (AutoIngestMonitor.AutoIngestMonitorException ex) {
String errorMessage = getErrorMessage();
logger.log(Level.SEVERE, errorMessage, ex);
@ -107,6 +109,8 @@ abstract class PrioritizationAction extends AbstractAction {
return super.clone(); //To change body of generated methods, choose Tools | Templates.
}
abstract AutoIngestNodeRefreshEvents.AutoIngestRefreshEvent getRefreshEvent();
/**
* Action to prioritize the specified AutoIngestJob
*/
@ -139,6 +143,11 @@ abstract class PrioritizationAction extends AbstractAction {
public Object clone() throws CloneNotSupportedException {
return super.clone(); //To change body of generated methods, choose Tools | Templates.
}
@Override
AutoIngestNodeRefreshEvents.AutoIngestRefreshEvent getRefreshEvent() {
return new AutoIngestNodeRefreshEvents.RefreshJobEvent(getJob());
}
}
/**
@ -173,6 +182,11 @@ abstract class PrioritizationAction extends AbstractAction {
public Object clone() throws CloneNotSupportedException {
return super.clone(); //To change body of generated methods, choose Tools | Templates.
}
@Override
AutoIngestNodeRefreshEvents.AutoIngestRefreshEvent getRefreshEvent() {
return new AutoIngestNodeRefreshEvents.RefreshJobEvent(getJob());
}
}
/**
@ -209,6 +223,11 @@ abstract class PrioritizationAction extends AbstractAction {
public Object clone() throws CloneNotSupportedException {
return super.clone(); //To change body of generated methods, choose Tools | Templates.
}
@Override
AutoIngestNodeRefreshEvents.AutoIngestRefreshEvent getRefreshEvent() {
return new AutoIngestNodeRefreshEvents.RefreshCaseEvent(getJob().getManifest().getCaseName());
}
}
/**
@ -245,5 +264,10 @@ abstract class PrioritizationAction extends AbstractAction {
public Object clone() throws CloneNotSupportedException {
return super.clone(); //To change body of generated methods, choose Tools | Templates.
}
@Override
AutoIngestNodeRefreshEvents.AutoIngestRefreshEvent getRefreshEvent() {
return new AutoIngestNodeRefreshEvents.RefreshCaseEvent(getJob().getManifest().getCaseName());
}
}
}

4
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java
View File

@ -753,9 +753,13 @@ public class Server {
org.apache.solr.client.solrj.request.CoreAdminRequest.unloadCore(coreName, true, true, solrServer);
}
} catch (SolrServerException | HttpSolrServer.RemoteSolrException | IOException ex) {
// We will get a RemoteSolrException with cause == null and detailsMessage
// == "Already closed" if the core is not loaded. This is not an error in this scenario.
if (!ex.getMessage().equals("Already closed")) { // NON-NLS
throw new KeywordSearchServiceException(Bundle.Server_deleteCore_exception_msg(coreName), ex);
}
}
}
/**
* Creates/opens a Solr core (index) for a case.

30
docs/doxygen-user/archive_extractor.dox
View File

@ -1,7 +1,7 @@
/*! \page embedded_file_extractor_page Embedded File Extraction Module
What Does It Do
========
\section embedded_files_overview What Does It Do
The Embedded File Extractor module opens ZIP, RAR, other archive formats, Doc, Docx, PPT, PPTX, XLS, and XLSX and sends the derived files from those files back through the ingest pipeline for analysis.
@ -9,21 +9,17 @@ This module expands archive files to enable Autopsy to analyze all files on the
NOTE: Certain media content embedded inside Doc, Docx, PPT, PPTX, XLS, and XLSX might not be extracted.
Configuration
=======
\section embedded_files_config Configuration
There is no configuration required.
Using the Module
======
\section embedded_files_usage Using the Module
Select the checkbox in the Ingest Modules settings screen to enable the Archive Extractor.
Ingest Settings
------
\subsection embedded_files_settings Ingest Settings
There are no runtime ingest settings required.
Seeing Results
------
\subsection embedded_files_results Seeing Results
Each file extracted shows up in the data source tree view as a child of the archive containing it,
\image html zipped_children_1.PNG
@ -32,4 +28,18 @@ Each file extracted shows up in the data source tree view as a child of the arch
and as an archive under "Views", "File Types", "Archives".
\image html zipped_children_2.PNG
\subsection embedded_files_encryption Encrypted Archives
When the Embedded File Extractor module encounters an encrypted archive, it will generate a warning bubble in the bottom right of the main screen:
\image html zipped_encryption_detected.png
After ingest, you can attempt to decrypt these archives if you know the password. Find the archive (either in the \ref tree_viewer_page "tree view" or \ref result_viewer_page "result view") and right-click on it, then select "Unzip contents with password".
\image html zipped_context_menu.png
After entering the password, you can select which ingest modules to run on the newly extracted files. When finished, you can browse to the encrypted archive in the tree view to see the newly extracted files. If the archive was already open in the tree, you may have to close and open the case in order to see the new data.
\image html zipped_tree.png
*/

26
docs/doxygen-user/communications.dox
View File

@ -20,8 +20,30 @@ The middle column displays each account, its device and type, and the number of
Selecting an account in the middle column will bring up the messages for that account in the right hand column. Here data about each message is displayed in the top section, and the messages itself can be seen in the bottom section (if applicable).
The middle column and the right hand column both have a \ref ui_quick_search feature which can be used to quickly find a visible item in their section's table.
\image html cvt_messages.png
The middle column and the right hand column both have a \ref ui_quick_search feature which can be used to quickly find a visible item in their section's table.
\section cvt_viz Visualization
The Visualize tab in the middle panel will show a graph of one or more accounts selected in the Browse tab.
To start, right click the first account you want to view.
\image html cvt_select_account.png
There are two options, which are equivalent when no accounts have previously been selected:
<ul>
<li>Add Selected Account to Visualization - Adds this account and its connections to the graph
<li>Visualize Only Selected Account - Clears the graph and only displays the connections for this account
</ul>
After selecting either option, the middle tab will switch to the Visualize view and the graph will be displayed.
\image html cvt_visualize.png
The options at the top allow you to clear the graph, try different graph layouts, and resize the graph. The nodes in the graph can be dragged around and nodes and edges can be selected to display their messages or relationships in the right side tab. For example, in the image below the link between two email addresses has been selected so the Messages viewer is displaying the single email between those two email addresses.
\image html cvt_links.png
*/

17
docs/doxygen-user/encryption_detection.dox
View File

@ -2,19 +2,30 @@
\section encrypt_overview Overview
The Encryption Detection Module searches for files that could be encrypted using an entropy calculation.
The Encryption Detection Module searches for files that could be encrypted using both a general entropy calculation and more specialized tests for certain file types.
\section encrypt_running Running the module
The module's settings can be configured at runtime.
The module's settings can be configured at runtime. These settings only effect the tests that are based on entropy.
\image html encrypt_module.png
Minimum entropy can be set higher or lower, depending on how many false hits are being produced. There is also an option to only run the test on files whose size is a multiple of 512, which is useful for finding certain encryption algorithms.
The module looks for the following types of encryption:
<ul>
<li> Any file that has an entropy equal to or greater than the threshold in the module settings and that fits the file size constraints
<li> Password protected Office files, PDF files, and Access database files
<li> BitLocker volumes
<li> SQLCipher (uses the minimum entropy from the module settings)
<li> VeraCrypt (uses the minimum entropy from the module settings)
</ul>
\section encrypt_results Viewing results
Files that pass the test are shown in the Results tree under "Encryption Suspected".
Files that pass the tests are shown in the Results tree under "Encryption Detected" or "Encryption Suspected". Generally, if the test used involved looking for a
specific header/file structure, the result will be "Encryption Detected" and the type of encryption will be displayed in the Comment field. If the test was based on the entropy of the file,
the result will be "Encryption Suspected" and the calculated entropy will be displayed in the Comment field.
\image html encrypt_tree.png

BIN
docs/doxygen-user/images/cvt_links.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 62 KiB

BIN
docs/doxygen-user/images/cvt_main.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 85 KiB

After

Width:  |  Height:  |  Size: 112 KiB

BIN
docs/doxygen-user/images/cvt_messages.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 52 KiB

After

Width:  |  Height:  |  Size: 49 KiB

BIN
docs/doxygen-user/images/cvt_select_account.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 19 KiB

BIN
docs/doxygen-user/images/cvt_visualize.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 38 KiB

BIN
docs/doxygen-user/images/encrypt_tree.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 11 KiB

After

Width:  |  Height:  |  Size: 25 KiB

BIN
docs/doxygen-user/images/zipped_context_menu.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 38 KiB

BIN
docs/doxygen-user/images/zipped_encryption_detected.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 9.5 KiB

BIN
docs/doxygen-user/images/zipped_tree.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 120 KiB