From e352817801ade27d55784c3bd388f2b89874a7eb Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Mon, 27 Aug 2018 13:33:17 -0400 Subject: [PATCH 01/20] 4163 add caches to case, type, and datasource for eamdb queries --- .../datamodel/AbstractSqlEamDb.java | 188 +++++++++++------- 1 file changed, 114 insertions(+), 74 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java index 288363fe95..0ce4941c09 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java @@ -18,6 +18,8 @@ */ package org.sleuthkit.autopsy.centralrepository.datamodel; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; import java.net.UnknownHostException; import java.util.ArrayList; import java.util.List; @@ -34,6 +36,8 @@ import java.time.LocalDate; import java.util.HashMap; import java.util.Map; import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; import java.util.logging.Level; import org.sleuthkit.autopsy.casemodule.Case; import static org.sleuthkit.autopsy.centralrepository.datamodel.EamDbUtil.updateSchemaVersion; @@ -57,6 +61,15 @@ abstract class AbstractSqlEamDb implements EamDb { private int bulkArtifactsCount; protected int bulkArtifactsThreshold; private final Map> bulkArtifacts; + private static final int CASE_CACHE_TIMEOUT = 5; + private static final int DATA_SOURCE_CACHE_TIMEOUT = 5; + private static final Cache typeCache = CacheBuilder.newBuilder().build(); + private static final Cache caseCache = CacheBuilder.newBuilder() + .expireAfterWrite(CASE_CACHE_TIMEOUT, TimeUnit.MINUTES). + build(); + private static final Cache dataSourceCache = CacheBuilder.newBuilder() + .expireAfterWrite(DATA_SOURCE_CACHE_TIMEOUT, TimeUnit.MINUTES). + build(); // Maximum length for the value column in the instance tables static final int MAX_VALUE_LENGTH = 256; @@ -88,7 +101,7 @@ abstract class AbstractSqlEamDb implements EamDb { /** * Add a new name/value pair in the db_info table. * - * @param name Key to set + * @param name Key to set * @param value Value to set * * @throws EamDbException @@ -149,10 +162,19 @@ abstract class AbstractSqlEamDb implements EamDb { return value; } + /** + * Reset the contents of the caches associated with EamDb results. + */ + protected final void clearCaches() { + typeCache.invalidateAll(); + caseCache.invalidateAll(); + dataSourceCache.invalidateAll(); + } + /** * Update the value for a name in the name/value db_info table. * - * @param name Name to find + * @param name Name to find * @param value Value to assign to name. * * @throws EamDbException @@ -278,7 +300,11 @@ abstract class AbstractSqlEamDb implements EamDb { @Override public CorrelationCase getCase(Case autopsyCase) throws EamDbException { - return getCaseByUUID(autopsyCase.getName()); + try { + return caseCache.get(autopsyCase.getName(), () -> getCaseByUUID(autopsyCase.getName())); + } catch (ExecutionException ex) { + throw new EamDbException("Error getting autopsy case from Central repo", ex); + } } /** @@ -505,51 +531,57 @@ abstract class AbstractSqlEamDb implements EamDb { /** * Retrieves Data Source details based on data source device ID * - * @param correlationCase the current CorrelationCase used for ensuring - * uniqueness of DataSource + * @param correlationCase the current CorrelationCase used for ensuring + * uniqueness of DataSource * @param dataSourceDeviceId the data source device ID number * * @return The data source */ @Override public CorrelationDataSource getDataSource(CorrelationCase correlationCase, String dataSourceDeviceId) throws EamDbException { + if (correlationCase == null) { throw new EamDbException("Correlation case is null"); } - - Connection conn = connect(); - - CorrelationDataSource eamDataSourceResult = null; - PreparedStatement preparedStatement = null; - ResultSet resultSet = null; - - String sql = "SELECT * FROM data_sources WHERE device_id=? AND case_id=?"; // NON-NLS - try { - preparedStatement = conn.prepareStatement(sql); - preparedStatement.setString(1, dataSourceDeviceId); - preparedStatement.setInt(2, correlationCase.getID()); - resultSet = preparedStatement.executeQuery(); - if (resultSet.next()) { - eamDataSourceResult = getEamDataSourceFromResultSet(resultSet); - } - } catch (SQLException ex) { - throw new EamDbException("Error getting data source.", ex); // NON-NLS - } finally { - EamDbUtil.closeStatement(preparedStatement); - EamDbUtil.closeResultSet(resultSet); - EamDbUtil.closeConnection(conn); - } + return dataSourceCache.get(correlationCase.getCaseUUID() + dataSourceDeviceId, () -> { + Connection conn = connect(); - return eamDataSourceResult; + CorrelationDataSource eamDataSourceResult = null; + PreparedStatement preparedStatement = null; + ResultSet resultSet = null; + + String sql = "SELECT * FROM data_sources WHERE device_id=? AND case_id=?"; // NON-NLS + + try { + preparedStatement = conn.prepareStatement(sql); + preparedStatement.setString(1, dataSourceDeviceId); + preparedStatement.setInt(2, correlationCase.getID()); + resultSet = preparedStatement.executeQuery(); + if (resultSet.next()) { + eamDataSourceResult = getEamDataSourceFromResultSet(resultSet); + } + } catch (SQLException ex) { + throw new EamDbException("Error getting data source.", ex); // NON-NLS + } finally { + EamDbUtil.closeStatement(preparedStatement); + EamDbUtil.closeResultSet(resultSet); + EamDbUtil.closeConnection(conn); + } + + return eamDataSourceResult; + }); + } catch (ExecutionException ex) { + throw new EamDbException("Error getting data source from central repository", ex); + } } /** * Retrieves Data Source details based on data source ID * * @param correlationCase the current CorrelationCase used for ensuring - * uniqueness of DataSource - * @param dataSourceId the data source ID number + * uniqueness of DataSource + * @param dataSourceId the data source ID number * * @return The data source */ @@ -764,7 +796,7 @@ abstract class AbstractSqlEamDb implements EamDb { * Retrieves eamArtifact instances from the database that are associated * with the aType and filePath * - * @param aType EamArtifact.Type to search for + * @param aType EamArtifact.Type to search for * @param filePath File path to search for * * @return List of 0 or more EamArtifactInstances @@ -831,7 +863,7 @@ abstract class AbstractSqlEamDb implements EamDb { * @param value The correlation value * * @return Number of artifact instances having ArtifactType and - * ArtifactValue. + * ArtifactValue. */ @Override public Long getCountArtifactInstancesByTypeValue(CorrelationAttributeInstance.Type aType, String value) throws EamDbException { @@ -960,11 +992,11 @@ abstract class AbstractSqlEamDb implements EamDb { * associated with the caseDisplayName and dataSource of the given * eamArtifact instance. * - * @param caseUUID Case ID to search for + * @param caseUUID Case ID to search for * @param dataSourceID Data source ID to search for * * @return Number of artifact instances having caseDisplayName and - * dataSource + * dataSource */ @Override public Long getCountArtifactInstancesByCaseDataSource(String caseUUID, String dataSourceID) throws EamDbException { @@ -1227,7 +1259,7 @@ abstract class AbstractSqlEamDb implements EamDb { * associated CorrelationAttribute object. * * @param eamArtifact The correlation attribute whose database instance will - * be updated. + * be updated. * * @throws EamDbException */ @@ -1277,11 +1309,11 @@ abstract class AbstractSqlEamDb implements EamDb { * Find a correlation attribute in the Central Repository database given the * instance type, case, data source, value, and file path. * - * @param type The type of instance. - * @param correlationCase The case tied to the instance. + * @param type The type of instance. + * @param correlationCase The case tied to the instance. * @param correlationDataSource The data source tied to the instance. - * @param value The value tied to the instance. - * @param filePath The file path tied to the instance. + * @param value The value tied to the instance. + * @param filePath The file path tied to the instance. * * @return The correlation attribute if it exists; otherwise null. * @@ -1356,7 +1388,7 @@ abstract class AbstractSqlEamDb implements EamDb { * * @param eamArtifact Artifact containing exactly one (1) ArtifactInstance. * @param knownStatus The status to change the artifact to. Should never be - * KNOWN + * KNOWN */ @Override public void setAttributeInstanceKnownStatus(CorrelationAttributeInstance eamArtifact, TskData.FileKnown knownStatus) throws EamDbException { @@ -1618,7 +1650,7 @@ abstract class AbstractSqlEamDb implements EamDb { * @param value Value to search for * * @return List of cases containing this artifact with instances marked as - * bad + * bad * * @throws EamDbException */ @@ -1858,7 +1890,7 @@ abstract class AbstractSqlEamDb implements EamDb { /** * Process the Artifact instance in the EamDb * - * @param type EamArtifact.Type to search for + * @param type EamArtifact.Type to search for * @param instanceTableCallback callback to process the instance * * @throws EamDbException @@ -1897,9 +1929,10 @@ abstract class AbstractSqlEamDb implements EamDb { /** * Process the Artifact instance in the EamDb give a where clause * - * @param type EamArtifact.Type to search for + * @param type EamArtifact.Type to search for * @param instanceTableCallback callback to process the instance - * @param whereClause query string to execute + * @param whereClause query string to execute + * * @throws EamDbException */ @Override @@ -2081,7 +2114,7 @@ abstract class AbstractSqlEamDb implements EamDb { * Update an existing organization. * * @param updatedOrganization the values the Organization with the same ID - * will be updated to in the database. + * will be updated to in the database. * * @throws EamDbException */ @@ -2284,7 +2317,8 @@ abstract class AbstractSqlEamDb implements EamDb { * Add a new reference instance * * @param eamGlobalFileInstance The reference instance to add - * @param correlationType Correlation Type that this Reference Instance is + * @param correlationType Correlation Type that this Reference + * Instance is * * @throws EamDbException */ @@ -2412,7 +2446,7 @@ abstract class AbstractSqlEamDb implements EamDb { /** * Get all reference entries having a given correlation type and value * - * @param aType Type to use for matching + * @param aType Type to use for matching * @param aValue Value to use for matching * * @return List of all global file instances with a type and value @@ -2613,7 +2647,7 @@ abstract class AbstractSqlEamDb implements EamDb { * artifacts. * * @return List of enabled EamArtifact.Type's. If none are defined in the - * database, the default list will be returned. + * database, the default list will be returned. * * @throws EamDbException */ @@ -2648,7 +2682,7 @@ abstract class AbstractSqlEamDb implements EamDb { * correlate artifacts. * * @return List of supported EamArtifact.Type's. If none are defined in the - * database, the default list will be returned. + * database, the default list will be returned. * * @throws EamDbException */ @@ -2721,30 +2755,36 @@ abstract class AbstractSqlEamDb implements EamDb { */ @Override public CorrelationAttributeInstance.Type getCorrelationTypeById(int typeId) throws EamDbException { - Connection conn = connect(); - - CorrelationAttributeInstance.Type aType; - PreparedStatement preparedStatement = null; - ResultSet resultSet = null; - String sql = "SELECT * FROM correlation_types WHERE id=?"; - try { - preparedStatement = conn.prepareStatement(sql); - preparedStatement.setInt(1, typeId); - resultSet = preparedStatement.executeQuery(); - if (resultSet.next()) { - aType = getCorrelationTypeFromResultSet(resultSet); - return aType; - } else { - throw new EamDbException("Failed to find entry for correlation type ID = " + typeId); - } + return typeCache.get(CorrelationAttributeInstance.FILES_TYPE_ID, () -> { + Connection conn = connect(); - } catch (SQLException ex) { - throw new EamDbException("Error getting correlation type by id.", ex); // NON-NLS - } finally { - EamDbUtil.closeStatement(preparedStatement); - EamDbUtil.closeResultSet(resultSet); - EamDbUtil.closeConnection(conn); + CorrelationAttributeInstance.Type aType; + PreparedStatement preparedStatement = null; + ResultSet resultSet = null; + String sql = "SELECT * FROM correlation_types WHERE id=?"; + + try { + preparedStatement = conn.prepareStatement(sql); + preparedStatement.setInt(1, typeId); + resultSet = preparedStatement.executeQuery(); + if (resultSet.next()) { + aType = getCorrelationTypeFromResultSet(resultSet); + return aType; + } else { + throw new EamDbException("Failed to find entry for correlation type ID = " + typeId); + } + + } catch (SQLException ex) { + throw new EamDbException("Error getting correlation type by id.", ex); // NON-NLS + } finally { + EamDbUtil.closeStatement(preparedStatement); + EamDbUtil.closeResultSet(resultSet); + EamDbUtil.closeConnection(conn); + } + }); + } catch (ExecutionException ex) { + throw new EamDbException("Error getting correlation type", ex); } } @@ -2752,7 +2792,7 @@ abstract class AbstractSqlEamDb implements EamDb { * Convert a ResultSet to a EamCase object * * @param resultSet A resultSet with a set of values to create a EamCase - * object. + * object. * * @return fully populated EamCase object, or null * @@ -2822,7 +2862,7 @@ abstract class AbstractSqlEamDb implements EamDb { * Convert a ResultSet to a EamArtifactInstance object * * @param resultSet A resultSet with a set of values to create a - * EamArtifactInstance object. + * EamArtifactInstance object. * * @return fully populated EamArtifactInstance, or null * From 2e195cf16efb6db69d056759bf1b50061f1aeac1 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Mon, 27 Aug 2018 13:34:23 -0400 Subject: [PATCH 02/20] 4163 reset caches when CR connections are shutdown --- .../autopsy/centralrepository/datamodel/PostgresEamDb.java | 1 + .../autopsy/centralrepository/datamodel/SqliteEamDb.java | 1 + 2 files changed, 2 insertions(+) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDb.java index 5db232b51e..9e701fad3a 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDb.java @@ -78,6 +78,7 @@ final class PostgresEamDb extends AbstractSqlEamDb { connectionPool.close(); connectionPool = null; // force it to be re-created on next connect() } + clearCaches(); } } catch (SQLException ex) { throw new EamDbException("Failed to close existing database connections.", ex); // NON-NLS diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDb.java index d300964b5f..34157dea68 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDb.java @@ -85,6 +85,7 @@ final class SqliteEamDb extends AbstractSqlEamDb { connectionPool.close(); connectionPool = null; // force it to be re-created on next connect() } + clearCaches(); } } catch (SQLException ex) { throw new EamDbException("Failed to close existing database connections.", ex); // NON-NLS From 0b3a9fab0eaf3520ff30f9effc6c086757b1a2a0 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Mon, 27 Aug 2018 13:47:33 -0400 Subject: [PATCH 03/20] 4163 move queries to helper functions when caching for readability --- .../datamodel/AbstractSqlEamDb.java | 135 +++++++++++------- 1 file changed, 81 insertions(+), 54 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java index 0ce4941c09..825c61f837 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java @@ -536,6 +536,8 @@ abstract class AbstractSqlEamDb implements EamDb { * @param dataSourceDeviceId the data source device ID number * * @return The data source + * + * @throws EamDbException */ @Override public CorrelationDataSource getDataSource(CorrelationCase correlationCase, String dataSourceDeviceId) throws EamDbException { @@ -544,38 +546,52 @@ abstract class AbstractSqlEamDb implements EamDb { throw new EamDbException("Correlation case is null"); } try { - return dataSourceCache.get(correlationCase.getCaseUUID() + dataSourceDeviceId, () -> { - Connection conn = connect(); - - CorrelationDataSource eamDataSourceResult = null; - PreparedStatement preparedStatement = null; - ResultSet resultSet = null; - - String sql = "SELECT * FROM data_sources WHERE device_id=? AND case_id=?"; // NON-NLS - - try { - preparedStatement = conn.prepareStatement(sql); - preparedStatement.setString(1, dataSourceDeviceId); - preparedStatement.setInt(2, correlationCase.getID()); - resultSet = preparedStatement.executeQuery(); - if (resultSet.next()) { - eamDataSourceResult = getEamDataSourceFromResultSet(resultSet); - } - } catch (SQLException ex) { - throw new EamDbException("Error getting data source.", ex); // NON-NLS - } finally { - EamDbUtil.closeStatement(preparedStatement); - EamDbUtil.closeResultSet(resultSet); - EamDbUtil.closeConnection(conn); - } - - return eamDataSourceResult; - }); + return dataSourceCache.get(correlationCase.getCaseUUID() + dataSourceDeviceId, () -> getDataSourceFromCr(correlationCase, dataSourceDeviceId)); } catch (ExecutionException ex) { throw new EamDbException("Error getting data source from central repository", ex); } } + /** + * Gets the Data Source details based on data source device ID from the + * central repository. + * + * @param correlationCase the current CorrelationCase used for ensuring + * uniqueness of DataSource + * @param dataSourceDeviceId the data source device ID number + * + * @return The data source + * + * @throws EamDbException + */ + private CorrelationDataSource getDataSourceFromCr(CorrelationCase correlationCase, String dataSourceDeviceId) throws EamDbException { + Connection conn = connect(); + + CorrelationDataSource eamDataSourceResult = null; + PreparedStatement preparedStatement = null; + ResultSet resultSet = null; + + String sql = "SELECT * FROM data_sources WHERE device_id=? AND case_id=?"; // NON-NLS + + try { + preparedStatement = conn.prepareStatement(sql); + preparedStatement.setString(1, dataSourceDeviceId); + preparedStatement.setInt(2, correlationCase.getID()); + resultSet = preparedStatement.executeQuery(); + if (resultSet.next()) { + eamDataSourceResult = getEamDataSourceFromResultSet(resultSet); + } + } catch (SQLException ex) { + throw new EamDbException("Error getting data source.", ex); // NON-NLS + } finally { + EamDbUtil.closeStatement(preparedStatement); + EamDbUtil.closeResultSet(resultSet); + EamDbUtil.closeConnection(conn); + } + + return eamDataSourceResult; + } + /** * Retrieves Data Source details based on data source ID * @@ -2756,37 +2772,48 @@ abstract class AbstractSqlEamDb implements EamDb { @Override public CorrelationAttributeInstance.Type getCorrelationTypeById(int typeId) throws EamDbException { try { - return typeCache.get(CorrelationAttributeInstance.FILES_TYPE_ID, () -> { - Connection conn = connect(); - - CorrelationAttributeInstance.Type aType; - PreparedStatement preparedStatement = null; - ResultSet resultSet = null; - String sql = "SELECT * FROM correlation_types WHERE id=?"; - - try { - preparedStatement = conn.prepareStatement(sql); - preparedStatement.setInt(1, typeId); - resultSet = preparedStatement.executeQuery(); - if (resultSet.next()) { - aType = getCorrelationTypeFromResultSet(resultSet); - return aType; - } else { - throw new EamDbException("Failed to find entry for correlation type ID = " + typeId); - } - - } catch (SQLException ex) { - throw new EamDbException("Error getting correlation type by id.", ex); // NON-NLS - } finally { - EamDbUtil.closeStatement(preparedStatement); - EamDbUtil.closeResultSet(resultSet); - EamDbUtil.closeConnection(conn); - } - }); + return typeCache.get(CorrelationAttributeInstance.FILES_TYPE_ID, () ->getCorrelationTypeByIdFromCr(typeId)); } catch (ExecutionException ex) { throw new EamDbException("Error getting correlation type", ex); } } + + /** + * Get the EamArtifact.Type that has the given Type.Id from the central repo + * + * @param typeId Type.Id of Correlation Type to get + * + * @return EamArtifact.Type or null if it doesn't exist. + * + * @throws EamDbException + */ + private CorrelationAttributeInstance.Type getCorrelationTypeByIdFromCr(int typeId) throws EamDbException { + Connection conn = connect(); + + CorrelationAttributeInstance.Type aType; + PreparedStatement preparedStatement = null; + ResultSet resultSet = null; + String sql = "SELECT * FROM correlation_types WHERE id=?"; + + try { + preparedStatement = conn.prepareStatement(sql); + preparedStatement.setInt(1, typeId); + resultSet = preparedStatement.executeQuery(); + if (resultSet.next()) { + aType = getCorrelationTypeFromResultSet(resultSet); + return aType; + } else { + throw new EamDbException("Failed to find entry for correlation type ID = " + typeId); + } + + } catch (SQLException ex) { + throw new EamDbException("Error getting correlation type by id.", ex); // NON-NLS + } finally { + EamDbUtil.closeStatement(preparedStatement); + EamDbUtil.closeResultSet(resultSet); + EamDbUtil.closeConnection(conn); + } + } /** * Convert a ResultSet to a EamCase object From 288391f7d0dc4a211323eafed6e70af9eaa234d9 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Mon, 27 Aug 2018 17:13:45 -0400 Subject: [PATCH 04/20] 4163 cache based on both criteria we use to get case and datasource --- .../datamodel/AbstractSqlEamDb.java | 135 +++++++++++++++--- 1 file changed, 112 insertions(+), 23 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java index 825c61f837..d088cd76d6 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java @@ -45,6 +45,7 @@ import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.healthmonitor.HealthMonitor; import org.sleuthkit.autopsy.healthmonitor.TimingMetric; import org.sleuthkit.datamodel.CaseDbSchemaVersionNumber; +import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskData; /** @@ -64,13 +65,18 @@ abstract class AbstractSqlEamDb implements EamDb { private static final int CASE_CACHE_TIMEOUT = 5; private static final int DATA_SOURCE_CACHE_TIMEOUT = 5; private static final Cache typeCache = CacheBuilder.newBuilder().build(); - private static final Cache caseCache = CacheBuilder.newBuilder() + private static final Cache caseCacheByUUID = CacheBuilder.newBuilder() .expireAfterWrite(CASE_CACHE_TIMEOUT, TimeUnit.MINUTES). build(); - private static final Cache dataSourceCache = CacheBuilder.newBuilder() + private static final Cache caseCacheById = CacheBuilder.newBuilder() + .expireAfterWrite(CASE_CACHE_TIMEOUT, TimeUnit.MINUTES). + build(); + private static final Cache dataSourceCacheByDeviceId = CacheBuilder.newBuilder() + .expireAfterWrite(DATA_SOURCE_CACHE_TIMEOUT, TimeUnit.MINUTES). + build(); + private static final Cache dataSourceCacheById = CacheBuilder.newBuilder() .expireAfterWrite(DATA_SOURCE_CACHE_TIMEOUT, TimeUnit.MINUTES). build(); - // Maximum length for the value column in the instance tables static final int MAX_VALUE_LENGTH = 256; @@ -167,8 +173,10 @@ abstract class AbstractSqlEamDb implements EamDb { */ protected final void clearCaches() { typeCache.invalidateAll(); - caseCache.invalidateAll(); - dataSourceCache.invalidateAll(); + caseCacheByUUID.invalidateAll(); + caseCacheById.invalidateAll(); + dataSourceCacheByDeviceId.invalidateAll(); + dataSourceCacheById.invalidateAll(); } /** @@ -225,7 +233,7 @@ abstract class AbstractSqlEamDb implements EamDb { + getConflictClause(); try { - preparedStatement = conn.prepareStatement(sql); + preparedStatement = conn.prepareStatement(sql, Statement.RETURN_GENERATED_KEYS); preparedStatement.setString(1, eamCase.getCaseUUID()); if (null == eamCase.getOrg()) { @@ -262,6 +270,17 @@ abstract class AbstractSqlEamDb implements EamDb { } preparedStatement.executeUpdate(); + //update the case in the caches + ResultSet resultSet = preparedStatement.getGeneratedKeys(); + if (!resultSet.next()) { + throw new EamDbException(String.format("Failed to INSERT case %s in central repo", eamCase.getCaseUUID())); + } + int caseID = resultSet.getInt(1); //last_insert_rowid() + CorrelationCase correlationCase = new CorrelationCase(caseID, eamCase.getCaseUUID(), eamCase.getOrg(), + eamCase.getDisplayName(), eamCase.getCreationDate(), eamCase.getCaseNumber(), eamCase.getExaminerName(), + eamCase.getExaminerEmail(), eamCase.getExaminerPhone(), eamCase.getNotes()); + caseCacheByUUID.put(eamCase.getCaseUUID(), correlationCase); + caseCacheById.put(caseID, correlationCase); } catch (SQLException ex) { throw new EamDbException("Error inserting new case.", ex); // NON-NLS } finally { @@ -300,11 +319,7 @@ abstract class AbstractSqlEamDb implements EamDb { @Override public CorrelationCase getCase(Case autopsyCase) throws EamDbException { - try { - return caseCache.get(autopsyCase.getName(), () -> getCaseByUUID(autopsyCase.getName())); - } catch (ExecutionException ex) { - throw new EamDbException("Error getting autopsy case from Central repo", ex); - } + return getCaseByUUID(autopsyCase.getName()); } /** @@ -365,6 +380,9 @@ abstract class AbstractSqlEamDb implements EamDb { preparedStatement.setString(9, eamCase.getCaseUUID()); preparedStatement.executeUpdate(); + //update the case in the cache + caseCacheById.put(eamCase.getID(), eamCase); + caseCacheByUUID.put(eamCase.getCaseUUID(), eamCase); } catch (SQLException ex) { throw new EamDbException("Error updating case.", ex); // NON-NLS } finally { @@ -373,6 +391,22 @@ abstract class AbstractSqlEamDb implements EamDb { } } + /** + * Retrieves Case details based on Case UUID from the central repo + * + * @param caseUUID unique identifier for a case + * + * @return The retrieved case + */ + @Override + public CorrelationCase getCaseByUUID(String caseUUID) throws EamDbException { + try { + return caseCacheByUUID.get(caseUUID, () -> getCaseByUUIDFromCr(caseUUID)); + } catch (ExecutionException ex) { + throw new EamDbException("Error getting autopsy case from Central repo", ex); + } + } + /** * Retrieves Case details based on Case UUID * @@ -380,10 +414,7 @@ abstract class AbstractSqlEamDb implements EamDb { * * @return The retrieved case */ - @Override - public CorrelationCase getCaseByUUID(String caseUUID) throws EamDbException { - // @@@ We should have a cache here... - + private CorrelationCase getCaseByUUIDFromCr(String caseUUID) throws EamDbException { Connection conn = connect(); CorrelationCase eamCaseResult = null; @@ -403,6 +434,10 @@ abstract class AbstractSqlEamDb implements EamDb { if (resultSet.next()) { eamCaseResult = getEamCaseFromResultSet(resultSet); } + if (eamCaseResult != null) { + //Update the version in the other cache + caseCacheById.put(eamCaseResult.getID(), eamCaseResult); + } } catch (SQLException ex) { throw new EamDbException("Error getting case details.", ex); // NON-NLS } finally { @@ -423,8 +458,21 @@ abstract class AbstractSqlEamDb implements EamDb { */ @Override public CorrelationCase getCaseById(int caseId) throws EamDbException { - // @@@ We should have a cache here... + try { + return caseCacheById.get(caseId, () -> getCaseByIdFromCr(caseId)); + } catch (ExecutionException ex) { + throw new EamDbException("Error getting autopsy case from Central repo", ex); + } + } + /** + * Retrieves Case details based on Case ID + * + * @param caseID unique identifier for a case + * + * @return The retrieved case + */ + private CorrelationCase getCaseByIdFromCr(int caseId) throws EamDbException { Connection conn = connect(); CorrelationCase eamCaseResult = null; @@ -436,7 +484,6 @@ abstract class AbstractSqlEamDb implements EamDb { + "FROM cases " + "LEFT JOIN organizations ON cases.org_id=organizations.id " + "WHERE cases.id=?"; - try { preparedStatement = conn.prepareStatement(sql); preparedStatement.setInt(1, caseId); @@ -444,6 +491,10 @@ abstract class AbstractSqlEamDb implements EamDb { if (resultSet.next()) { eamCaseResult = getEamCaseFromResultSet(resultSet); } + if (eamCaseResult != null) { + //Update the version in the other cache + caseCacheByUUID.put(eamCaseResult.getCaseUUID(), eamCaseResult); + } } catch (SQLException ex) { throw new EamDbException("Error getting case details.", ex); // NON-NLS } finally { @@ -492,6 +543,14 @@ abstract class AbstractSqlEamDb implements EamDb { return cases; } + private static String getDataSourceCacheKey(int caseId, String dataSourceDeviceId) { + return "Case" + caseId + "DeviceId" + dataSourceDeviceId; + } + + private static String getDataSourceCacheKey(int caseId, int dataSourceId) { + return "Case" + caseId + "Id" + dataSourceId; + } + /** * Creates new Data Source in the database * @@ -513,13 +572,21 @@ abstract class AbstractSqlEamDb implements EamDb { + getConflictClause(); try { - preparedStatement = conn.prepareStatement(sql); + preparedStatement = conn.prepareStatement(sql, Statement.RETURN_GENERATED_KEYS); preparedStatement.setString(1, eamDataSource.getDeviceID()); preparedStatement.setInt(2, eamDataSource.getCaseID()); preparedStatement.setString(3, eamDataSource.getName()); preparedStatement.executeUpdate(); + ResultSet resultSet = preparedStatement.getGeneratedKeys(); + if (!resultSet.next()) { + throw new EamDbException(String.format("Failed to INSERT data source %s in central repo", eamDataSource.getName())); + } + int dataSourceId = resultSet.getInt(1); //last_insert_rowid() + CorrelationDataSource dataSource = new CorrelationDataSource(eamDataSource.getCaseID(), dataSourceId, eamDataSource.getDeviceID(), eamDataSource.getName()); + dataSourceCacheByDeviceId.put(getDataSourceCacheKey(dataSource.getCaseID(), dataSource.getDeviceID()), dataSource); + dataSourceCacheById.put(getDataSourceCacheKey(dataSource.getCaseID(), dataSource.getID()), dataSource); } catch (SQLException ex) { throw new EamDbException("Error inserting new data source.", ex); // NON-NLS } finally { @@ -546,7 +613,7 @@ abstract class AbstractSqlEamDb implements EamDb { throw new EamDbException("Correlation case is null"); } try { - return dataSourceCache.get(correlationCase.getCaseUUID() + dataSourceDeviceId, () -> getDataSourceFromCr(correlationCase, dataSourceDeviceId)); + return dataSourceCacheByDeviceId.get(getDataSourceCacheKey(correlationCase.getID(), dataSourceDeviceId), () -> getDataSourceFromCr(correlationCase, dataSourceDeviceId)); } catch (ExecutionException ex) { throw new EamDbException("Error getting data source from central repository", ex); } @@ -581,6 +648,9 @@ abstract class AbstractSqlEamDb implements EamDb { if (resultSet.next()) { eamDataSourceResult = getEamDataSourceFromResultSet(resultSet); } + if (eamDataSourceResult != null) { + dataSourceCacheById.put(getDataSourceCacheKey(correlationCase.getID(), eamDataSourceResult.getID()), eamDataSourceResult); + } } catch (SQLException ex) { throw new EamDbException("Error getting data source.", ex); // NON-NLS } finally { @@ -606,7 +676,23 @@ abstract class AbstractSqlEamDb implements EamDb { if (correlationCase == null) { throw new EamDbException("Correlation case is null"); } + try { + return dataSourceCacheByDeviceId.get(getDataSourceCacheKey(correlationCase.getID(), dataSourceId), () -> getDataSourceByIdFromCr(correlationCase, dataSourceId)); + } catch (ExecutionException ex) { + throw new EamDbException("Error getting data source from central repository", ex); + } + } + /** + * Retrieves Data Source details based on data source ID + * + * @param correlationCase the current CorrelationCase used for ensuring + * uniqueness of DataSource + * @param dataSourceId the data source ID number + * + * @return The data source + */ + private CorrelationDataSource getDataSourceByIdFromCr(CorrelationCase correlationCase, int dataSourceId) throws EamDbException { Connection conn = connect(); CorrelationDataSource eamDataSourceResult = null; @@ -623,6 +709,9 @@ abstract class AbstractSqlEamDb implements EamDb { if (resultSet.next()) { eamDataSourceResult = getEamDataSourceFromResultSet(resultSet); } + if (eamDataSourceResult != null) { + dataSourceCacheByDeviceId.put(getDataSourceCacheKey(correlationCase.getID(), eamDataSourceResult.getDeviceID()), eamDataSourceResult); + } } catch (SQLException ex) { throw new EamDbException("Error getting data source.", ex); // NON-NLS } finally { @@ -2750,7 +2839,7 @@ abstract class AbstractSqlEamDb implements EamDb { preparedStatement.setInt(4, aType.isEnabled() ? 1 : 0); preparedStatement.setInt(5, aType.getId()); preparedStatement.executeUpdate(); - + typeCache.put(aType.getId(), aType); } catch (SQLException ex) { throw new EamDbException("Error updating correlation type.", ex); // NON-NLS } finally { @@ -2772,13 +2861,13 @@ abstract class AbstractSqlEamDb implements EamDb { @Override public CorrelationAttributeInstance.Type getCorrelationTypeById(int typeId) throws EamDbException { try { - return typeCache.get(CorrelationAttributeInstance.FILES_TYPE_ID, () ->getCorrelationTypeByIdFromCr(typeId)); + return typeCache.get(CorrelationAttributeInstance.FILES_TYPE_ID, () -> getCorrelationTypeByIdFromCr(typeId)); } catch (ExecutionException ex) { throw new EamDbException("Error getting correlation type", ex); } } - - /** + + /** * Get the EamArtifact.Type that has the given Type.Id from the central repo * * @param typeId Type.Id of Correlation Type to get From fc4e549dd160cfed31b84ca9293bb0da30774b79 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Mon, 27 Aug 2018 17:15:54 -0400 Subject: [PATCH 05/20] 4163 remove unused import --- .../autopsy/centralrepository/datamodel/AbstractSqlEamDb.java | 1 - 1 file changed, 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java index d088cd76d6..be1518cb42 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java @@ -45,7 +45,6 @@ import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.healthmonitor.HealthMonitor; import org.sleuthkit.autopsy.healthmonitor.TimingMetric; import org.sleuthkit.datamodel.CaseDbSchemaVersionNumber; -import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskData; /** From ebcac1eb244ef04676b8c639eef6bbfb2a69714a Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Tue, 28 Aug 2018 12:01:39 -0400 Subject: [PATCH 06/20] 4163 comments and refactoring for clarity with new caches --- .../datamodel/AbstractSqlEamDb.java | 35 +++++++++++++------ 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java index be1518cb42..cfa39a8f32 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java @@ -542,12 +542,27 @@ abstract class AbstractSqlEamDb implements EamDb { return cases; } - private static String getDataSourceCacheKey(int caseId, String dataSourceDeviceId) { - return "Case" + caseId + "DeviceId" + dataSourceDeviceId; + /** + * Create a key to the DataSourceCacheByDeviceId + * + * @param caseId - the id of the CorrelationCase in the Central Repository + * @param dataSourceDeviceId - the device Id of the data source + * + * @return a String to be used as a key for the dataSourceCacheByDeviceId + */ + private static String getDataSourceByDeviceIdCacheKey(int caseId, String dataSourceDeviceId) { + return "Case" + caseId + "DeviceId" + dataSourceDeviceId; //NON-NLS } - private static String getDataSourceCacheKey(int caseId, int dataSourceId) { - return "Case" + caseId + "Id" + dataSourceId; + /** + * Create a key to the DataSourceCacheById + * + * @param caseId - the id of the CorrelationCase in the Central Repository + * @param dataSourceId - the id of the datasource in the central repository + * @return a String to be used as a key for the dataSourceCacheById + */ + private static String getDataSourceByIdCacheKey(int caseId, int dataSourceId) { + return "Case" + caseId + "Id" + dataSourceId; //NON-NLS } /** @@ -584,8 +599,8 @@ abstract class AbstractSqlEamDb implements EamDb { } int dataSourceId = resultSet.getInt(1); //last_insert_rowid() CorrelationDataSource dataSource = new CorrelationDataSource(eamDataSource.getCaseID(), dataSourceId, eamDataSource.getDeviceID(), eamDataSource.getName()); - dataSourceCacheByDeviceId.put(getDataSourceCacheKey(dataSource.getCaseID(), dataSource.getDeviceID()), dataSource); - dataSourceCacheById.put(getDataSourceCacheKey(dataSource.getCaseID(), dataSource.getID()), dataSource); + dataSourceCacheByDeviceId.put(getDataSourceByDeviceIdCacheKey(dataSource.getCaseID(), dataSource.getDeviceID()), dataSource); + dataSourceCacheById.put(getDataSourceByIdCacheKey(dataSource.getCaseID(), dataSource.getID()), dataSource); } catch (SQLException ex) { throw new EamDbException("Error inserting new data source.", ex); // NON-NLS } finally { @@ -612,7 +627,7 @@ abstract class AbstractSqlEamDb implements EamDb { throw new EamDbException("Correlation case is null"); } try { - return dataSourceCacheByDeviceId.get(getDataSourceCacheKey(correlationCase.getID(), dataSourceDeviceId), () -> getDataSourceFromCr(correlationCase, dataSourceDeviceId)); + return dataSourceCacheByDeviceId.get(getDataSourceByDeviceIdCacheKey(correlationCase.getID(), dataSourceDeviceId), () -> getDataSourceFromCr(correlationCase, dataSourceDeviceId)); } catch (ExecutionException ex) { throw new EamDbException("Error getting data source from central repository", ex); } @@ -648,7 +663,7 @@ abstract class AbstractSqlEamDb implements EamDb { eamDataSourceResult = getEamDataSourceFromResultSet(resultSet); } if (eamDataSourceResult != null) { - dataSourceCacheById.put(getDataSourceCacheKey(correlationCase.getID(), eamDataSourceResult.getID()), eamDataSourceResult); + dataSourceCacheById.put(getDataSourceByIdCacheKey(correlationCase.getID(), eamDataSourceResult.getID()), eamDataSourceResult); } } catch (SQLException ex) { throw new EamDbException("Error getting data source.", ex); // NON-NLS @@ -676,7 +691,7 @@ abstract class AbstractSqlEamDb implements EamDb { throw new EamDbException("Correlation case is null"); } try { - return dataSourceCacheByDeviceId.get(getDataSourceCacheKey(correlationCase.getID(), dataSourceId), () -> getDataSourceByIdFromCr(correlationCase, dataSourceId)); + return dataSourceCacheByDeviceId.get(getDataSourceByIdCacheKey(correlationCase.getID(), dataSourceId), () -> getDataSourceByIdFromCr(correlationCase, dataSourceId)); } catch (ExecutionException ex) { throw new EamDbException("Error getting data source from central repository", ex); } @@ -709,7 +724,7 @@ abstract class AbstractSqlEamDb implements EamDb { eamDataSourceResult = getEamDataSourceFromResultSet(resultSet); } if (eamDataSourceResult != null) { - dataSourceCacheByDeviceId.put(getDataSourceCacheKey(correlationCase.getID(), eamDataSourceResult.getDeviceID()), eamDataSourceResult); + dataSourceCacheByDeviceId.put(getDataSourceByDeviceIdCacheKey(correlationCase.getID(), eamDataSourceResult.getDeviceID()), eamDataSourceResult); } } catch (SQLException ex) { throw new EamDbException("Error getting data source.", ex); // NON-NLS From 056f4e953430fb275355e623d8881b8f1078a1bf Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Tue, 28 Aug 2018 12:04:15 -0400 Subject: [PATCH 07/20] 4163 update copyright dates --- .../autopsy/centralrepository/datamodel/PostgresEamDb.java | 2 +- .../autopsy/centralrepository/datamodel/SqliteEamDb.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDb.java index 9e701fad3a..97abd1dec9 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDb.java @@ -1,7 +1,7 @@ /* * Central Repository * - * Copyright 2015-2017 Basis Technology Corp. + * Copyright 2015-2018 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDb.java index 34157dea68..e4d7b7bd9d 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDb.java @@ -1,7 +1,7 @@ /* * Central Repository * - * Copyright 2015-2017 Basis Technology Corp. + * Copyright 2015-2018 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); From 2f57b436768c0bdedd17336c99504f9f5b34bd3f Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Tue, 28 Aug 2018 12:52:42 -0400 Subject: [PATCH 08/20] 4163 fix bug using wrong cache --- .../autopsy/centralrepository/datamodel/AbstractSqlEamDb.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java index cfa39a8f32..68f1b4587a 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java @@ -691,7 +691,7 @@ abstract class AbstractSqlEamDb implements EamDb { throw new EamDbException("Correlation case is null"); } try { - return dataSourceCacheByDeviceId.get(getDataSourceByIdCacheKey(correlationCase.getID(), dataSourceId), () -> getDataSourceByIdFromCr(correlationCase, dataSourceId)); + return dataSourceCacheById.get(getDataSourceByIdCacheKey(correlationCase.getID(), dataSourceId), () -> getDataSourceByIdFromCr(correlationCase, dataSourceId)); } catch (ExecutionException ex) { throw new EamDbException("Error getting data source from central repository", ex); } From 740d5e2378047c1435ed90d57a6adfc6178ca882 Mon Sep 17 00:00:00 2001 From: millmanorama Date: Tue, 18 Sep 2018 14:03:00 +0200 Subject: [PATCH 09/20] fix minor bugs in SortChooser.java --- .../autopsy/imagegallery/gui/SortChooser.java | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/SortChooser.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/SortChooser.java index 49543d5f2d..5f14f1d8b9 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/SortChooser.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/SortChooser.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2016 Basis Technology Corp. + * Copyright 2016-18 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -54,7 +54,7 @@ public class SortChooser> extends HBox { private final ReadOnlyObjectWrapper sortOrder = new ReadOnlyObjectWrapper<>(SortOrder.ASCENDING); private final SimpleBooleanProperty sortOrderDisabled = new SimpleBooleanProperty(false); - private final SimpleObjectProperty valueType = new SimpleObjectProperty<>(ValueType.NUMERIC); + private final SimpleObjectProperty valueType = new SimpleObjectProperty<>(ValueType.LEXICOGRAPHIC); public SortChooser(ObservableList comps) { this.comparators = comps; @@ -73,15 +73,13 @@ public class SortChooser> extends HBox { descRadio.getStyleClass().remove("radio-button"); descRadio.getStyleClass().add("toggle-button"); - valueType.addListener((observable, oldValue, newValue) -> { - ascRadio.setGraphic(new ImageView(newValue.getAscendingImage())); - descRadio.setGraphic(new ImageView(newValue.getDescendingImage())); - }); + valueType.addListener(observable -> setValueTypeIcon(valueType.getValue())); + setValueTypeIcon(valueType.getValue()); ascRadio.disableProperty().bind(sortOrderDisabled); descRadio.disableProperty().bind(sortOrderDisabled); ascRadio.selectedProperty().addListener(selectedToggle -> { - sortOrder.set(orderGroup.getSelectedToggle() == ascRadio ? SortOrder.ASCENDING : SortOrder.DESCENDING); + sortOrder.set(ascRadio.isSelected() ? SortOrder.ASCENDING : SortOrder.DESCENDING); }); sortByBox.setItems(comparators); @@ -89,6 +87,11 @@ public class SortChooser> extends HBox { sortByBox.setButtonCell(new ComparatorCell()); } + private void setValueTypeIcon(ValueType newValue) { + ascRadio.setGraphic(new ImageView(newValue.getAscendingImage())); + descRadio.setGraphic(new ImageView(newValue.getDescendingImage())); + } + public ValueType getValueType() { return valueType.get(); } From 8c737281d40de9eefc04bf7041aaeb20c3bf44c7 Mon Sep 17 00:00:00 2001 From: Ann Priestman Date: Tue, 18 Sep 2018 09:21:57 -0400 Subject: [PATCH 10/20] Don't use the intracase data source list in intercase searches. Fix bug with common file parent path in the root directory. --- .../AbstractCommonAttributeSearcher.java | 8 +-- .../AllInterCaseCommonAttributeSearcher.java | 6 +-- .../CentralRepoCommonAttributeInstance.java | 53 ++++++++++--------- .../CommonAttributePanel.java | 4 +- .../InterCaseCommonAttributeSearcher.java | 4 +- .../InterCaseSearchResultsProcessor.java | 19 +------ .../IntraCaseCommonAttributeSearcher.java | 11 +++- ...ingleInterCaseCommonAttributeSearcher.java | 6 +-- 8 files changed, 52 insertions(+), 59 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/AbstractCommonAttributeSearcher.java b/Core/src/org/sleuthkit/autopsy/commonfilesearch/AbstractCommonAttributeSearcher.java index f79737c116..2e95e83007 100644 --- a/Core/src/org/sleuthkit/autopsy/commonfilesearch/AbstractCommonAttributeSearcher.java +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/AbstractCommonAttributeSearcher.java @@ -39,22 +39,16 @@ import org.sleuthkit.datamodel.TskCoreException; */ public abstract class AbstractCommonAttributeSearcher { - private final Map dataSourceIdToNameMap; private boolean filterByMedia; private boolean filterByDoc; final int frequencyPercentageThreshold; - AbstractCommonAttributeSearcher(Map dataSourceIdMap, boolean filterByMedia, boolean filterByDoc, int percentageThreshold) { + AbstractCommonAttributeSearcher(boolean filterByMedia, boolean filterByDoc, int percentageThreshold) { this.filterByDoc = filterByDoc; this.filterByMedia = filterByMedia; - this.dataSourceIdToNameMap = dataSourceIdMap; this.frequencyPercentageThreshold = percentageThreshold; } - Map getDataSourceIdToNameMap() { - return Collections.unmodifiableMap(this.dataSourceIdToNameMap); - } - /** * Implement this to search for files with common attributes. Creates an * object (CommonAttributeSearchResults) which contains all of the diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/AllInterCaseCommonAttributeSearcher.java b/Core/src/org/sleuthkit/autopsy/commonfilesearch/AllInterCaseCommonAttributeSearcher.java index f28d636be0..a64d8b0758 100644 --- a/Core/src/org/sleuthkit/autopsy/commonfilesearch/AllInterCaseCommonAttributeSearcher.java +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/AllInterCaseCommonAttributeSearcher.java @@ -43,13 +43,13 @@ public class AllInterCaseCommonAttributeSearcher extends InterCaseCommonAttribut * * @throws EamDbException */ - public AllInterCaseCommonAttributeSearcher(Map dataSourceIdMap, boolean filterByMediaMimeType, boolean filterByDocMimeType, Type corAttrType, int percentageThreshold) throws EamDbException { - super(dataSourceIdMap, filterByMediaMimeType, filterByDocMimeType, corAttrType, percentageThreshold); + public AllInterCaseCommonAttributeSearcher(boolean filterByMediaMimeType, boolean filterByDocMimeType, Type corAttrType, int percentageThreshold) throws EamDbException { + super(filterByMediaMimeType, filterByDocMimeType, corAttrType, percentageThreshold); } @Override public CommonAttributeSearchResults findMatches() throws TskCoreException, NoCurrentCaseException, SQLException, EamDbException { - InterCaseSearchResultsProcessor eamDbAttrInst = new InterCaseSearchResultsProcessor(this.getDataSourceIdToNameMap(), corAttrType); + InterCaseSearchResultsProcessor eamDbAttrInst = new InterCaseSearchResultsProcessor(corAttrType); Map interCaseCommonFiles = eamDbAttrInst.findInterCaseCommonAttributeValues(Case.getCurrentCase()); return new CommonAttributeSearchResults(interCaseCommonFiles, this.frequencyPercentageThreshold, this.corAttrType); } diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CentralRepoCommonAttributeInstance.java b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CentralRepoCommonAttributeInstance.java index df6d24b0bb..855fb3e7c1 100644 --- a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CentralRepoCommonAttributeInstance.java +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CentralRepoCommonAttributeInstance.java @@ -24,6 +24,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.logging.Level; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; @@ -31,6 +32,7 @@ import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeIns import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.datamodel.DisplayableItemNode; import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.datamodel.DataSource; import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.TskCoreException; @@ -45,12 +47,10 @@ final public class CentralRepoCommonAttributeInstance extends AbstractCommonAttr private final Integer crFileId; private CorrelationAttributeInstance currentAttribute; private final CorrelationAttributeInstance.Type correlationType; - private final Map dataSourceNameToIdMap; - CentralRepoCommonAttributeInstance(Integer attrInstId, Map dataSourceIdToNameMap, CorrelationAttributeInstance.Type correlationType) { + CentralRepoCommonAttributeInstance(Integer attrInstId, CorrelationAttributeInstance.Type correlationType) { super(); this.crFileId = attrInstId; - this.dataSourceNameToIdMap = invertMap(dataSourceIdToNameMap); this.correlationType = correlationType; } @@ -71,22 +71,35 @@ final public class CentralRepoCommonAttributeInstance extends AbstractCommonAttr final CorrelationAttributeInstance currentAttributeInstance = this.currentAttribute; - String currentFullPath = currentAttributeInstance.getFilePath(); - String currentDataSource = currentAttributeInstance.getCorrelationDataSource().getName(); + try { + String currentFullPath = currentAttributeInstance.getFilePath(); + currentCase = Case.getCurrentCaseThrows(); - if (this.dataSourceNameToIdMap.containsKey(currentDataSource)) { - Long dataSourceObjectId = this.dataSourceNameToIdMap.get(currentDataSource); - - try { - currentCase = Case.getCurrentCaseThrows(); + // Only attempt to make the abstract file if the attribute is from the current case + if (currentCase.getName().equals(currentAttributeInstance.getCorrelationCase().getCaseUUID())) { SleuthkitCase tskDb = currentCase.getSleuthkitCase(); + + // Find the correct data source + Optional dataSource = tskDb.getDataSources().stream() + .filter(p -> p.getDeviceId().equals(currentAttribute.getCorrelationDataSource().getDeviceID())) + .findFirst(); + if (! dataSource.isPresent()) { + LOGGER.log(Level.WARNING, String.format("Unable to find data source with device ID %s in the current case", currentAttribute.getCorrelationDataSource().getDeviceID())); + return null; + } File fileFromPath = new File(currentFullPath); String fileName = fileFromPath.getName(); - String parentPath = (fileFromPath.getParent() + File.separator).replace("\\", "/"); + + // Create the parent path. Make sure not to add a separator if there is already one there. + String parentPath = fileFromPath.getParent(); + if (! parentPath.endsWith(File.separator)) { + parentPath = parentPath + File.separator; + } + parentPath = parentPath.replace("\\", "/"); - final String whereClause = String.format("lower(name) = '%s' AND md5 = '%s' AND lower(parent_path) = '%s' AND data_source_obj_id = %s", fileName, currentAttribute.getCorrelationValue(), parentPath, dataSourceObjectId); + final String whereClause = String.format("lower(name) = '%s' AND md5 = '%s' AND lower(parent_path) = '%s' AND data_source_obj_id = %s", fileName, currentAttribute.getCorrelationValue(), parentPath, dataSource.get().getId()); List potentialAbstractFiles = tskDb.findAllFilesWhere(whereClause); if (potentialAbstractFiles.isEmpty()) { @@ -97,14 +110,14 @@ final public class CentralRepoCommonAttributeInstance extends AbstractCommonAttr } else { return potentialAbstractFiles.get(0); } - - } catch (TskCoreException | NoCurrentCaseException ex) { - LOGGER.log(Level.SEVERE, String.format("Unable to find AbstractFile for record with filePath: %s. Node not created.", new Object[]{currentFullPath}), ex); + } else { return null; } - } else { + } catch (TskCoreException | NoCurrentCaseException ex) { + LOGGER.log(Level.SEVERE, String.format("Unable to find AbstractFile for record with filePath: %s. Node not created.", new Object[]{currentAttributeInstance.getFilePath()}), ex); return null; } + } return null; } @@ -131,12 +144,4 @@ final public class CentralRepoCommonAttributeInstance extends AbstractCommonAttr return attrInstNodeList.toArray(new DisplayableItemNode[attrInstNodeList.size()]); } - - private Map invertMap(Map dataSourceIdToNameMap) { - HashMap invertedMap = new HashMap<>(); - for (Map.Entry entry : dataSourceIdToNameMap.entrySet()) { - invertedMap.put(entry.getValue(), entry.getKey()); - } - return invertedMap; - } } diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonAttributePanel.java b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonAttributePanel.java index c4aa8019f2..40324ab64a 100644 --- a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonAttributePanel.java +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CommonAttributePanel.java @@ -231,10 +231,10 @@ final class CommonAttributePanel extends javax.swing.JDialog implements Observer corType = CorrelationAttributeInstance.getDefaultCorrelationTypes().get(0); } if (caseId == InterCasePanel.NO_CASE_SELECTED) { - builder = new AllInterCaseCommonAttributeSearcher(intraCasePanel.getDataSourceMap(), filterByMedia, filterByDocuments, corType, percentageThreshold); + builder = new AllInterCaseCommonAttributeSearcher(filterByMedia, filterByDocuments, corType, percentageThreshold); } else { - builder = new SingleInterCaseCommonAttributeSearcher(caseId, intraCasePanel.getDataSourceMap(), filterByMedia, filterByDocuments, corType, percentageThreshold); + builder = new SingleInterCaseCommonAttributeSearcher(caseId, filterByMedia, filterByDocuments, corType, percentageThreshold); } } else { diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/InterCaseCommonAttributeSearcher.java b/Core/src/org/sleuthkit/autopsy/commonfilesearch/InterCaseCommonAttributeSearcher.java index 931ee7ffcc..8219ab9196 100644 --- a/Core/src/org/sleuthkit/autopsy/commonfilesearch/InterCaseCommonAttributeSearcher.java +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/InterCaseCommonAttributeSearcher.java @@ -48,8 +48,8 @@ abstract class InterCaseCommonAttributeSearcher extends AbstractCommonAttributeS * * @throws EamDbException */ - InterCaseCommonAttributeSearcher(Map dataSourceIdMap, boolean filterByMediaMimeType, boolean filterByDocMimeType, Type corAttrType, int percentageThreshold) throws EamDbException { - super(dataSourceIdMap, filterByMediaMimeType, filterByDocMimeType, percentageThreshold); + InterCaseCommonAttributeSearcher(boolean filterByMediaMimeType, boolean filterByDocMimeType, Type corAttrType, int percentageThreshold) throws EamDbException { + super(filterByMediaMimeType, filterByDocMimeType, percentageThreshold); dbManager = EamDb.getInstance(); this.corAttrType = corAttrType; } diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/InterCaseSearchResultsProcessor.java b/Core/src/org/sleuthkit/autopsy/commonfilesearch/InterCaseSearchResultsProcessor.java index d258db2cc1..5b178eea73 100644 --- a/Core/src/org/sleuthkit/autopsy/commonfilesearch/InterCaseSearchResultsProcessor.java +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/InterCaseSearchResultsProcessor.java @@ -44,8 +44,6 @@ import org.sleuthkit.datamodel.HashUtility; */ final class InterCaseSearchResultsProcessor { - private Map dataSources; - /** * The CorrelationAttributeInstance.Type this Processor will query on */ @@ -70,9 +68,8 @@ final class InterCaseSearchResultsProcessor { * @param dataSources the cases to filter and correlate on * @param theType the type of CR data to search */ - InterCaseSearchResultsProcessor(Map dataSources, CorrelationAttributeInstance.Type theType) { + InterCaseSearchResultsProcessor(CorrelationAttributeInstance.Type theType) { this.correlationType = theType; - this.dataSources = dataSources; interCaseWhereClause = getInterCaseWhereClause(); singleInterCaseWhereClause = getSingleInterCaseWhereClause(); } @@ -101,18 +98,6 @@ final class InterCaseSearchResultsProcessor { return sqlString.toString(); } - /** - * Used in the CentralRepoCommonAttributeInstance to find common attribute - * instances and generate nodes at the UI level. - * - * @param theType the type of CR data to search - */ - InterCaseSearchResultsProcessor(CorrelationAttributeInstance.Type theType) { - this.correlationType = theType; - interCaseWhereClause = getInterCaseWhereClause(); - singleInterCaseWhereClause = getSingleInterCaseWhereClause(); - } - /** * Finds a single CorrelationAttribute given an id. * @@ -252,7 +237,7 @@ final class InterCaseSearchResultsProcessor { // we don't *have* all the information for the rows in the CR, // so we need to consult the present case via the SleuthkitCase object // Later, when the FileInstanceNode is built. Therefore, build node generators for now. - AbstractCommonAttributeInstance searchResult = new CentralRepoCommonAttributeInstance(resultId, InterCaseSearchResultsProcessor.this.dataSources, correlationType); + AbstractCommonAttributeInstance searchResult = new CentralRepoCommonAttributeInstance(resultId, correlationType); commonAttributeValue.addInstance(searchResult); } diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/IntraCaseCommonAttributeSearcher.java b/Core/src/org/sleuthkit/autopsy/commonfilesearch/IntraCaseCommonAttributeSearcher.java index 569ae5232f..ac8a009b11 100644 --- a/Core/src/org/sleuthkit/autopsy/commonfilesearch/IntraCaseCommonAttributeSearcher.java +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/IntraCaseCommonAttributeSearcher.java @@ -21,6 +21,7 @@ package org.sleuthkit.autopsy.commonfilesearch; import java.sql.ResultSet; import java.sql.SQLException; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -46,6 +47,8 @@ public abstract class IntraCaseCommonAttributeSearcher extends AbstractCommonAtt private static final String FILTER_BY_MIME_TYPES_WHERE_CLAUSE = " and mime_type in (%s)"; //NON-NLS // where %s is csv list of mime_types to filter on + private final Map dataSourceIdToNameMap; + /** * Subclass this to implement different algorithms for getting common files. * @@ -56,7 +59,13 @@ public abstract class IntraCaseCommonAttributeSearcher extends AbstractCommonAtt * broadly categorized as document types */ IntraCaseCommonAttributeSearcher(Map dataSourceIdMap, boolean filterByMediaMimeType, boolean filterByDocMimeType, int percentageThreshold) { - super(dataSourceIdMap, filterByMediaMimeType, filterByDocMimeType, percentageThreshold); + super(filterByMediaMimeType, filterByDocMimeType, percentageThreshold); + this.dataSourceIdToNameMap = dataSourceIdMap; + } + + + Map getDataSourceIdToNameMap() { + return Collections.unmodifiableMap(this.dataSourceIdToNameMap); } /** diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/SingleInterCaseCommonAttributeSearcher.java b/Core/src/org/sleuthkit/autopsy/commonfilesearch/SingleInterCaseCommonAttributeSearcher.java index 3d2abda13c..de8abfce20 100644 --- a/Core/src/org/sleuthkit/autopsy/commonfilesearch/SingleInterCaseCommonAttributeSearcher.java +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/SingleInterCaseCommonAttributeSearcher.java @@ -46,9 +46,9 @@ public class SingleInterCaseCommonAttributeSearcher extends InterCaseCommonAttri * * @throws EamDbException */ - public SingleInterCaseCommonAttributeSearcher(int correlationCaseId, Map dataSourceIdMap, boolean filterByMediaMimeType, + public SingleInterCaseCommonAttributeSearcher(int correlationCaseId, boolean filterByMediaMimeType, boolean filterByDocMimeType, Type corAttrType, int percentageThreshold) throws EamDbException { - super(dataSourceIdMap, filterByMediaMimeType, filterByDocMimeType, corAttrType, percentageThreshold); + super(filterByMediaMimeType, filterByDocMimeType, corAttrType, percentageThreshold); this.corrleationCaseId = correlationCaseId; this.correlationCaseName = ""; @@ -77,7 +77,7 @@ public class SingleInterCaseCommonAttributeSearcher extends InterCaseCommonAttri } CommonAttributeSearchResults findFiles(CorrelationCase correlationCase) throws TskCoreException, NoCurrentCaseException, SQLException, EamDbException { - InterCaseSearchResultsProcessor eamDbAttrInst = new InterCaseSearchResultsProcessor(this.getDataSourceIdToNameMap(), this.corAttrType); + InterCaseSearchResultsProcessor eamDbAttrInst = new InterCaseSearchResultsProcessor(this.corAttrType); Map interCaseCommonFiles = eamDbAttrInst.findSingleInterCaseCommonAttributeValues(Case.getCurrentCase(), correlationCase); return new CommonAttributeSearchResults(interCaseCommonFiles, this.frequencyPercentageThreshold, this.corAttrType); From d195fe002824642b7a6d0a917422b786afc10095 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Tue, 18 Sep 2018 18:13:23 -0400 Subject: [PATCH 11/20] 4163 close unclosed result sets codacy noticed --- .../centralrepository/datamodel/AbstractSqlEamDb.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java index 6385a2eafc..cd2a11b031 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java @@ -230,7 +230,7 @@ abstract class AbstractSqlEamDb implements EamDb { + "examiner_name, examiner_email, examiner_phone, notes) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) " + getConflictClause(); - + ResultSet resultSet = null; try { preparedStatement = conn.prepareStatement(sql, Statement.RETURN_GENERATED_KEYS); @@ -270,7 +270,7 @@ abstract class AbstractSqlEamDb implements EamDb { preparedStatement.executeUpdate(); //update the case in the caches - ResultSet resultSet = preparedStatement.getGeneratedKeys(); + resultSet = preparedStatement.getGeneratedKeys(); if (!resultSet.next()) { throw new EamDbException(String.format("Failed to INSERT case %s in central repo", eamCase.getCaseUUID())); } @@ -283,6 +283,7 @@ abstract class AbstractSqlEamDb implements EamDb { } catch (SQLException ex) { throw new EamDbException("Error inserting new case.", ex); // NON-NLS } finally { + EamDbUtil.closeResultSet(resultSet); EamDbUtil.closeStatement(preparedStatement); EamDbUtil.closeConnection(conn); } @@ -584,7 +585,7 @@ abstract class AbstractSqlEamDb implements EamDb { String sql = "INSERT INTO data_sources(device_id, case_id, name) VALUES (?, ?, ?) " + getConflictClause(); - + ResultSet resultSet = null; try { preparedStatement = conn.prepareStatement(sql, Statement.RETURN_GENERATED_KEYS); @@ -593,7 +594,7 @@ abstract class AbstractSqlEamDb implements EamDb { preparedStatement.setString(3, eamDataSource.getName()); preparedStatement.executeUpdate(); - ResultSet resultSet = preparedStatement.getGeneratedKeys(); + resultSet = preparedStatement.getGeneratedKeys(); if (!resultSet.next()) { throw new EamDbException(String.format("Failed to INSERT data source %s in central repo", eamDataSource.getName())); } @@ -604,6 +605,7 @@ abstract class AbstractSqlEamDb implements EamDb { } catch (SQLException ex) { throw new EamDbException("Error inserting new data source.", ex); // NON-NLS } finally { + EamDbUtil.closeResultSet(resultSet); EamDbUtil.closeStatement(preparedStatement); EamDbUtil.closeConnection(conn); } From dc478fc05a9b564e80e155469795c3a2d5562b9b Mon Sep 17 00:00:00 2001 From: millmanorama Date: Wed, 19 Sep 2018 12:35:05 +0200 Subject: [PATCH 12/20] update seen status of groups when files are added --- .../datamodel/grouping/GroupManager.java | 38 +++++++++++-------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupManager.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupManager.java index 95d00ee84a..618f48af79 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupManager.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupManager.java @@ -42,6 +42,7 @@ import java.util.TreeSet; import java.util.concurrent.CancellationException; import java.util.concurrent.ExecutionException; import java.util.concurrent.Executors; +import java.util.function.Consumer; import java.util.logging.Level; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -286,17 +287,19 @@ public class GroupManager { group.removeFile(fileID); // If we're grouping by category, we don't want to remove empty groups. - if (groupKey.getAttribute() != DrawableAttribute.CATEGORY - && group.getFileIDs().isEmpty()) { - if (analyzedGroups.contains(group)) { - analyzedGroups.remove(group); - sortAnalyzedGroups(); + if (group.getFileIDs().isEmpty()) { + markGroupSeen(group, true); + if (groupKey.getAttribute() != DrawableAttribute.CATEGORY) { + if (analyzedGroups.contains(group)) { + analyzedGroups.remove(group); + sortAnalyzedGroups(); + } + + if (unSeenGroups.contains(group)) { + unSeenGroups.remove(group); + sortUnseenGroups(); + } } - if (unSeenGroups.contains(group)) { - unSeenGroups.remove(group); - sortUnseenGroups(); - } - } return group; } @@ -501,21 +504,26 @@ public class GroupManager { //if there is aleady a group that was previously deemed fully analyzed, then add this newly analyzed file to it. group.addFile(fileID); } + markGroupSeen(group, false); } @Subscribe synchronized public void handleTagDeleted(ContentTagDeletedEvent evt) { GroupKey groupKey = null; final ContentTagDeletedEvent.DeletedContentTagInfo deletedTagInfo = evt.getDeletedTagInfo(); - final TagName tagName = deletedTagInfo.getName(); - if (getGroupBy() == DrawableAttribute.CATEGORY && CategoryManager.isCategoryTagName(tagName)) { - groupKey = new GroupKey<>(DrawableAttribute.CATEGORY, CategoryManager.categoryFromTagName(tagName), getDataSource()); - } else if (getGroupBy() == DrawableAttribute.TAGS && CategoryManager.isNotCategoryTagName(tagName)) { - groupKey = new GroupKey<>(DrawableAttribute.TAGS, tagName, getDataSource()); + final TagName deletedTagName = deletedTagInfo.getName(); + if (getGroupBy() == DrawableAttribute.CATEGORY && CategoryManager.isCategoryTagName(deletedTagName)) { + groupKey = new GroupKey<>(DrawableAttribute.CATEGORY, CategoryManager.categoryFromTagName(deletedTagName), null); + } else if (getGroupBy() == DrawableAttribute.TAGS && CategoryManager.isNotCategoryTagName(deletedTagName)) { + groupKey = new GroupKey<>(DrawableAttribute.TAGS, deletedTagName, null); } if (groupKey != null) { final long fileID = deletedTagInfo.getContentID(); DrawableGroup g = removeFromGroup(groupKey, fileID); + + if (controller.getCategoryManager().getTagName(DhsImageCategory.ZERO).equals(deletedTagName) == false) { + addFileToGroup(null, new GroupKey<>(DrawableAttribute.CATEGORY, DhsImageCategory.ZERO, null), fileID); + } } } From 5b8066ca6d6470192c5a8b652c31ebc5625d1fcd Mon Sep 17 00:00:00 2001 From: millmanorama Date: Wed, 19 Sep 2018 13:22:18 +0200 Subject: [PATCH 13/20] adjustments to TopComponent and Controller lifecycle and regrouping to prevent 'mixed groups' in the UI --- .../ImageGalleryTopComponent.java | 78 ++++++++++--------- .../datamodel/grouping/GroupManager.java | 9 +-- 2 files changed, 44 insertions(+), 43 deletions(-) diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryTopComponent.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryTopComponent.java index 9263913748..055b0cc623 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryTopComponent.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryTopComponent.java @@ -229,45 +229,47 @@ public final class ImageGalleryTopComponent extends TopComponent implements Expl } synchronized private void setController(ImageGalleryController controller) { - if (this.controller != null && notEqual(this.controller, controller)) { - this.controller.reset(); - } - this.controller = controller; - Platform.runLater(new Runnable() { - @Override - public void run() { - //initialize jfx ui - fullUIStack = new StackPane(); //this is passed into controller - myScene = new Scene(fullUIStack); - jfxPanel.setScene(myScene); - groupPane = new GroupPane(controller); - centralStack = new StackPane(groupPane); //this is passed into controller - fullUIStack.getChildren().add(borderPane); - splitPane = new SplitPane(); - borderPane.setCenter(splitPane); - Toolbar toolbar = new Toolbar(controller); - borderPane.setTop(toolbar); - borderPane.setBottom(new StatusBar(controller)); - metaDataTable = new MetaDataPane(controller); - groupTree = new GroupTree(controller); - hashHitList = new HashHitGroupList(controller); - TabPane tabPane = new TabPane(groupTree, hashHitList); - tabPane.setPrefWidth(TabPane.USE_COMPUTED_SIZE); - tabPane.setMinWidth(TabPane.USE_PREF_SIZE); - VBox.setVgrow(tabPane, Priority.ALWAYS); - leftPane = new VBox(tabPane, new SummaryTablePane(controller)); - SplitPane.setResizableWithParent(leftPane, Boolean.FALSE); - SplitPane.setResizableWithParent(groupPane, Boolean.TRUE); - SplitPane.setResizableWithParent(metaDataTable, Boolean.FALSE); - splitPane.getItems().addAll(leftPane, centralStack, metaDataTable); - splitPane.setDividerPositions(0.1, 1.0); - - controller.regroupDisabledProperty().addListener((Observable observable) -> checkForGroups()); - controller.getGroupManager().getAnalyzedGroups().addListener((Observable observable) -> Platform.runLater(() -> checkForGroups())); - - Platform.runLater(() -> checkForGroups()); + if (notEqual(this.controller, controller)) { + if (this.controller != null) { + this.controller.reset(); } - }); + this.controller = controller; + Platform.runLater(new Runnable() { + @Override + public void run() { + //initialize jfx ui + fullUIStack = new StackPane(); //this is passed into controller + myScene = new Scene(fullUIStack); + jfxPanel.setScene(myScene); + groupPane = new GroupPane(controller); + centralStack = new StackPane(groupPane); //this is passed into controller + fullUIStack.getChildren().add(borderPane); + splitPane = new SplitPane(); + borderPane.setCenter(splitPane); + Toolbar toolbar = new Toolbar(controller); + borderPane.setTop(toolbar); + borderPane.setBottom(new StatusBar(controller)); + metaDataTable = new MetaDataPane(controller); + groupTree = new GroupTree(controller); + hashHitList = new HashHitGroupList(controller); + TabPane tabPane = new TabPane(groupTree, hashHitList); + tabPane.setPrefWidth(TabPane.USE_COMPUTED_SIZE); + tabPane.setMinWidth(TabPane.USE_PREF_SIZE); + VBox.setVgrow(tabPane, Priority.ALWAYS); + leftPane = new VBox(tabPane, new SummaryTablePane(controller)); + SplitPane.setResizableWithParent(leftPane, Boolean.FALSE); + SplitPane.setResizableWithParent(groupPane, Boolean.TRUE); + SplitPane.setResizableWithParent(metaDataTable, Boolean.FALSE); + splitPane.getItems().addAll(leftPane, centralStack, metaDataTable); + splitPane.setDividerPositions(0.1, 1.0); + + controller.regroupDisabledProperty().addListener((Observable observable) -> checkForGroups()); + controller.getGroupManager().getAnalyzedGroups().addListener((Observable observable) -> Platform.runLater(() -> checkForGroups())); + + Platform.runLater(() -> checkForGroups()); + } + }); + } } /** diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupManager.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupManager.java index 95d00ee84a..76e27a5719 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupManager.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupManager.java @@ -447,7 +447,8 @@ public class GroupManager { if (!Case.isCaseOpen()) { return; } - + setSortBy(sortBy); + setSortOrder(sortOrder); //only re-query the db if the data source or group by attribute changed or it is forced if (dataSource != getDataSource() || groupBy != getGroupBy() @@ -455,13 +456,11 @@ public class GroupManager { setDataSource(dataSource); setGroupBy(groupBy); - setSortBy(sortBy); - setSortOrder(sortOrder); + Platform.runLater(regrouper::restart); } else { // resort the list of groups - setSortBy(sortBy); - setSortOrder(sortOrder); + sortAnalyzedGroups(); sortUnseenGroups(); } From 4e78c40afa7ea9bb77e87c445777eba7c47a58d4 Mon Sep 17 00:00:00 2001 From: millmanorama Date: Wed, 19 Sep 2018 14:21:28 +0200 Subject: [PATCH 14/20] use the order reversed property in GroupComparators to correct the used comparator. --- .../autopsy/imagegallery/gui/navpanel/GroupComparators.java | 4 ++++ .../autopsy/imagegallery/gui/navpanel/NavPanel.java | 6 ++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/navpanel/GroupComparators.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/navpanel/GroupComparators.java index 16ac00d0a2..52b5ffc525 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/navpanel/GroupComparators.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/navpanel/GroupComparators.java @@ -56,6 +56,10 @@ final class GroupComparators> implements Comparator extractor; private final Function valueFormatter; private final boolean orderReveresed; + + boolean isOrderReveresed() { + return orderReveresed; + } private final String displayName; private GroupComparators(String displayName, Function extractor, Function formatter, boolean defaultOrderReversed) { diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/navpanel/NavPanel.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/navpanel/NavPanel.java index e8c97c1379..46131cb4e5 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/navpanel/NavPanel.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/navpanel/NavPanel.java @@ -123,10 +123,12 @@ abstract class NavPanel extends Tab { */ @ThreadConfined(type = ThreadConfined.ThreadType.JFX) Comparator getComparator() { - Comparator comparator = sortChooser.getComparator(); - return (sortChooser.getSortOrder() == SortOrder.ASCENDING) + GroupComparators comparator = sortChooser.getComparator(); + Comparator comparator2 = (sortChooser.getSortOrder() == SortOrder.ASCENDING) ? comparator : comparator.reversed(); + + return comparator.isOrderReveresed() ? comparator2.reversed() : comparator2; } /** From ed8592c03d9dec4bbee1aa3b706a629816e9b976 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Wed, 19 Sep 2018 12:26:20 -0400 Subject: [PATCH 15/20] Modify tests to use appropriate method signature for AbstractCommonAttributeSearcher --- .../CommonAttributeSearchInterCaseTests.java | 11 ++++------- .../IngestedWithHashAndFileTypeInterCaseTests.java | 11 +++-------- 2 files changed, 7 insertions(+), 15 deletions(-) diff --git a/Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/CommonAttributeSearchInterCaseTests.java b/Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/CommonAttributeSearchInterCaseTests.java index 966d0b8bce..e6ff7b743d 100644 --- a/Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/CommonAttributeSearchInterCaseTests.java +++ b/Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/CommonAttributeSearchInterCaseTests.java @@ -21,7 +21,6 @@ package org.sleuthkit.autopsy.commonfilessearch; import java.nio.file.Path; import java.sql.SQLException; -import java.util.Map; import junit.framework.Assert; import junit.framework.Test; import org.netbeans.junit.NbModuleSuite; @@ -111,9 +110,8 @@ public class CommonAttributeSearchInterCaseTests extends NbTestCase { private void assertResultsAreOfType(CorrelationAttributeInstance.Type type) { try { - Map dataSources = this.utils.getDataSourceMap(); - AbstractCommonAttributeSearcher builder = new AllInterCaseCommonAttributeSearcher(dataSources, false, false, type, 0); + AbstractCommonAttributeSearcher builder = new AllInterCaseCommonAttributeSearcher(false, false, type, 0); CommonAttributeSearchResults metadata = builder.findMatches(); @@ -146,22 +144,21 @@ public class CommonAttributeSearchInterCaseTests extends NbTestCase { */ public void testTwo() { try { - Map dataSources = this.utils.getDataSourceMap(); AbstractCommonAttributeSearcher builder; CommonAttributeSearchResults metadata; - builder = new AllInterCaseCommonAttributeSearcher(dataSources, false, false, this.utils.USB_ID_TYPE, 100); + builder = new AllInterCaseCommonAttributeSearcher(false, false, this.utils.USB_ID_TYPE, 100); metadata = builder.findMatches(); metadata.size(); //assertTrue("This should yield 13 results.", verifyInstanceCount(metadata, 13)); - builder = new AllInterCaseCommonAttributeSearcher(dataSources, false, false, this.utils.USB_ID_TYPE, 20); + builder = new AllInterCaseCommonAttributeSearcher(false, false, this.utils.USB_ID_TYPE, 20); metadata = builder.findMatches(); metadata.size(); //assertTrue("This should yield no results.", verifyInstanceCount(metadata, 0)); - builder = new AllInterCaseCommonAttributeSearcher(dataSources, false, false, this.utils.USB_ID_TYPE, 90); + builder = new AllInterCaseCommonAttributeSearcher(false, false, this.utils.USB_ID_TYPE, 90); metadata = builder.findMatches(); metadata.size(); //assertTrue("This should yield 2 results.", verifyInstanceCount(metadata, 2)); diff --git a/Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/IngestedWithHashAndFileTypeInterCaseTests.java b/Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/IngestedWithHashAndFileTypeInterCaseTests.java index 8f4f96e7a3..9c32eb0f94 100644 --- a/Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/IngestedWithHashAndFileTypeInterCaseTests.java +++ b/Core/test/qa-functional/src/org/sleuthkit/autopsy/commonfilessearch/IngestedWithHashAndFileTypeInterCaseTests.java @@ -21,7 +21,6 @@ package org.sleuthkit.autopsy.commonfilessearch; import java.nio.file.Path; import java.sql.SQLException; -import java.util.Map; import junit.framework.Test; import org.netbeans.junit.NbModuleSuite; import org.netbeans.junit.NbTestCase; @@ -96,10 +95,8 @@ public class IngestedWithHashAndFileTypeInterCaseTests extends NbTestCase { */ public void testOne() { try { - Map dataSources = this.utils.getDataSourceMap(); - //note that the params false and false are presently meaningless because that feature is not supported yet - AbstractCommonAttributeSearcher builder = new AllInterCaseCommonAttributeSearcher(dataSources, false, false, this.utils.FILE_TYPE, 0); + AbstractCommonAttributeSearcher builder = new AllInterCaseCommonAttributeSearcher(false, false, this.utils.FILE_TYPE, 0); CommonAttributeSearchResults metadata = builder.findMatches(); assertTrue("Results should not be empty", metadata.size() != 0); @@ -146,11 +143,10 @@ public class IngestedWithHashAndFileTypeInterCaseTests extends NbTestCase { */ public void testTwo() { try { - Map dataSources = this.utils.getDataSourceMap(); int matchesMustAlsoBeFoundInThisCase = this.utils.getCaseMap().get(CASE2); CorrelationAttributeInstance.Type fileType = CorrelationAttributeInstance.getDefaultCorrelationTypes().get(0); - AbstractCommonAttributeSearcher builder = new SingleInterCaseCommonAttributeSearcher(matchesMustAlsoBeFoundInThisCase, dataSources, false, false, fileType, 0); + AbstractCommonAttributeSearcher builder = new SingleInterCaseCommonAttributeSearcher(matchesMustAlsoBeFoundInThisCase, false, false, fileType, 0); CommonAttributeSearchResults metadata = builder.findMatches(); @@ -199,11 +195,10 @@ public class IngestedWithHashAndFileTypeInterCaseTests extends NbTestCase { */ public void testThree(){ try { - Map dataSources = this.utils.getDataSourceMap(); //note that the params false and false are presently meaningless because that feature is not supported yet CorrelationAttributeInstance.Type fileType = CorrelationAttributeInstance.getDefaultCorrelationTypes().get(0); - AbstractCommonAttributeSearcher builder = new AllInterCaseCommonAttributeSearcher(dataSources, false, false, fileType, 50); + AbstractCommonAttributeSearcher builder = new AllInterCaseCommonAttributeSearcher(false, false, fileType, 50); CommonAttributeSearchResults metadata = builder.findMatches(); From 831126e61cdce3cf12d5302d5a7c500f6224406d Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Wed, 19 Sep 2018 13:03:49 -0400 Subject: [PATCH 16/20] Fixed switch case to avoid null values. --- .../autopsy/casemodule/CasePreferences.java | 26 ++++++++++++------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/CasePreferences.java b/Core/src/org/sleuthkit/autopsy/casemodule/CasePreferences.java index 1c853c7951..dcee43fc82 100755 --- a/Core/src/org/sleuthkit/autopsy/casemodule/CasePreferences.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/CasePreferences.java @@ -98,16 +98,22 @@ public final class CasePreferences { Properties props = new Properties(); props.load(inputStream); String groupByDataSourceValue = props.getProperty(KEY_GROUP_BY_DATA_SOURCE); - switch (groupByDataSourceValue) { - case VALUE_TRUE: - groupItemsInTreeByDataSource = true; - break; - case VALUE_FALSE: - groupItemsInTreeByDataSource = false; - break; - default: - groupItemsInTreeByDataSource = null; - break; + if (groupByDataSourceValue != null) { + switch (groupByDataSourceValue) { + case VALUE_TRUE: + groupItemsInTreeByDataSource = true; + break; + case VALUE_FALSE: + groupItemsInTreeByDataSource = false; + break; + default: + logger.log(Level.WARNING, String.format("Unexpected value '%s' for key '%s'. Using 'null' instead.", + groupByDataSourceValue, KEY_GROUP_BY_DATA_SOURCE)); + groupItemsInTreeByDataSource = null; + break; + } + } else { + groupItemsInTreeByDataSource = null; } } catch (IOException ex) { logger.log(Level.SEVERE, "Error reading settings file", ex); From 25fc3f5ace2390557b0f3d3cca78af89eefedad8 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Wed, 19 Sep 2018 17:09:57 -0400 Subject: [PATCH 17/20] 4163 Handle new cases, data sources and other null values that can not go in cache --- .../datamodel/AbstractSqlEamDb.java | 52 ++++++++++++++----- 1 file changed, 38 insertions(+), 14 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java index cd2a11b031..6f54f79812 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java @@ -20,6 +20,7 @@ package org.sleuthkit.autopsy.centralrepository.datamodel; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; import java.net.UnknownHostException; import java.util.ArrayList; import java.util.List; @@ -402,6 +403,10 @@ abstract class AbstractSqlEamDb implements EamDb { public CorrelationCase getCaseByUUID(String caseUUID) throws EamDbException { try { return caseCacheByUUID.get(caseUUID, () -> getCaseByUUIDFromCr(caseUUID)); + } catch (CacheLoader.InvalidCacheLoadException ex) { + //cache can not store null values + logger.log(Level.INFO, "Unable to get current autopsy case from Central repo returning null as current case", ex); + return null; } catch (ExecutionException ex) { throw new EamDbException("Error getting autopsy case from Central repo", ex); } @@ -460,6 +465,10 @@ abstract class AbstractSqlEamDb implements EamDb { public CorrelationCase getCaseById(int caseId) throws EamDbException { try { return caseCacheById.get(caseId, () -> getCaseByIdFromCr(caseId)); + } catch (CacheLoader.InvalidCacheLoadException ex) { + //cache can not store null values + logger.log(Level.INFO, "Unable to get current autopsy case from Central repo returning null as current case", ex); + return null; } catch (ExecutionException ex) { throw new EamDbException("Error getting autopsy case from Central repo", ex); } @@ -544,9 +553,10 @@ abstract class AbstractSqlEamDb implements EamDb { } /** - * Create a key to the DataSourceCacheByDeviceId + * Create a key to the DataSourceCacheByDeviceId * - * @param caseId - the id of the CorrelationCase in the Central Repository + * @param caseId - the id of the CorrelationCase in the Central + * Repository * @param dataSourceDeviceId - the device Id of the data source * * @return a String to be used as a key for the dataSourceCacheByDeviceId @@ -558,8 +568,10 @@ abstract class AbstractSqlEamDb implements EamDb { /** * Create a key to the DataSourceCacheById * - * @param caseId - the id of the CorrelationCase in the Central Repository + * @param caseId - the id of the CorrelationCase in the Central + * Repository * @param dataSourceId - the id of the datasource in the central repository + * * @return a String to be used as a key for the dataSourceCacheById */ private static String getDataSourceByIdCacheKey(int caseId, int dataSourceId) { @@ -630,6 +642,10 @@ abstract class AbstractSqlEamDb implements EamDb { } try { return dataSourceCacheByDeviceId.get(getDataSourceByDeviceIdCacheKey(correlationCase.getID(), dataSourceDeviceId), () -> getDataSourceFromCr(correlationCase, dataSourceDeviceId)); + } catch (CacheLoader.InvalidCacheLoadException ex) { + //cache can not store null values + logger.log(Level.INFO, "Unable to current get data source from Central repo returning null as current data source", ex); + return null; } catch (ExecutionException ex) { throw new EamDbException("Error getting data source from central repository", ex); } @@ -694,6 +710,10 @@ abstract class AbstractSqlEamDb implements EamDb { } try { return dataSourceCacheById.get(getDataSourceByIdCacheKey(correlationCase.getID(), dataSourceId), () -> getDataSourceByIdFromCr(correlationCase, dataSourceId)); + } catch (CacheLoader.InvalidCacheLoadException ex) { + //cache can not store null values + logger.log(Level.INFO, "Unable to current get data source from Central repo returning null as current data source", ex); + return null; } catch (ExecutionException ex) { throw new EamDbException("Error getting data source from central repository", ex); } @@ -868,7 +888,7 @@ abstract class AbstractSqlEamDb implements EamDb { public List getArtifactInstancesByTypeValue(CorrelationAttributeInstance.Type aType, String value) throws EamDbException, CorrelationAttributeNormalizationException { String normalizedValue = CorrelationAttributeNormalizer.normalize(aType, value); - + Connection conn = connect(); List artifactInstances = new ArrayList<>(); @@ -1440,7 +1460,7 @@ abstract class AbstractSqlEamDb implements EamDb { @Override public CorrelationAttributeInstance getCorrelationAttributeInstance(CorrelationAttributeInstance.Type type, CorrelationCase correlationCase, CorrelationDataSource correlationDataSource, String value, String filePath) throws EamDbException, CorrelationAttributeNormalizationException { - + if (correlationCase == null) { throw new EamDbException("Correlation case is null"); } @@ -1459,7 +1479,7 @@ abstract class AbstractSqlEamDb implements EamDb { try { String normalizedValue = CorrelationAttributeNormalizer.normalize(type, value); - + String tableName = EamDbUtil.correlationTypeToInstanceTableName(type); String sql = "SELECT id, known_status, comment FROM " @@ -1701,7 +1721,7 @@ abstract class AbstractSqlEamDb implements EamDb { artifactInstances.add(artifactInstance); } catch (CorrelationAttributeNormalizationException ex) { logger.log(Level.INFO, "Unable to get artifact instance from resultset.", ex); - } + } } } catch (SQLException ex) { throw new EamDbException("Error getting notable artifact instances.", ex); // NON-NLS @@ -1724,7 +1744,7 @@ abstract class AbstractSqlEamDb implements EamDb { */ @Override public Long getCountArtifactInstancesKnownBad(CorrelationAttributeInstance.Type aType, String value) throws EamDbException, CorrelationAttributeNormalizationException { - + String normalizedValue = CorrelationAttributeNormalizer.normalize(aType, value); Connection conn = connect(); @@ -1771,7 +1791,7 @@ abstract class AbstractSqlEamDb implements EamDb { */ @Override public List getListCasesHavingArtifactInstancesKnownBad(CorrelationAttributeInstance.Type aType, String value) throws EamDbException, CorrelationAttributeNormalizationException { - + String normalizedValue = CorrelationAttributeNormalizer.normalize(aType, value); Connection conn = connect(); @@ -1931,7 +1951,7 @@ abstract class AbstractSqlEamDb implements EamDb { public boolean isValueInReferenceSet(String value, int referenceSetID, int correlationTypeID) throws EamDbException, CorrelationAttributeNormalizationException { String normalizeValued = CorrelationAttributeNormalizer.normalize(this.getCorrelationTypeById(correlationTypeID), value); - + Connection conn = connect(); Long matchingInstances = 0L; @@ -1969,10 +1989,10 @@ abstract class AbstractSqlEamDb implements EamDb { */ @Override public boolean isArtifactKnownBadByReference(CorrelationAttributeInstance.Type aType, String value) throws EamDbException, CorrelationAttributeNormalizationException { - + //this should be done here so that we can be certain that aType and value are valid before we proceed String normalizeValued = CorrelationAttributeNormalizer.normalize(aType, value); - + // TEMP: Only support file correlation type if (aType.getId() != CorrelationAttributeInstance.FILES_TYPE_ID) { return false; @@ -1985,7 +2005,7 @@ abstract class AbstractSqlEamDb implements EamDb { ResultSet resultSet = null; String sql = "SELECT count(*) FROM %s WHERE value=? AND known_status=?"; - try { + try { preparedStatement = conn.prepareStatement(String.format(sql, EamDbUtil.correlationTypeToReferenceTableName(aType))); preparedStatement.setString(1, normalizeValued); preparedStatement.setByte(2, TskData.FileKnown.BAD.getFileKnownValue()); @@ -2595,7 +2615,7 @@ abstract class AbstractSqlEamDb implements EamDb { EamDbUtil.closeResultSet(resultSet); EamDbUtil.closeConnection(conn); } - + return globalFileInstances; } @@ -2872,6 +2892,10 @@ abstract class AbstractSqlEamDb implements EamDb { public CorrelationAttributeInstance.Type getCorrelationTypeById(int typeId) throws EamDbException { try { return typeCache.get(CorrelationAttributeInstance.FILES_TYPE_ID, () -> getCorrelationTypeByIdFromCr(typeId)); + } catch (CacheLoader.InvalidCacheLoadException ex) { + //cache can not store null values + logger.log(Level.INFO, "Unable to get correlation type from Central repo returning null as the correlation type", ex); + return null; } catch (ExecutionException ex) { throw new EamDbException("Error getting correlation type", ex); } From 4a4be1dc9f70e5355e1e6ada913ba512f1399192 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Wed, 19 Sep 2018 17:27:30 -0400 Subject: [PATCH 18/20] Change logging level so messages will not be logged with every case creation --- .../centralrepository/datamodel/AbstractSqlEamDb.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java index 6f54f79812..5719f6f44d 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java @@ -405,7 +405,7 @@ abstract class AbstractSqlEamDb implements EamDb { return caseCacheByUUID.get(caseUUID, () -> getCaseByUUIDFromCr(caseUUID)); } catch (CacheLoader.InvalidCacheLoadException ex) { //cache can not store null values - logger.log(Level.INFO, "Unable to get current autopsy case from Central repo returning null as current case", ex); + logger.log(Level.FINE, "Unable to get current autopsy case from Central repo returning null as current case", ex); return null; } catch (ExecutionException ex) { throw new EamDbException("Error getting autopsy case from Central repo", ex); @@ -467,7 +467,7 @@ abstract class AbstractSqlEamDb implements EamDb { return caseCacheById.get(caseId, () -> getCaseByIdFromCr(caseId)); } catch (CacheLoader.InvalidCacheLoadException ex) { //cache can not store null values - logger.log(Level.INFO, "Unable to get current autopsy case from Central repo returning null as current case", ex); + logger.log(Level.FINE, "Unable to get current autopsy case from Central repo returning null as current case", ex); return null; } catch (ExecutionException ex) { throw new EamDbException("Error getting autopsy case from Central repo", ex); @@ -644,7 +644,7 @@ abstract class AbstractSqlEamDb implements EamDb { return dataSourceCacheByDeviceId.get(getDataSourceByDeviceIdCacheKey(correlationCase.getID(), dataSourceDeviceId), () -> getDataSourceFromCr(correlationCase, dataSourceDeviceId)); } catch (CacheLoader.InvalidCacheLoadException ex) { //cache can not store null values - logger.log(Level.INFO, "Unable to current get data source from Central repo returning null as current data source", ex); + logger.log(Level.FINE, "Unable to current get data source from Central repo returning null as current data source", ex); return null; } catch (ExecutionException ex) { throw new EamDbException("Error getting data source from central repository", ex); @@ -712,7 +712,7 @@ abstract class AbstractSqlEamDb implements EamDb { return dataSourceCacheById.get(getDataSourceByIdCacheKey(correlationCase.getID(), dataSourceId), () -> getDataSourceByIdFromCr(correlationCase, dataSourceId)); } catch (CacheLoader.InvalidCacheLoadException ex) { //cache can not store null values - logger.log(Level.INFO, "Unable to current get data source from Central repo returning null as current data source", ex); + logger.log(Level.FINE, "Unable to current get data source from Central repo returning null as current data source", ex); return null; } catch (ExecutionException ex) { throw new EamDbException("Error getting data source from central repository", ex); @@ -2894,7 +2894,7 @@ abstract class AbstractSqlEamDb implements EamDb { return typeCache.get(CorrelationAttributeInstance.FILES_TYPE_ID, () -> getCorrelationTypeByIdFromCr(typeId)); } catch (CacheLoader.InvalidCacheLoadException ex) { //cache can not store null values - logger.log(Level.INFO, "Unable to get correlation type from Central repo returning null as the correlation type", ex); + logger.log(Level.FINE, "Unable to get correlation type from Central repo returning null as the correlation type", ex); return null; } catch (ExecutionException ex) { throw new EamDbException("Error getting correlation type", ex); From 8b53713dd2a2c993706a2c5097218268d46a8373 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Wed, 19 Sep 2018 17:39:36 -0400 Subject: [PATCH 19/20] remove logging for expected cache loading exceptions --- .../datamodel/AbstractSqlEamDb.java | 25 ++++++++----------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java index 5719f6f44d..2aae94cb24 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java @@ -403,9 +403,8 @@ abstract class AbstractSqlEamDb implements EamDb { public CorrelationCase getCaseByUUID(String caseUUID) throws EamDbException { try { return caseCacheByUUID.get(caseUUID, () -> getCaseByUUIDFromCr(caseUUID)); - } catch (CacheLoader.InvalidCacheLoadException ex) { - //cache can not store null values - logger.log(Level.FINE, "Unable to get current autopsy case from Central repo returning null as current case", ex); + } catch (CacheLoader.InvalidCacheLoadException ignored) { + //lambda valueloader returned a null value and cache can not store null values this is normal if the case does not exist in the central repo yet return null; } catch (ExecutionException ex) { throw new EamDbException("Error getting autopsy case from Central repo", ex); @@ -465,9 +464,8 @@ abstract class AbstractSqlEamDb implements EamDb { public CorrelationCase getCaseById(int caseId) throws EamDbException { try { return caseCacheById.get(caseId, () -> getCaseByIdFromCr(caseId)); - } catch (CacheLoader.InvalidCacheLoadException ex) { - //cache can not store null values - logger.log(Level.FINE, "Unable to get current autopsy case from Central repo returning null as current case", ex); + } catch (CacheLoader.InvalidCacheLoadException ignored) { + //lambda valueloader returned a null value and cache can not store null values this is normal if the case does not exist in the central repo yet return null; } catch (ExecutionException ex) { throw new EamDbException("Error getting autopsy case from Central repo", ex); @@ -642,9 +640,8 @@ abstract class AbstractSqlEamDb implements EamDb { } try { return dataSourceCacheByDeviceId.get(getDataSourceByDeviceIdCacheKey(correlationCase.getID(), dataSourceDeviceId), () -> getDataSourceFromCr(correlationCase, dataSourceDeviceId)); - } catch (CacheLoader.InvalidCacheLoadException ex) { - //cache can not store null values - logger.log(Level.FINE, "Unable to current get data source from Central repo returning null as current data source", ex); + } catch (CacheLoader.InvalidCacheLoadException ignored) { + //lambda valueloader returned a null value and cache can not store null values this is normal if the dataSource does not exist in the central repo yet return null; } catch (ExecutionException ex) { throw new EamDbException("Error getting data source from central repository", ex); @@ -710,9 +707,8 @@ abstract class AbstractSqlEamDb implements EamDb { } try { return dataSourceCacheById.get(getDataSourceByIdCacheKey(correlationCase.getID(), dataSourceId), () -> getDataSourceByIdFromCr(correlationCase, dataSourceId)); - } catch (CacheLoader.InvalidCacheLoadException ex) { - //cache can not store null values - logger.log(Level.FINE, "Unable to current get data source from Central repo returning null as current data source", ex); + } catch (CacheLoader.InvalidCacheLoadException ignored) { + //lambda valueloader returned a null value and cache can not store null values this is normal if the dataSource does not exist in the central repo yet return null; } catch (ExecutionException ex) { throw new EamDbException("Error getting data source from central repository", ex); @@ -2892,9 +2888,8 @@ abstract class AbstractSqlEamDb implements EamDb { public CorrelationAttributeInstance.Type getCorrelationTypeById(int typeId) throws EamDbException { try { return typeCache.get(CorrelationAttributeInstance.FILES_TYPE_ID, () -> getCorrelationTypeByIdFromCr(typeId)); - } catch (CacheLoader.InvalidCacheLoadException ex) { - //cache can not store null values - logger.log(Level.FINE, "Unable to get correlation type from Central repo returning null as the correlation type", ex); + } catch (CacheLoader.InvalidCacheLoadException ignored) { + //lambda valueloader returned a null value and cache can not store null values this is normal if the correlation type does not exist in the central repo yet return null; } catch (ExecutionException ex) { throw new EamDbException("Error getting correlation type", ex); From 626712c7b0bdf9e1c6bbd4b22592a2a318d47b2e Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Wed, 19 Sep 2018 17:58:01 -0400 Subject: [PATCH 20/20] 4235 add back code to inherit highlighting to icon columns --- .../autopsy/corecomponents/DataResultViewerTable.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Core/src/org/sleuthkit/autopsy/corecomponents/DataResultViewerTable.java b/Core/src/org/sleuthkit/autopsy/corecomponents/DataResultViewerTable.java index 4b93eba8cc..25709df0df 100644 --- a/Core/src/org/sleuthkit/autopsy/corecomponents/DataResultViewerTable.java +++ b/Core/src/org/sleuthkit/autopsy/corecomponents/DataResultViewerTable.java @@ -854,6 +854,8 @@ public class DataResultViewerTable extends AbstractDataResultViewer { "DataResultViewerTable.commentRenderer.noComment.toolTip=No comments found"}) @Override public Component getTableCellRendererComponent(JTable table, Object value, boolean isSelected, boolean hasFocus, int row, int column) { + Component component = super.getTableCellRendererComponent(table, value, isSelected, hasFocus, row, column); + setBackground(component.getBackground()); //inherit highlighting for selection setHorizontalAlignment(CENTER); Object switchValue = null; if ((value instanceof NodeProperty)) { @@ -908,6 +910,8 @@ public class DataResultViewerTable extends AbstractDataResultViewer { @Override public Component getTableCellRendererComponent(JTable table, Object value, boolean isSelected, boolean hasFocus, int row, int column) { + Component component = super.getTableCellRendererComponent(table, value, isSelected, hasFocus, row, column); + setBackground(component.getBackground()); //inherit highlighting for selection setHorizontalAlignment(CENTER); Object switchValue = null; if ((value instanceof NodeProperty)) { @@ -955,6 +959,8 @@ public class DataResultViewerTable extends AbstractDataResultViewer { @Override public Component getTableCellRendererComponent(JTable table, Object value, boolean isSelected, boolean hasFocus, int row, int column) { + Component component = super.getTableCellRendererComponent(table, value, isSelected, hasFocus, row, column); + setBackground(component.getBackground()); //inherit highlighting for selection setHorizontalAlignment(LEFT); Object countValue = null; if ((value instanceof NodeProperty)) {