diff --git a/Core/src/org/sleuthkit/autopsy/core/UserPreferences.java b/Core/src/org/sleuthkit/autopsy/core/UserPreferences.java index 8f3c2e8e14..0d3fd0198a 100644 --- a/Core/src/org/sleuthkit/autopsy/core/UserPreferences.java +++ b/Core/src/org/sleuthkit/autopsy/core/UserPreferences.java @@ -84,6 +84,7 @@ public final class UserPreferences { private static final boolean DISPLAY_TRANSLATED_NAMES_DEFAULT = true; public static final String EXTERNAL_HEX_EDITOR_PATH = "ExternalHexEditorPath"; public static final String SOLR_MAX_JVM_SIZE = "SolrMaxJVMSize"; + private static final int DEFAULT_SOLR_HEAP_SIZE_MB = 2048; public static final String RESULTS_TABLE_PAGE_SIZE = "ResultsTablePageSize"; private static final String GEO_TILE_OPTION = "GeolocationTileOption"; private static final String GEO_OSM_TILE_ZIP_PATH = "GeolocationOsmZipPath"; @@ -535,10 +536,10 @@ public final class UserPreferences { /** * Get the maximum JVM heap size (in MB) for the embedded Solr server. * - * @return Saved value or default (512) + * @return Saved value or default (2 GB) */ public static int getMaxSolrVMSize() { - return preferences.getInt(SOLR_MAX_JVM_SIZE, 512); + return preferences.getInt(SOLR_MAX_JVM_SIZE, DEFAULT_SOLR_HEAP_SIZE_MB); } /** diff --git a/Core/src/org/sleuthkit/autopsy/corecomponents/AutopsyOptionsPanel.java b/Core/src/org/sleuthkit/autopsy/corecomponents/AutopsyOptionsPanel.java index 7f4f8bf3a7..e7c0f9eaeb 100644 --- a/Core/src/org/sleuthkit/autopsy/corecomponents/AutopsyOptionsPanel.java +++ b/Core/src/org/sleuthkit/autopsy/corecomponents/AutopsyOptionsPanel.java @@ -82,6 +82,7 @@ final class AutopsyOptionsPanel extends javax.swing.JPanel { private static final String CONFIG_FILE_EXTENSION = ".conf"; private static final long ONE_BILLION = 1000000000L; //used to roughly convert system memory from bytes to gigabytes private static final int MEGA_IN_GIGA = 1024; //used to convert memory settings saved as megabytes to gigabytes + private static final int DEFAULT_SOLR_HEAP_SIZE_MB = 2048; private static final int MIN_MEMORY_IN_GB = 2; //the enforced minimum memory in gigabytes private static final Logger logger = Logger.getLogger(AutopsyOptionsPanel.class.getName()); private String initialMemValue = Long.toString(Runtime.getRuntime().maxMemory() / ONE_BILLION); @@ -113,7 +114,7 @@ final class AutopsyOptionsPanel extends javax.swing.JPanel { // The cast to int in the following is to ensure that the correct SpinnerNumberModel // constructor is called. solrMaxHeapSpinner.setModel(new javax.swing.SpinnerNumberModel(UserPreferences.getMaxSolrVMSize(), - 512, ((int) getSystemMemoryInGB()) * MEGA_IN_GIGA, 512)); + DEFAULT_SOLR_HEAP_SIZE_MB, ((int) getSystemMemoryInGB()) * MEGA_IN_GIGA, DEFAULT_SOLR_HEAP_SIZE_MB)); textFieldListener = new TextFieldListener(); agencyLogoPathField.getDocument().addDocumentListener(textFieldListener); diff --git a/Core/src/org/sleuthkit/autopsy/discovery/search/DiscoveryAttributes.java b/Core/src/org/sleuthkit/autopsy/discovery/search/DiscoveryAttributes.java index 15171cf945..77f67899eb 100644 --- a/Core/src/org/sleuthkit/autopsy/discovery/search/DiscoveryAttributes.java +++ b/Core/src/org/sleuthkit/autopsy/discovery/search/DiscoveryAttributes.java @@ -140,9 +140,10 @@ public class DiscoveryAttributes { return new DiscoveryKeyUtils.FileTypeGroupKey(file); } } - + /** - * Attribute for grouping/sorting by domain category (TSK_WEB_CATEGORY artifacts). + * Attribute for grouping/sorting by domain category (TSK_WEB_CATEGORY + * artifacts). */ static class DomainCategoryAttribute extends AttributeType { @@ -150,7 +151,7 @@ public class DiscoveryAttributes { public DiscoveryKeyUtils.GroupKey getGroupKey(Result result) { return new DiscoveryKeyUtils.DomainCategoryGroupKey(result); } - + @Override public void addAttributeToResults(List results, SleuthkitCase caseDb, CentralRepository centralRepoDb) throws DiscoveryException { @@ -167,10 +168,11 @@ public class DiscoveryAttributes { throw new DiscoveryException("Error fetching TSK_WEB_CATEGORY artifacts from the database", ex); } } - + /** - * Loads all TSK_WEB_CATEGORY artifacts and maps the domain attribute to the category name attribute. - * Each ResultDomain is then parsed and matched against this map of values. + * Loads all TSK_WEB_CATEGORY artifacts and maps the domain attribute to + * the category name attribute. Each ResultDomain is then parsed and + * matched against this map of values. */ private Map getDomainsWithWebCategories(SleuthkitCase caseDb) throws TskCoreException, InterruptedException { Map domainToCategory = new HashMap<>(); @@ -190,7 +192,7 @@ public class DiscoveryAttributes { } } - return domainToCategory; + return domainToCategory; } } @@ -269,36 +271,36 @@ public class DiscoveryAttributes { } } } - + /** - * Organizes the domain instances by normalized domain value. - * This helps reduce the complexity of updating ResultDomain instances - * after the query has been executed. - * - * Example: query for notable status of google.com. Result: notable - * With this map, all domain instances that represent google.com can - * be updated after one simple lookup. + * Organizes the domain instances by normalized domain value. This helps + * reduce the complexity of updating ResultDomain instances after the query + * has been executed. + * + * Example: query for notable status of google.com. Result: notable With + * this map, all domain instances that represent google.com can be updated + * after one simple lookup. */ private static Map> organizeByValue(List domainsBatch, CorrelationAttributeInstance.Type attributeType) { - final Map> resultDomainTable = new HashMap<>(); - for (ResultDomain domainInstance : domainsBatch) { - try { - final String domainValue = domainInstance.getDomain(); - final String normalizedDomain = CorrelationAttributeNormalizer.normalize(attributeType, domainValue); - final List bucket = resultDomainTable.getOrDefault(normalizedDomain, new ArrayList<>()); - bucket.add(domainInstance); - resultDomainTable.put(normalizedDomain, bucket); - } catch (CorrelationAttributeNormalizationException ex) { - logger.log(Level.INFO, String.format("Domain [%s] failed normalization, skipping...", domainInstance.getDomain())); - } + final Map> resultDomainTable = new HashMap<>(); + for (ResultDomain domainInstance : domainsBatch) { + try { + final String domainValue = domainInstance.getDomain(); + final String normalizedDomain = CorrelationAttributeNormalizer.normalize(attributeType, domainValue); + final List bucket = resultDomainTable.getOrDefault(normalizedDomain, new ArrayList<>()); + bucket.add(domainInstance); + resultDomainTable.put(normalizedDomain, bucket); + } catch (CorrelationAttributeNormalizationException ex) { + logger.log(Level.INFO, String.format("Domain [%s] failed normalization, skipping...", domainInstance.getDomain())); } - return resultDomainTable; + } + return resultDomainTable; } /** - * Helper function to create a string of comma separated values. - * Each value is wrapped in `'`. This method is used to bundle up - * a collection of values for use in a SQL WHERE IN (...) clause. + * Helper function to create a string of comma separated values. Each value + * is wrapped in `'`. This method is used to bundle up a collection of + * values for use in a SQL WHERE IN (...) clause. */ private static String createCSV(Set values) { StringJoiner joiner = new StringJoiner(", "); @@ -307,30 +309,30 @@ public class DiscoveryAttributes { } return joiner.toString(); } - + /** * Attribute for grouping/sorting by notability in the CR. */ static class PreviouslyNotableAttribute extends AttributeType { - + static final int DOMAIN_BATCH_SIZE = 500; // Number of domains to look up at one time @Override public DiscoveryKeyUtils.GroupKey getGroupKey(Result result) { return new DiscoveryKeyUtils.PreviouslyNotableGroupKey(result); } - + @Override public void addAttributeToResults(List results, SleuthkitCase caseDb, CentralRepository centralRepoDb) throws DiscoveryException { - + if (centralRepoDb != null) { processFilesWithCr(results, centralRepoDb); - } + } } - + private void processFilesWithCr(List results, CentralRepository centralRepo) throws DiscoveryException { - + List domainsBatch = new ArrayList<>(); for (Result result : results) { if (result.getType() == SearchData.Type.DOMAIN) { @@ -341,15 +343,15 @@ public class DiscoveryAttributes { } } } - + queryPreviouslyNotable(domainsBatch, centralRepo); } - + private void queryPreviouslyNotable(List domainsBatch, CentralRepository centralRepo) throws DiscoveryException { if (domainsBatch.isEmpty()) { return; } - + try { final CorrelationAttributeInstance.Type attributeType = centralRepo.getCorrelationTypeById(CorrelationAttributeInstance.DOMAIN_TYPE_ID); final Map> resultDomainTable = organizeByValue(domainsBatch, attributeType); @@ -371,16 +373,16 @@ public class DiscoveryAttributes { throw new DiscoveryException("Fatal exception encountered querying the CR.", ex); } } - + private static class DomainPreviouslyNotableCallback implements InstanceTableCallback { - + private final Map> domainLookup; private SQLException sqlCause; private DomainPreviouslyNotableCallback(Map> domainLookup) { this.domainLookup = domainLookup; } - + @Override public void process(ResultSet resultSet) { try { @@ -401,7 +403,7 @@ public class DiscoveryAttributes { */ SQLException getCause() { return this.sqlCause; - } + } } } @@ -499,12 +501,13 @@ public class DiscoveryAttributes { final CorrelationAttributeInstance.Type attributeType = centralRepository.getCorrelationTypeById(CorrelationAttributeInstance.DOMAIN_TYPE_ID); final Map> resultDomainTable = organizeByValue(domainsToQuery, attributeType); final String values = createCSV(resultDomainTable.keySet()); - final String tableName = CentralRepoDbUtil.correlationTypeToInstanceTableName(attributeType); - final String domainFrequencyQuery = " value AS domain_name, COUNT(*) AS frequency " - + "FROM " + tableName + " " - + "WHERE value IN (" + values + ") " - + "GROUP BY value"; + final String domainFrequencyQuery = " value AS domain_name, COUNT(value) AS frequency FROM" + + "(SELECT DISTINCT case_id, value FROM " + + tableName + + " WHERE value IN (" + + values + + ")) AS foo GROUP BY value"; final DomainFrequencyCallback frequencyCallback = new DomainFrequencyCallback(resultDomainTable); centralRepository.processSelectClause(domainFrequencyQuery, frequencyCallback); @@ -784,8 +787,8 @@ public class DiscoveryAttributes { } /** - * Attribute for grouping/sorting domains by number of page views. - * Page views is defined at the number of TSK_WEB_HISTORY artifacts. + * Attribute for grouping/sorting domains by number of page views. Page + * views is defined at the number of TSK_WEB_HISTORY artifacts. */ static class PageViewsAttribute extends AttributeType { @@ -1074,4 +1077,4 @@ public class DiscoveryAttributes { private DiscoveryAttributes() { // Class should not be instantiated } - } +} diff --git a/Core/src/org/sleuthkit/autopsy/geolocation/datamodel/Waypoint.java b/Core/src/org/sleuthkit/autopsy/geolocation/datamodel/Waypoint.java index f0ec1e50d3..223b1e9cb3 100755 --- a/Core/src/org/sleuthkit/autopsy/geolocation/datamodel/Waypoint.java +++ b/Core/src/org/sleuthkit/autopsy/geolocation/datamodel/Waypoint.java @@ -198,8 +198,13 @@ public class Waypoint { try { List attributeList = artifact.getAttributes(); for (BlackboardAttribute attribute : attributeList) { - BlackboardAttribute.ATTRIBUTE_TYPE type = BlackboardAttribute.ATTRIBUTE_TYPE.fromID(attribute.getAttributeType().getTypeID()); - attributeMap.put(type, attribute); + try{ + BlackboardAttribute.ATTRIBUTE_TYPE type = BlackboardAttribute.ATTRIBUTE_TYPE.fromID(attribute.getAttributeType().getTypeID()); + attributeMap.put(type, attribute); + } catch(IllegalArgumentException ex) { + // This was thrown due to a custom attribute that geolocation + // does not currently support. + } } } catch (TskCoreException ex) { throw new GeoLocationDataException("Unable to get attributes from artifact", ex); diff --git a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/ALeappAnalyzerIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/ALeappAnalyzerIngestModule.java index 705468ae15..1492670ada 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/ALeappAnalyzerIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/ALeappAnalyzerIngestModule.java @@ -155,7 +155,7 @@ public class ALeappAnalyzerIngestModule implements DataSourceIngestModule { statusHelper.switchToDeterminate(aLeappFilesToProcess.size()); processALeappFs(dataSource, currentCase, statusHelper, tempOutputPath.toString()); } else { - aLeappFilesToProcess = findaLeappFilesToProcess(dataSource); + aLeappFilesToProcess = LeappFileProcessor.findLeappFilesToProcess(dataSource); statusHelper.switchToDeterminate(aLeappFilesToProcess.size()); Integer filesProcessedCount = 0; @@ -268,40 +268,7 @@ public class ALeappAnalyzerIngestModule implements DataSourceIngestModule { } - /** - * Find the files that will be processed by the aLeapp program - * - * @param dataSource - * - * @return List of abstract files to process. - */ - private List findaLeappFilesToProcess(Content dataSource) { - List aLeappFiles = new ArrayList<>(); - - FileManager fileManager = getCurrentCase().getServices().getFileManager(); - - // findFiles use the SQL wildcard % in the file name - try { - aLeappFiles = fileManager.findFiles(dataSource, "%", "/"); //NON-NLS - } catch (TskCoreException ex) { - logger.log(Level.WARNING, "No files found to process"); //NON-NLS - return aLeappFiles; - } - - List aLeappFilesToProcess = new ArrayList<>(); - for (AbstractFile aLeappFile : aLeappFiles) { - if (((aLeappFile.getLocalAbsPath() != null) - && (!aLeappFile.getNameExtension().isEmpty() && (!aLeappFile.isVirtual()))) - && ((aLeappFile.getName().toLowerCase().contains(".zip") || (aLeappFile.getName().toLowerCase().contains(".tar"))) - || aLeappFile.getName().toLowerCase().contains(".tgz"))) { - aLeappFilesToProcess.add(aLeappFile); - - } - } - - return aLeappFilesToProcess; - } /** * Build the aLeapp command to run diff --git a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/ILeappAnalyzerIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/ILeappAnalyzerIngestModule.java index 19cf344942..d124801046 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/ILeappAnalyzerIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/ILeappAnalyzerIngestModule.java @@ -155,7 +155,7 @@ public class ILeappAnalyzerIngestModule implements DataSourceIngestModule { statusHelper.switchToDeterminate(iLeappFilesToProcess.size()); processILeappFs(dataSource, currentCase, statusHelper, tempOutputPath.toString()); } else { - iLeappFilesToProcess = findiLeappFilesToProcess(dataSource); + iLeappFilesToProcess = LeappFileProcessor.findLeappFilesToProcess(dataSource); statusHelper.switchToDeterminate(iLeappFilesToProcess.size()); Integer filesProcessedCount = 0; @@ -268,41 +268,6 @@ public class ILeappAnalyzerIngestModule implements DataSourceIngestModule { } - /** - * Find the files that will be processed by the iLeapp program - * - * @param dataSource - * - * @return List of abstract files to process. - */ - private List findiLeappFilesToProcess(Content dataSource) { - - List iLeappFiles = new ArrayList<>(); - - FileManager fileManager = getCurrentCase().getServices().getFileManager(); - - // findFiles use the SQL wildcard % in the file name - try { - iLeappFiles = fileManager.findFiles(dataSource, "%", "/"); //NON-NLS - } catch (TskCoreException ex) { - logger.log(Level.WARNING, "No files found to process"); //NON-NLS - return iLeappFiles; - } - - List iLeappFilesToProcess = new ArrayList<>(); - for (AbstractFile iLeappFile : iLeappFiles) { - if (((iLeappFile.getLocalAbsPath() != null) - && (!iLeappFile.getNameExtension().isEmpty() && (!iLeappFile.isVirtual()))) - && ((iLeappFile.getName().toLowerCase().contains(".zip") || (iLeappFile.getName().toLowerCase().contains(".tar"))) - || iLeappFile.getName().toLowerCase().contains(".tgz"))) { - iLeappFilesToProcess.add(iLeappFile); - - } - } - - return iLeappFilesToProcess; - } - /** * Build the command to run xLeapp * @param moduleOutputPath output path for xLeapp diff --git a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/LeappFileProcessor.java b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/LeappFileProcessor.java index cb76601da0..e9026055cf 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/LeappFileProcessor.java +++ b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/LeappFileProcessor.java @@ -36,8 +36,10 @@ import java.util.Collection; import java.util.Collections; import java.util.Date; import java.util.HashMap; +import java.util.HashSet; import static java.util.Locale.US; import java.util.Map; +import java.util.Set; import java.util.logging.Level; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -49,7 +51,9 @@ import org.apache.commons.collections4.MapUtils; import org.apache.commons.io.FilenameUtils; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.casemodule.Case; +import static org.sleuthkit.autopsy.casemodule.Case.getCurrentCase; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; +import org.sleuthkit.autopsy.casemodule.services.FileManager; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.PlatformUtil; import org.sleuthkit.autopsy.ingest.IngestModule.IngestModuleException; @@ -705,23 +709,40 @@ public final class LeappFileProcessor { xmlFile, true); } - /** - * Create custom artifacts that are defined in the xLeapp xml file(s). - * - */ - private void createCustomArtifacts(Blackboard blkBoard) { + + private static final Set ALLOWED_EXTENSIONS = new HashSet<>(Arrays.asList("zip", "tar", "tgz")); - for (Map.Entry customArtifact : CUSTOM_ARTIFACT_MAP.entrySet()) { - String artifactName = customArtifact.getKey(); - String artifactDescription = customArtifact.getValue(); + /** + * Find the files that will be processed by the iLeapp program + * + * @param dataSource + * + * @return List of abstract files to process. + */ + static List findLeappFilesToProcess(Content dataSource) { - try { - BlackboardArtifact.Type customArtifactType = blkBoard.getOrAddArtifactType(artifactName, artifactDescription); - } catch (Blackboard.BlackboardException ex) { - logger.log(Level.WARNING, String.format("Failed to create custom artifact type %s.", artifactName), ex); - } - + List leappFiles = new ArrayList<>(); + + FileManager fileManager = getCurrentCase().getServices().getFileManager(); + + // findFiles use the SQL wildcard % in the file name + try { + leappFiles = fileManager.findFiles(dataSource, "%", "/"); //NON-NLS + } catch (TskCoreException ex) { + logger.log(Level.WARNING, "No files found to process"); //NON-NLS + return leappFiles; } - } + List leappFilesToProcess = new ArrayList<>(); + for (AbstractFile leappFile : leappFiles) { + if (((leappFile.getLocalAbsPath() != null) + && !leappFile.isVirtual()) + && leappFile.getNameExtension() != null + && ALLOWED_EXTENSIONS.contains(leappFile.getNameExtension().toLowerCase())) { + leappFilesToProcess.add(leappFile); + } + } + + return leappFilesToProcess; + } } diff --git a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/aleap-artifact-attribute-reference.xml b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/aleap-artifact-attribute-reference.xml index 1ed5bdef1b..784ccefa18 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/aleap-artifact-attribute-reference.xml +++ b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/aleap-artifact-attribute-reference.xml @@ -39,7 +39,7 @@ - + diff --git a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/ileap-artifact-attribute-reference.xml b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/ileap-artifact-attribute-reference.xml index a4169395aa..c0c1b43985 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/ileap-artifact-attribute-reference.xml +++ b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/ileap-artifact-attribute-reference.xml @@ -42,8 +42,8 @@ - - + + @@ -84,7 +84,7 @@ - + @@ -113,7 +113,7 @@ - + @@ -143,13 +143,13 @@ - - - - - - - + + + + + + + @@ -160,7 +160,7 @@ - + @@ -209,7 +209,7 @@ - + @@ -226,12 +226,12 @@ - + - + @@ -255,7 +255,7 @@ - + @@ -271,7 +271,7 @@ - + @@ -286,7 +286,7 @@ - + @@ -301,7 +301,7 @@ - + @@ -344,12 +344,12 @@ - + - + @@ -362,7 +362,7 @@ - + @@ -388,7 +388,7 @@ - + @@ -397,7 +397,7 @@ - + @@ -412,14 +412,14 @@ - + - + @@ -528,8 +528,8 @@ - - + + @@ -584,11 +584,11 @@ - - - - - + + + + + --> @@ -596,10 +596,10 @@ - - - - + + + + @@ -651,7 +651,7 @@ - + @@ -680,7 +680,7 @@ - + @@ -705,7 +705,7 @@ - +