Merge branch 'develop' of github.com:sleuthkit/autopsy into 7187-translationMap

Core/src/org/sleuthkit/autopsy/apputils/ApplicationLoggers.java

@@ -0,0 +1,103 @@
+/*
+ * Autopsy Forensic Browser
+ *
+ * Copyright 2020 Basis Technology Corp.
+ * Contact: carrier <at> sleuthkit <dot> org
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.sleuthkit.autopsy.apputils;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.sql.Timestamp;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.logging.FileHandler;
+import java.util.logging.Formatter;
+import java.util.logging.LogRecord;
+import java.util.logging.Logger;
+import org.sleuthkit.autopsy.coreutils.PlatformUtil;
+
+/**
+ * A utility that creates and stores application loggers.
+ *
+ * TODO (Jira-7175): This code is the third copy of code that originally
+ * appeared in org.sleuthkit.autopsy.coreutils.Logger. The second copy is in
+ * org.sleuthkit.autopsy.experimental.autoingest.AutoIngestSystemLogger. This
+ * class should allow the replacement of AutoIngestSystemLogger and the
+ * elimination of duplicate code in coreutils.Logger through delegation
+ * (maintaining the public API for coreutils.Logger).
+ */
+final public class ApplicationLoggers {
+
+    private static final int LOG_SIZE = 50000000; // In bytes, zero is unlimited.
+    private static final int LOG_FILE_COUNT = 10;
+    private static final String NEWLINE = System.lineSeparator();
+    private static final Map<String, Logger> loggers = new HashMap<>();
+
+    /**
+     * Gets the logger for a given application log file. The log file will be
+     * located in the var/log directory of the platform user directory and will
+     * have a name of the form [log name].log.
+     *
+     * @return The logger.
+     */
+    synchronized public static Logger getLogger(String logName) {
+        Logger logger;
+        if (loggers.containsKey(logName)) {
+            logger = loggers.get(logName);
+        } else {
+            logger = Logger.getLogger(logName);
+            Path logFilePath = Paths.get(PlatformUtil.getUserDirectory().getAbsolutePath(), "var", "log", String.format("%s.log", logName));
+            try {
+                FileHandler fileHandler = new FileHandler(logFilePath.toString(), LOG_SIZE, LOG_FILE_COUNT);
+                fileHandler.setEncoding(PlatformUtil.getLogFileEncoding());
+                fileHandler.setFormatter(new Formatter() {
+                    @Override
+                    public String format(LogRecord record) {
+                        Throwable thrown = record.getThrown();
+                        String stackTrace = ""; //NON-NLS
+                        while (thrown != null) {
+                            stackTrace += thrown.toString() + NEWLINE;
+                            for (StackTraceElement traceElem : record.getThrown().getStackTrace()) {
+                                stackTrace += "\t" + traceElem.toString() + NEWLINE; //NON-NLS
+                            }
+                            thrown = thrown.getCause();
+                        }
+                        return (new Timestamp(record.getMillis())).toString() + " " //NON-NLS
+                                + record.getSourceClassName() + " " //NON-NLS
+                                + record.getSourceMethodName() + NEWLINE
+                                + record.getLevel() + ": " //NON-NLS
+                                + this.formatMessage(record) + NEWLINE
+                                + stackTrace;
+                    }
+                });
+                logger.addHandler(fileHandler);
+                logger.setUseParentHandlers(false);
+            } catch (SecurityException | IOException ex) {
+                throw new RuntimeException(String.format("Error initializing file handler for %s", logFilePath), ex); //NON-NLS
+            }
+            loggers.put(logName, logger);
+        }
+        return logger;
+    }
+
+    /**
+     * Prevents instantiation of this utility class.
+     */
+    private ApplicationLoggers() {
+    }
+
+}

Core/src/org/sleuthkit/autopsy/casemodule/Bundle.properties-MERGED

@@ -364,13 +364,14 @@ AddImageWizardIngestConfigPanel.CANCEL_BUTTON.text=Cancel
 NewCaseVisualPanel1.CaseFolderOnCDriveError.text=Warning: Path to multi-user case folder is on \"C:\" drive
 NewCaseVisualPanel1.CaseFolderOnInternalDriveWindowsError.text=Warning: Path to case folder is on \"C:\" drive. Case folder is created on the target system
 NewCaseVisualPanel1.CaseFolderOnInternalDriveLinuxError.text=Warning: Path to case folder is on the target system. Create case folder in mounted drive.
+NewCaseVisualPanel1.uncPath.error=Error: UNC paths are not allowed for Single-User cases
 CollaborationMonitor.addingDataSourceStatus.msg={0} adding data source
 CollaborationMonitor.analyzingDataSourceStatus.msg={0} analyzing {1}
 MissingImageDialog.lbWarning.text=
 MissingImageDialog.lbWarning.toolTipText=
 NewCaseVisualPanel1.caseParentDirWarningLabel.text=
-NewCaseVisualPanel1.multiUserCaseRadioButton.text=Multi-user		
+NewCaseVisualPanel1.multiUserCaseRadioButton.text=Multi-User		
-NewCaseVisualPanel1.singleUserCaseRadioButton.text=Single-user
+NewCaseVisualPanel1.singleUserCaseRadioButton.text=Single-User
 NewCaseVisualPanel1.caseTypeLabel.text=Case Type: 
 SingleUserCaseConverter.BadDatabaseFileName=Database file does not exist!
 SingleUserCaseConverter.AlreadyMultiUser=Case is already multi-user!

Core/src/org/sleuthkit/autopsy/communications/CVTPersonaCache.java

@@ -82,9 +82,9 @@ final public class CVTPersonaCache {
     }
 
     /**
-     * Returns the list of PersonaAccounts for the given Account typeSpecificId.
+     * Returns the list of PersonaAccounts for the given Account.
      *
-     * @param typeSpecificID Account typeSpecificId.
+     * @param account The account.
      *
      * @return List of PersonaAccounts for id or empty list if none were found.
      *

Core/src/org/sleuthkit/autopsy/contentviewers/artifactviewers/GeneralPurposeArtifactViewer.java

@@ -243,7 +243,7 @@ public class GeneralPurposeArtifactViewer extends AbstractArtifactDetailsPanel i
      * @param attributeMap   The map of attributes that exist for the artifact.
      * @param dataSourceName The name of the datasource that caused the creation
      *                       of the artifact.
-     * @param sourceFileName The name of the file that caused the creation of
+     * @param sourceFilePath The path of the file that caused the creation of
      *                       the artifact.
      */
     @NbBundle.Messages({"GeneralPurposeArtifactViewer.dates.created=Created",

Core/src/org/sleuthkit/autopsy/datamodel/DataSourceGroupingNode.java

@@ -22,6 +22,7 @@ import java.util.Arrays;
 import java.util.Collections;
 import java.util.Optional;
 import java.util.logging.Level;
+import org.openide.util.lookup.Lookups;
 import org.sleuthkit.autopsy.casemodule.Case;
 import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException;
 import org.sleuthkit.autopsy.coreutils.Logger;
@@ -46,7 +47,8 @@ class DataSourceGroupingNode extends DisplayableItemNode {
     DataSourceGroupingNode(DataSource dataSource) {
 
         super (Optional.ofNullable(createDSGroupingNodeChildren(dataSource))
-                        .orElse(new RootContentChildren(Arrays.asList(Collections.EMPTY_LIST))));
+                        .orElse(new RootContentChildren(Arrays.asList(Collections.EMPTY_LIST))), 
+                Lookups.singleton(dataSource));
 
         if (dataSource instanceof Image) {
             Image image = (Image) dataSource;

Core/src/org/sleuthkit/autopsy/datamodel/utils/IconsUtil.java

@@ -125,6 +125,8 @@ public final class IconsUtil {
             imageFile = "domain-16.png"; //NON-NLS
         } else if (typeID == ARTIFACT_TYPE.TSK_GPS_AREA.getTypeID()) {
             imageFile = "gps-area.png"; //NON-NLS
+        } else if (typeID == ARTIFACT_TYPE.TSK_YARA_HIT.getTypeID()) {
+            imageFile = "yara_16.png"; //NON-NLS
         } else {
             imageFile = "artifact-icon.png"; //NON-NLS
         }

Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/CityRecord.java

@@ -28,18 +28,21 @@ public class CityRecord extends KdTree.XYZPoint {
 
     private final String cityName;
     private final String country;
+    private final String state;
 
     /**
      * Main constructor.
      *
      * @param cityName The name of the city.
+     * @param state The state of the city.
      * @param country The country of that city.
      * @param latitude Latitude for the city.
      * @param longitude Longitude for the city.
      */
-    CityRecord(String cityName, String country, double latitude, double longitude) {
+    CityRecord(String cityName, String state, String country, double latitude, double longitude) {
         super(latitude, longitude);
         this.cityName = cityName;
+        this.state = state;
         this.country = country;
     }
 
@@ -50,6 +53,13 @@ public class CityRecord extends KdTree.XYZPoint {
         return cityName;
     }
 
+    /**
+     * @return The state of the city.
+     */
+    public String getState() {
+        return state;
+    }
+    
     /**
      * @return The country of that city.
      */

Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/ClosestCityMapper.java

@@ -42,6 +42,7 @@ class ClosestCityMapper {
 
     // index within a csv row of pertinent data
     private static final int CITY_NAME_IDX = 0;
+    private static final int STATE_NAME_IDX = 7;
     private static final int COUNTRY_NAME_IDX = 4;
     private static final int LAT_IDX = 2;
     private static final int LONG_IDX = 3;
@@ -52,7 +53,7 @@ class ClosestCityMapper {
     // Identifies if cities are in last, first format like "Korea, South"
     private static final Pattern COUNTRY_WITH_COMMA = Pattern.compile("^\\s*([^,]*)\\s*,\\s*([^,]*)\\s*$");
 
-    private static final int MAX_IDX = Stream.of(CITY_NAME_IDX, COUNTRY_NAME_IDX, LAT_IDX, LONG_IDX)
+    private static final int MAX_IDX = Stream.of(CITY_NAME_IDX, STATE_NAME_IDX, COUNTRY_NAME_IDX, LAT_IDX, LONG_IDX)
             .max(Integer::compare)
             .get();
 
@@ -169,12 +170,15 @@ class ClosestCityMapper {
             return null;
         }
 
+        // city is required
         String cityName = csvRow.get(CITY_NAME_IDX);
         if (StringUtils.isBlank(cityName)) {
             logger.log(Level.WARNING, String.format("No city name determined for line %d.", lineNum));
             return null;
         }
 
+        // state and country can be optional
+        String stateName = csvRow.get(STATE_NAME_IDX);
         String countryName = parseCountryName(csvRow.get(COUNTRY_NAME_IDX), lineNum);
 
         Double lattitude = tryParse(csvRow.get(LAT_IDX));
@@ -189,7 +193,7 @@ class ClosestCityMapper {
             return null;
         }
 
-        return new CityRecord(cityName, countryName, lattitude, longitude);
+        return new CityRecord(cityName, stateName, countryName, lattitude, longitude);
     }
 
     /**

Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/GeolocationSummary.java

@@ -309,7 +309,7 @@ public class GeolocationSummary implements DefaultArtifactUpdateGovernor {
         Long mostRecent = null;
 
         for (MapWaypoint pt : dataSourcePoints) {
-            CityRecord city = closestCityMapper.findClosest(new CityRecord(null, null, pt.getX(), pt.getY()));
+            CityRecord city = closestCityMapper.findClosest(new CityRecord(null, null, null, pt.getX(), pt.getY()));
             Long curTime = pt.getTimestamp();
             if (curTime != null && (mostRecent == null || curTime > mostRecent)) {
                 mostRecent = curTime;

Core/src/org/sleuthkit/autopsy/datasourcesummary/datamodel/IngestModuleCheckUtil.java

@@ -1,145 +0,0 @@
-/*
- * Autopsy Forensic Browser
- *
- * Copyright 2020 Basis Technology Corp.
- * Contact: carrier <at> sleuthkit <dot> org
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.sleuthkit.autopsy.datasourcesummary.datamodel;
-
-import java.util.Map;
-import java.util.stream.Collectors;
-import org.apache.commons.lang3.StringUtils;
-import org.openide.util.NbBundle.Messages;
-import org.sleuthkit.autopsy.datasourcesummary.datamodel.SleuthkitCaseProvider.SleuthkitCaseProviderException;
-import org.sleuthkit.datamodel.DataSource;
-import org.sleuthkit.datamodel.IngestJobInfo;
-import org.sleuthkit.datamodel.IngestModuleInfo;
-import org.sleuthkit.datamodel.SleuthkitCase;
-import org.sleuthkit.datamodel.TskCoreException;
-
-/**
- * Utilities for checking if an ingest module has been run on a datasource.
- */
-@Messages({
-    "IngestModuleCheckUtil_recentActivityModuleName=Recent Activity",
-    
-})
-public class IngestModuleCheckUtil {
-    public static final String RECENT_ACTIVITY_FACTORY = "org.sleuthkit.autopsy.recentactivity.RecentActivityExtracterModuleFactory";
-    public static final String RECENT_ACTIVITY_MODULE_NAME = Bundle.IngestModuleCheckUtil_recentActivityModuleName();
-    
-    // IngestModuleInfo separator for unique_name
-    private static final String UNIQUE_NAME_SEPARATOR = "-";
-
-    private final SleuthkitCaseProvider caseProvider;
-        
-    /**
-     * Main constructor.
-     */
-    public IngestModuleCheckUtil() {
-        this(SleuthkitCaseProvider.DEFAULT);
-
-    }
-
-    /**
-     * Main constructor with external dependencies specified. This constructor
-     * is designed with unit testing in mind since mocked dependencies can be
-     * utilized.
-     *
-     * @param provider The object providing the current SleuthkitCase.
-     */
-    public IngestModuleCheckUtil(SleuthkitCaseProvider provider) {
-
-        this.caseProvider = provider;
-    }
-
-    
-    /**
-     * Gets the fully qualified factory from the IngestModuleInfo.
-     * @param info The IngestJobInfo.
-     * @return The fully qualified factory.
-     */
-    private static String getFullyQualifiedFactory(IngestModuleInfo info) {
-        if (info == null) {
-            return null;
-        }
-        
-        String qualifiedName = info.getUniqueName();
-        if (StringUtils.isBlank(qualifiedName)) {
-            return null;
-        }
-        
-        return qualifiedName.split(UNIQUE_NAME_SEPARATOR)[0];
-    }
-        
-
-    /**
-     * Whether or not the ingest job info contains the ingest modulename.
-     * @param info The IngestJobInfo.
-     * @param fullyQualifiedFactory The fully qualified classname of the relevant factory.
-     * @return True if the ingest module name is contained in the data.
-     */
-    private static boolean hasIngestModule(IngestJobInfo info, String fullyQualifiedFactory) {
-        if (info == null || info.getIngestModuleInfo() == null || StringUtils.isBlank(fullyQualifiedFactory)) {
-            return false;
-        }
-
-        return info.getIngestModuleInfo().stream()
-                .anyMatch((moduleInfo) -> {
-                    String thisQualifiedFactory = getFullyQualifiedFactory(moduleInfo);
-                    return fullyQualifiedFactory.equalsIgnoreCase(thisQualifiedFactory);
-                });
-    }
-
-    /**
-     * Whether or not a data source has been ingested with a particular ingest module.
-     * @param dataSource The datasource.
-     * @param fullyQualifiedFactory The fully qualified classname of the relevant factory.
-     * @return Whether or not a data source has been ingested with a particular ingest module.
-     * @throws TskCoreException 
-     * @throws SleuthkitCaseProviderException
-     */
-    public boolean isModuleIngested(DataSource dataSource, String fullyQualifiedFactory)
-            throws TskCoreException, SleuthkitCaseProviderException {
-        if (dataSource == null) {
-            return false;
-        }
-
-        long dataSourceId = dataSource.getId();
-
-        return caseProvider.get().getIngestJobs().stream()
-                .anyMatch((ingestJob) -> {
-                    return ingestJob != null
-                            && ingestJob.getObjectId() == dataSourceId
-                            && hasIngestModule(ingestJob, fullyQualifiedFactory);
-                });
-
-    }
-    
-    /**
-     * Get a mapping of fully qualified factory name to display name.
-     * @param skCase The SleuthkitCase.
-     * @return The mapping of fully qualified factory name to display name.
-     * @throws TskCoreException 
-     */
-    public static Map<String, String> getFactoryDisplayNames(SleuthkitCase skCase) throws TskCoreException {
-        return skCase.getIngestJobs().stream()
-                .flatMap(ingestJob -> ingestJob.getIngestModuleInfo().stream())
-                .collect(Collectors.toMap(
-                        (moduleInfo) -> getFullyQualifiedFactory(moduleInfo), 
-                        (moduleInfo) -> moduleInfo.getDisplayName(),
-                        (a,b) -> a));
-    }
-}

Core/src/org/sleuthkit/autopsy/datasourcesummary/ui/AnalysisPanel.java

@@ -29,8 +29,6 @@ import org.sleuthkit.autopsy.datasourcesummary.uiutils.DataFetchWorker;
 import org.sleuthkit.autopsy.datasourcesummary.uiutils.IngestRunningLabel;
 import org.sleuthkit.autopsy.datasourcesummary.uiutils.JTablePanel;
 import org.sleuthkit.autopsy.datasourcesummary.uiutils.JTablePanel.ColumnModel;
-import org.sleuthkit.autopsy.modules.hashdatabase.HashLookupModuleFactory;
-import org.sleuthkit.autopsy.modules.interestingitems.InterestingItemsIngestModuleFactory;
 import org.sleuthkit.datamodel.DataSource;
 
 /**
@@ -46,15 +44,6 @@ public class AnalysisPanel extends BaseDataSourceSummaryPanel {
 
     private static final long serialVersionUID = 1L;
 
-    private static final String KEYWORD_SEARCH_MODULE_NAME = Bundle.AnalysisPanel_keywordSearchModuleName();
-    private static final String KEYWORD_SEARCH_FACTORY = "org.sleuthkit.autopsy.keywordsearch.KeywordSearchModuleFactory";
-
-    private static final String INTERESTING_ITEM_MODULE_NAME = new InterestingItemsIngestModuleFactory().getModuleDisplayName();
-    private static final String INTERESTING_ITEM_FACTORY = InterestingItemsIngestModuleFactory.class.getCanonicalName();
-
-    private static final String HASHSET_MODULE_NAME = HashLookupModuleFactory.getModuleName();
-    private static final String HASHSET_FACTORY = HashLookupModuleFactory.class.getCanonicalName();
-
     /**
      * Default Column definitions for each table
      */
@@ -90,9 +79,8 @@ public class AnalysisPanel extends BaseDataSourceSummaryPanel {
             keywordHitsTable,
             interestingItemsTable
     );
-    
+
     private final IngestRunningLabel ingestRunningLabel = new IngestRunningLabel();
-    
 
     /**
      * All of the components necessary for data fetch swing workers to load data
@@ -115,28 +103,26 @@ public class AnalysisPanel extends BaseDataSourceSummaryPanel {
                 // hashset hits loading components
                 new DataFetchWorker.DataFetchComponents<>(
                         (dataSource) -> analysisData.getHashsetCounts(dataSource),
-                        (result) -> showResultWithModuleCheck(hashsetHitsTable, result, HASHSET_FACTORY, HASHSET_MODULE_NAME)),
+                        (result) -> hashsetHitsTable.showDataFetchResult(result)),
                 // keyword hits loading components
                 new DataFetchWorker.DataFetchComponents<>(
                         (dataSource) -> analysisData.getKeywordCounts(dataSource),
-                        (result) -> showResultWithModuleCheck(keywordHitsTable, result, KEYWORD_SEARCH_FACTORY, KEYWORD_SEARCH_MODULE_NAME)),
+                        (result) -> keywordHitsTable.showDataFetchResult(result)),
                 // interesting item hits loading components
                 new DataFetchWorker.DataFetchComponents<>(
                         (dataSource) -> analysisData.getInterestingItemCounts(dataSource),
-                        (result) -> showResultWithModuleCheck(interestingItemsTable, result, INTERESTING_ITEM_FACTORY, INTERESTING_ITEM_MODULE_NAME))
+                        (result) -> interestingItemsTable.showDataFetchResult(result))
         );
 
         initComponents();
     }
 
-    
     @Override
     public void close() {
         ingestRunningLabel.unregister();
         super.close();
     }
-    
+
-    
     @Override
     protected void fetchInformation(DataSource dataSource) {
         fetchInformation(dataFetchComponents, dataSource);

Core/src/org/sleuthkit/autopsy/datasourcesummary/ui/BaseDataSourceSummaryPanel.java

@@ -25,7 +25,6 @@ import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import java.util.Set;
-import java.util.function.Predicate;
 import java.util.logging.Level;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
@@ -39,11 +38,8 @@ import org.openide.util.NbBundle.Messages;
 import org.sleuthkit.autopsy.casemodule.Case;
 import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException;
 import org.sleuthkit.autopsy.coreutils.Logger;
-import org.sleuthkit.autopsy.datasourcesummary.datamodel.IngestModuleCheckUtil;
-import org.sleuthkit.autopsy.datasourcesummary.datamodel.SleuthkitCaseProvider.SleuthkitCaseProviderException;
 import org.sleuthkit.autopsy.datasourcesummary.uiutils.CellModelTableCellRenderer;
 import org.sleuthkit.autopsy.datasourcesummary.uiutils.DataFetchResult;
-import org.sleuthkit.autopsy.datasourcesummary.uiutils.DataFetchResult.ResultType;
 import org.sleuthkit.autopsy.datasourcesummary.uiutils.DataFetchWorker;
 import org.sleuthkit.autopsy.datasourcesummary.uiutils.DataFetchWorker.DataFetchComponents;
 import org.sleuthkit.autopsy.datasourcesummary.uiutils.EventUpdateHandler;
@@ -73,7 +69,6 @@ abstract class BaseDataSourceSummaryPanel extends JPanel {
     private static final Logger logger = Logger.getLogger(BaseDataSourceSummaryPanel.class.getName());
 
     private final SwingWorkerSequentialExecutor executor = new SwingWorkerSequentialExecutor();
-    private final IngestModuleCheckUtil ingestModuleCheck = new IngestModuleCheckUtil();
     private final EventUpdateHandler updateHandler;
     private final List<UpdateGovernor> governors;
 
@@ -319,8 +314,8 @@ abstract class BaseDataSourceSummaryPanel extends JPanel {
     }
 
     /**
-     * Given the relevant file, navigates to the file in the
+     * Given the relevant file, navigates to the file in the tree and closes
-     * tree and closes data source summary dialog if open.
+     * data source summary dialog if open.
      *
      * @param file The file.
      * @return The menu item list for a go to artifact menu item.
@@ -471,76 +466,4 @@ abstract class BaseDataSourceSummaryPanel extends JPanel {
             fetchInformation(dataSource);
         }
     }
-
-    /**
-     * Get default message when there is a NotIngestedWithModuleException.
-     *
-     * @param moduleName The moduleName.
-     *
-     * @return Message specifying that the ingest module was not run.
-     */
-    @Messages({
-        "# {0} - module name",
-        "BaseDataSourceSummaryPanel_defaultNotIngestMessage=The {0} ingest module has not been run on this data source."
-    })
-    protected String getDefaultNoIngestMessage(String moduleName) {
-        return Bundle.BaseDataSourceSummaryPanel_defaultNotIngestMessage(moduleName);
-    }
-
-    /**
-     * Utility method to return the IngestModuleCheckUtil.
-     *
-     * @return The IngestModuleCheckUtil.
-     */
-    protected IngestModuleCheckUtil getIngestModuleCheckUtil() {
-        return this.ingestModuleCheck;
-    }
-
-    /**
-     * Utility method that in the event of a) there are no results and b) a
-     * relevant ingest module has not been run on this datasource, then a
-     * message indicating the unrun ingest module will be shown. Otherwise, the
-     * default LoadableComponent.showDataFetchResult behavior will be used.
-     *
-     * @param component The component.
-     * @param result The data result.
-     * @param factoryClass The fully qualified class name of the relevant
-     * factory.
-     * @param moduleName The name of the ingest module (i.e. 'Keyword Search').
-     */
-    protected <T> void showResultWithModuleCheck(LoadableComponent<List<T>> component, DataFetchResult<List<T>> result, String factoryClass, String moduleName) {
-        Predicate<List<T>> hasResults = (lst) -> lst != null && !lst.isEmpty();
-        showResultWithModuleCheck(component, result, hasResults, factoryClass, moduleName);
-    }
-
-    /**
-     * Utility method that in the event of a) there are no results and b) a
-     * relevant ingest module has not been run on this datasource, then a
-     * message indicating the unrun ingest module will be shown. Otherwise, the
-     * default LoadableComponent.showDataFetchResult behavior will be used.
-     *
-     * @param component The component.
-     * @param result The data result.
-     * @param hasResults Given the data type, will provide whether or not the
-     * data contains any actual results.
-     * @param factoryClass The fully qualified class name of the relevant
-     * factory.
-     * @param moduleName The name of the ingest module (i.e. 'Keyword Search').
-     */
-    protected <T> void showResultWithModuleCheck(LoadableComponent<T> component, DataFetchResult<T> result,
-            Predicate<T> hasResults, String factoryClass, String moduleName) {
-
-        if (result != null && result.getResultType() == ResultType.SUCCESS && !hasResults.test(result.getData())) {
-            try {
-                if (!ingestModuleCheck.isModuleIngested(getDataSource(), factoryClass)) {
-                    component.showMessage(getDefaultNoIngestMessage(moduleName));
-                    return;
-                }
-            } catch (TskCoreException | SleuthkitCaseProviderException ex) {
-                logger.log(Level.WARNING, "There was an error while checking for ingest modules for datasource.", ex);
-            }
-        }
-
-        component.showDataFetchResult(result);
-    }
 }

Core/src/org/sleuthkit/autopsy/datasourcesummary/ui/GeolocationPanel.java

@@ -63,8 +63,6 @@ import org.sleuthkit.datamodel.DataSource;
 public class GeolocationPanel extends BaseDataSourceSummaryPanel {
 
     private static final long serialVersionUID = 1L;
-    private static final String GPX_FACTORY = "org.python.proxies.GPX_Parser_Module$GPXParserFileIngestModuleFactory";
-    private static final String GPX_NAME = "GPX Parser";
     private static final int DAYS_COUNT = 30;
     private static final int MAX_COUNT = 10;
 
@@ -114,6 +112,8 @@ public class GeolocationPanel extends BaseDataSourceSummaryPanel {
      * @param whereUsedData The GeolocationSummary instance to use.
      */
     public GeolocationPanel(GeolocationSummary whereUsedData) {
+        super(whereUsedData);
+
         this.whereUsedData = whereUsedData;
         // set up data acquisition methods
         dataFetchComponents = Arrays.asList(
@@ -145,11 +145,19 @@ public class GeolocationPanel extends BaseDataSourceSummaryPanel {
             return null;
         }
 
-        if (StringUtils.isBlank(record.getCountry())) {
+        List<String> cityIdentifiers = Stream.of(record.getCityName(), record.getState(), record.getCountry())
-            return record.getCityName();
+                .filter(StringUtils::isNotBlank)
-        }
+                .collect(Collectors.toList());
 
-        return String.format("%s, %s", record.getCityName(), record.getCountry());
+        if (cityIdentifiers.size() == 1) {
+            return cityIdentifiers.get(0);
+        } else if (cityIdentifiers.size() == 2) {
+            return String.format("%s, %s", cityIdentifiers.get(0), cityIdentifiers.get(1));
+        } else if (cityIdentifiers.size() >= 3) {
+            return String.format("%s, %s; %s", cityIdentifiers.get(0), cityIdentifiers.get(1), cityIdentifiers.get(2));
+        }
+        
+        return null;
     }
 
     /**
@@ -211,7 +219,7 @@ public class GeolocationPanel extends BaseDataSourceSummaryPanel {
             goToGeolocation.setEnabled(true);
         }
 
-        showResultWithModuleCheck(table, convertedData, GPX_FACTORY, GPX_NAME);
+        table.showDataFetchResult(convertedData);
     }
 
     /**
@@ -225,7 +233,7 @@ public class GeolocationPanel extends BaseDataSourceSummaryPanel {
     private void openGeolocationWindow(DataSource dataSource, Integer daysLimit) {
         // notify dialog (if in dialog) should close.
         notifyParentClose();
-        
+
         // set the filter
         TopComponent topComponent = WindowManager.getDefault().findTopComponent(GeolocationTopComponent.class.getSimpleName());
         if (topComponent instanceof GeolocationTopComponent) {

Core/src/org/sleuthkit/autopsy/datasourcesummary/ui/PastCasesPanel.java

@@ -22,7 +22,6 @@ import java.util.Arrays;
 import java.util.List;
 import org.apache.commons.lang3.tuple.Pair;
 import org.openide.util.NbBundle.Messages;
-import org.sleuthkit.autopsy.centralrepository.ingestmodule.CentralRepoIngestModuleFactory;
 import org.sleuthkit.autopsy.datasourcesummary.datamodel.PastCasesSummary;
 import org.sleuthkit.autopsy.datasourcesummary.datamodel.PastCasesSummary.PastCasesResult;
 import org.sleuthkit.autopsy.datasourcesummary.uiutils.CellModelTableCellRenderer.DefaultCellModel;
@@ -46,8 +45,6 @@ import org.sleuthkit.datamodel.DataSource;
 public class PastCasesPanel extends BaseDataSourceSummaryPanel {
 
     private static final long serialVersionUID = 1L;
-    private static final String CR_FACTORY = CentralRepoIngestModuleFactory.class.getName();
-    private static final String CR_NAME = CentralRepoIngestModuleFactory.getModuleName();
 
     private static final ColumnModel<Pair<String, Long>> CASE_COL = new ColumnModel<>(
             Bundle.PastCasesPanel_caseColumn_title(),
@@ -84,6 +81,8 @@ public class PastCasesPanel extends BaseDataSourceSummaryPanel {
      * Creates new form PastCasesPanel
      */
     public PastCasesPanel(PastCasesSummary pastCaseData) {
+        super(pastCaseData);
+
         // set up data acquisition methods
         dataFetchComponents = Arrays.asList(
                 new DataFetchWorker.DataFetchComponents<>(
@@ -101,8 +100,8 @@ public class PastCasesPanel extends BaseDataSourceSummaryPanel {
      * @param result The result.
      */
     private void handleResult(DataFetchResult<PastCasesResult> result) {
-        showResultWithModuleCheck(notableFileTable, DataFetchResult.getSubResult(result, (res) -> res.getTaggedNotable()), CR_FACTORY, CR_NAME);
+        notableFileTable.showDataFetchResult(DataFetchResult.getSubResult(result, (res) -> res.getTaggedNotable()));
-        showResultWithModuleCheck(sameIdTable, DataFetchResult.getSubResult(result, (res) -> res.getSameIdsResults()), CR_FACTORY, CR_NAME);
+        sameIdTable.showDataFetchResult(DataFetchResult.getSubResult(result, (res) -> res.getSameIdsResults()));
     }
 
     @Override

Core/src/org/sleuthkit/autopsy/datasourcesummary/ui/RecentFilesPanel.java

@@ -23,7 +23,6 @@ import java.util.Arrays;
 import java.util.List;
 import java.util.function.Supplier;
 import org.openide.util.NbBundle.Messages;
-import org.sleuthkit.autopsy.datasourcesummary.datamodel.IngestModuleCheckUtil;
 import org.sleuthkit.autopsy.datasourcesummary.datamodel.RecentFilesSummary;
 import org.sleuthkit.autopsy.datasourcesummary.datamodel.RecentFilesSummary.RecentAttachmentDetails;
 import org.sleuthkit.autopsy.datasourcesummary.datamodel.RecentFilesSummary.RecentDownloadDetails;
@@ -44,8 +43,6 @@ import org.sleuthkit.datamodel.DataSource;
 public final class RecentFilesPanel extends BaseDataSourceSummaryPanel {
 
     private static final long serialVersionUID = 1L;
-    private static final String EMAIL_PARSER_FACTORY = "org.sleuthkit.autopsy.thunderbirdparser.EmailParserModuleFactory";
-    private static final String EMAIL_PARSER_MODULE_NAME = Bundle.RecentFilePanel_emailParserModuleName();
 
     private final List<JTablePanel<?>> tablePanelList = new ArrayList<>();
     private final List<DataFetchWorker.DataFetchComponents<DataSource, ?>> dataFetchComponents = new ArrayList<>();
@@ -167,11 +164,7 @@ public final class RecentFilesPanel extends BaseDataSourceSummaryPanel {
         DataFetchWorker.DataFetchComponents<DataSource, List<RecentFileDetails>> worker
                 = new DataFetchWorker.DataFetchComponents<>(
                         (dataSource) -> dataHandler.getRecentlyOpenedDocuments(dataSource, 10),
-                        (result) -> {
+                        (result) -> pane.showDataFetchResult(result));
-                            showResultWithModuleCheck(pane, result,
-                                    IngestModuleCheckUtil.RECENT_ACTIVITY_FACTORY,
-                                    IngestModuleCheckUtil.RECENT_ACTIVITY_MODULE_NAME);
-                        });
 
         dataFetchComponents.add(worker);
     }
@@ -210,11 +203,7 @@ public final class RecentFilesPanel extends BaseDataSourceSummaryPanel {
         DataFetchWorker.DataFetchComponents<DataSource, List<RecentDownloadDetails>> worker
                 = new DataFetchWorker.DataFetchComponents<>(
                         (dataSource) -> dataHandler.getRecentDownloads(dataSource, 10),
-                        (result) -> {
+                        (result) -> pane.showDataFetchResult(result));
-                            showResultWithModuleCheck(pane, result,
-                                    IngestModuleCheckUtil.RECENT_ACTIVITY_FACTORY,
-                                    IngestModuleCheckUtil.RECENT_ACTIVITY_MODULE_NAME);
-                        });
 
         dataFetchComponents.add(worker);
     }
@@ -253,7 +242,7 @@ public final class RecentFilesPanel extends BaseDataSourceSummaryPanel {
         DataFetchWorker.DataFetchComponents<DataSource, List<RecentAttachmentDetails>> worker
                 = new DataFetchWorker.DataFetchComponents<>(
                         (dataSource) -> dataHandler.getRecentAttachments(dataSource, 10),
-                        (result) -> showResultWithModuleCheck(pane, result, EMAIL_PARSER_FACTORY, EMAIL_PARSER_MODULE_NAME)
+                        (result) -> pane.showDataFetchResult(result)
                 );
 
         dataFetchComponents.add(worker);

Core/src/org/sleuthkit/autopsy/datasourcesummary/ui/TimelinePanel.java

@@ -69,7 +69,7 @@ public class TimelinePanel extends BaseDataSourceSummaryPanel {
     private static final Logger logger = Logger.getLogger(TimelinePanel.class.getName());
     private static final long serialVersionUID = 1L;
     private static final DateFormat EARLIEST_LATEST_FORMAT = getUtcFormat("MMM d, yyyy");
-    private static final DateFormat CHART_FORMAT = getUtcFormat("MMM d");
+    private static final DateFormat CHART_FORMAT = getUtcFormat("MMM d, yyyy");
     private static final int MOST_RECENT_DAYS_COUNT = 30;
 
     /**
@@ -103,6 +103,8 @@ public class TimelinePanel extends BaseDataSourceSummaryPanel {
      * Creates new form PastCasesPanel
      */
     public TimelinePanel(TimelineSummary timelineData) {
+        super(timelineData);
+        
         // set up data acquisition methods
         dataFetchComponents = Arrays.asList(
                 new DataFetchWorker.DataFetchComponents<>(

Core/src/org/sleuthkit/autopsy/datasourcesummary/ui/TypesPanel.java

@@ -26,16 +26,13 @@ import java.util.Arrays;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
-import java.util.logging.Level;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
-import org.apache.commons.lang3.StringUtils;
 import org.openide.util.NbBundle.Messages;
 import org.sleuthkit.autopsy.coreutils.FileTypeUtils.FileTypeCategory;
 import org.sleuthkit.autopsy.coreutils.Logger;
 import org.sleuthkit.autopsy.datasourcesummary.datamodel.TypesSummary;
 import org.sleuthkit.autopsy.datasourcesummary.datamodel.ContainerSummary;
-import org.sleuthkit.autopsy.datasourcesummary.datamodel.IngestModuleCheckUtil;
 import org.sleuthkit.autopsy.datasourcesummary.datamodel.MimeTypeSummary;
 import org.sleuthkit.autopsy.datasourcesummary.datamodel.SleuthkitCaseProvider.SleuthkitCaseProviderException;
 import org.sleuthkit.autopsy.datasourcesummary.uiutils.DataFetchResult;
@@ -251,25 +248,11 @@ class TypesPanel extends BaseDataSourceSummaryPanel {
                 // usage label worker
                 new DataFetchWorker.DataFetchComponents<>(
                         containerData::getDataSourceType,
-                        (result) -> {
+                        (result) -> usageLabel.showDataFetchResult(result)),
-                            showResultWithModuleCheck(
-                                    usageLabel,
-                                    result,
-                                    StringUtils::isNotBlank,
-                                    IngestModuleCheckUtil.RECENT_ACTIVITY_FACTORY,
-                                    IngestModuleCheckUtil.RECENT_ACTIVITY_MODULE_NAME);
-                        }),
                 // os label worker
                 new DataFetchWorker.DataFetchComponents<>(
                         containerData::getOperatingSystems,
-                        (result) -> {
+                        (result) -> osLabel.showDataFetchResult(result)),
-                            showResultWithModuleCheck(
-                                    osLabel,
-                                    result,
-                                    StringUtils::isNotBlank,
-                                    IngestModuleCheckUtil.RECENT_ACTIVITY_FACTORY,
-                                    IngestModuleCheckUtil.RECENT_ACTIVITY_MODULE_NAME);
-                        }),
                 // size label worker
                 new DataFetchWorker.DataFetchComponents<>(
                         (dataSource) -> {
@@ -379,50 +362,22 @@ class TypesPanel extends BaseDataSourceSummaryPanel {
      * @param result The result to be shown.
      */
     private void showMimeTypeCategories(DataFetchResult<TypesPieChartData> result) {
-        // if result is null check for ingest module and show empty results.
         if (result == null) {
-            showPieResultWithModuleCheck(null);
+            fileMimeTypesChart.showDataFetchResult(DataFetchResult.getSuccessResult(null));
             return;
         }
 
         // if error, show error
         if (result.getResultType() == ResultType.ERROR) {
-            this.fileMimeTypesChart.showDataFetchResult(DataFetchResult.getErrorResult(result.getException()));
+            fileMimeTypesChart.showDataFetchResult(DataFetchResult.getErrorResult(result.getException()));
             return;
         }
 
         TypesPieChartData data = result.getData();
         if (data == null) {
-            // if no data, do an ingest module check with empty results
+            fileMimeTypesChart.showDataFetchResult(DataFetchResult.getSuccessResult(null));
-            showPieResultWithModuleCheck(null);
-        } else if (!data.isUsefulContent()) {
-            // if no useful data, do an ingest module check and show data
-            showPieResultWithModuleCheck(data.getPieSlices());
         } else {
-            // otherwise, show the data
+            fileMimeTypesChart.showDataFetchResult(DataFetchResult.getSuccessResult(data.getPieSlices()));
-            this.fileMimeTypesChart.showDataFetchResult(DataFetchResult.getSuccessResult(data.getPieSlices()));
-        }
-    }
-
-    /**
-     * Shows a message in the fileMimeTypesChart about the data source not being
-     * ingested with the file type ingest module if the data source has not been
-     * ingested with that module. Also shows data if present.
-     *
-     * @param items The list of items to show.
-     */
-    private void showPieResultWithModuleCheck(List<PieChartItem> items) {
-        boolean hasBeenIngested = false;
-        try {
-            hasBeenIngested = this.getIngestModuleCheckUtil().isModuleIngested(getDataSource(), FILE_TYPE_FACTORY);
-        } catch (TskCoreException | SleuthkitCaseProviderException ex) {
-            logger.log(Level.WARNING, "There was an error fetching whether or not the current data source has been ingested with the file type ingest module.", ex);
-        }
-
-        if (hasBeenIngested) {
-            this.fileMimeTypesChart.showDataFetchResult(DataFetchResult.getSuccessResult(items));
-        } else {
-            this.fileMimeTypesChart.showDataWithMessage(items, getDefaultNoIngestMessage(FILE_TYPE_MODULE_NAME));
         }
     }
 

Core/src/org/sleuthkit/autopsy/datasourcesummary/ui/UserActivityPanel.java

@@ -27,7 +27,6 @@ import java.util.List;
 import java.util.Locale;
 import org.apache.commons.lang.StringUtils;
 import org.openide.util.NbBundle.Messages;
-import org.sleuthkit.autopsy.datasourcesummary.datamodel.IngestModuleCheckUtil;
 import org.sleuthkit.autopsy.datasourcesummary.datamodel.UserActivitySummary;
 import org.sleuthkit.autopsy.datasourcesummary.datamodel.UserActivitySummary.LastAccessedArtifact;
 import org.sleuthkit.autopsy.datasourcesummary.datamodel.UserActivitySummary.TopAccountResult;
@@ -287,43 +286,23 @@ public class UserActivityPanel extends BaseDataSourceSummaryPanel {
                 // top programs query
                 new DataFetchComponents<DataSource, List<TopProgramsResult>>(
                         (dataSource) -> userActivityData.getTopPrograms(dataSource, TOP_PROGS_COUNT),
-                        (result) -> {
+                        (result) -> topProgramsTable.showDataFetchResult(result)),
-                            showResultWithModuleCheck(topProgramsTable, result,
-                                    IngestModuleCheckUtil.RECENT_ACTIVITY_FACTORY,
-                                    IngestModuleCheckUtil.RECENT_ACTIVITY_MODULE_NAME);
-                        }),
                 // top domains query
                 new DataFetchComponents<DataSource, List<TopDomainsResult>>(
                         (dataSource) -> userActivityData.getRecentDomains(dataSource, TOP_DOMAINS_COUNT),
-                        (result) -> {
+                        (result) -> recentDomainsTable.showDataFetchResult(result)),
-                            showResultWithModuleCheck(recentDomainsTable, result,
-                                    IngestModuleCheckUtil.RECENT_ACTIVITY_FACTORY,
-                                    IngestModuleCheckUtil.RECENT_ACTIVITY_MODULE_NAME);
-                        }),
                 // top web searches query
                 new DataFetchComponents<DataSource, List<TopWebSearchResult>>(
                         (dataSource) -> userActivityData.getMostRecentWebSearches(dataSource, TOP_SEARCHES_COUNT),
-                        (result) -> {
+                        (result) -> topWebSearchesTable.showDataFetchResult(result)),
-                            showResultWithModuleCheck(topWebSearchesTable, result,
-                                    IngestModuleCheckUtil.RECENT_ACTIVITY_FACTORY,
-                                    IngestModuleCheckUtil.RECENT_ACTIVITY_MODULE_NAME);
-                        }),
                 // top devices query
                 new DataFetchComponents<DataSource, List<TopDeviceAttachedResult>>(
                         (dataSource) -> userActivityData.getRecentDevices(dataSource, TOP_DEVICES_COUNT),
-                        (result) -> {
+                        (result) -> topDevicesAttachedTable.showDataFetchResult(result)),
-                            showResultWithModuleCheck(topDevicesAttachedTable, result,
-                                    IngestModuleCheckUtil.RECENT_ACTIVITY_FACTORY,
-                                    IngestModuleCheckUtil.RECENT_ACTIVITY_MODULE_NAME);
-                        }),
                 // top accounts query
                 new DataFetchComponents<DataSource, List<TopAccountResult>>(
                         (dataSource) -> userActivityData.getRecentAccounts(dataSource, TOP_ACCOUNTS_COUNT),
-                        (result) -> {
+                        (result) -> topAccountsTable.showDataFetchResult(result))
-                            showResultWithModuleCheck(topAccountsTable, result,
-                                    ANDROID_FACTORY,
-                                    ANDROID_MODULE_NAME);
-                        })
         );
 
         initComponents();

Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/Bundle.properties

@@ -31,8 +31,6 @@ EmbeddedFileExtractorIngestModule.ImageExtractor.pptxContainer.init.err=Pptx con
 EmbeddedFileExtractorIngestModule.ImageExtractor.xlsContainer.init.err=Xls container could not be initialized while reading: {0}
 EmbeddedFileExtractorIngestModule.ImageExtractor.xlsxContainer.init.err=Xlsx container could not be initialized while reading: {0}
 EmbeddedFileExtractorIngestModule.ImageExtractor.extractImage.addToDB.exception.msg=Unable to add the derived files to the database.
-EmbeddedFileExtractorIngestModule.ImageExtractor.getOutputFolderPath.exception.msg=Could not get path for image extraction from Abstract File: {0}
-EmbeddedFileExtractorIngestModule.ImageExtractor.getOutputFolderPath.exception.msg=Could not get path for image extraction from Abstract File: {0}
 EmbeddedFileExtractorIngestModule.ArchiveExtractor.UnpackStream.write.noSpace.msg=Unable to write content to disk. Not enough space.
 SevenZipContentReadStream.seek.exception.invalidOrigin=Invalid origin {0}
 SevenZipContentReadStream.read.exception.errReadStream=Error reading stream

Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/Bundle.properties-MERGED

@@ -1,5 +1,6 @@
 CannotCreateOutputFolder=Unable to create output folder.
 CannotRunFileTypeDetection=Unable to run file type detection.
+EmbeddedFileExtractor_make_output_dir_err=Failed to create module output directory for Embedded File Extractor
 EmbeddedFileExtractorIngestModule.ArchiveExtractor.moduleDesc.text=Extracts embedded files (doc, docx, ppt, pptx, xls, xlsx, zip, rar, arj, 7z, gzip, bzip2, tar), schedules them for ingestion, and populates the directory tree with them.
 EmbeddedFileExtractorIngestModule.ArchiveExtractor.moduleName=Embedded File Extractor
 EmbeddedFileExtractorIngestModule.NoOpenCase.errMsg=No open case available.
@@ -43,8 +44,6 @@ EmbeddedFileExtractorIngestModule.ImageExtractor.pptxContainer.init.err=Pptx con
 EmbeddedFileExtractorIngestModule.ImageExtractor.xlsContainer.init.err=Xls container could not be initialized while reading: {0}
 EmbeddedFileExtractorIngestModule.ImageExtractor.xlsxContainer.init.err=Xlsx container could not be initialized while reading: {0}
 EmbeddedFileExtractorIngestModule.ImageExtractor.extractImage.addToDB.exception.msg=Unable to add the derived files to the database.
-EmbeddedFileExtractorIngestModule.ImageExtractor.getOutputFolderPath.exception.msg=Could not get path for image extraction from Abstract File: {0}
-EmbeddedFileExtractorIngestModule.ImageExtractor.getOutputFolderPath.exception.msg=Could not get path for image extraction from Abstract File: {0}
 EmbeddedFileExtractorIngestModule.ArchiveExtractor.UnpackStream.write.noSpace.msg=Unable to write content to disk. Not enough space.
 SevenZipContentReadStream.seek.exception.invalidOrigin=Invalid origin {0}
 SevenZipContentReadStream.read.exception.errReadStream=Error reading stream

Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/Bundle_ja.properties

@@ -22,7 +22,6 @@ EmbeddedFileExtractorIngestModule.ArchiveExtractor.unpack.warnMsg.zipBomb=ZIP\u7
 EmbeddedFileExtractorIngestModule.ImageExtractor.docContainer.init.err=\u8aad\u307f\u53d6\u308a\u4e2d\u306bDOC\u30b3\u30f3\u30c6\u30ca\u30fc\u3092\u521d\u671f\u5316\u3067\u304d\u307e\u305b\u3093\u3067\u3057\u305f 
 EmbeddedFileExtractorIngestModule.ImageExtractor.docxContainer.init.err=\u6b21\u3092\u8aad\u307f\u53d6\u308a\u4e2d\u306bDOCX\u30b3\u30f3\u30c6\u30ca\u30fc\u3092\u521d\u671f\u5316\u3067\u304d\u307e\u305b\u3093\u3067\u3057\u305f\: {0}
 EmbeddedFileExtractorIngestModule.ImageExtractor.extractImage.addToDB.exception.msg=\u6d3e\u751f\u30d5\u30a1\u30a4\u30eb\u3092\u30c7\u30fc\u30bf\u30d9\u30fc\u30b9\u306b\u8ffd\u52a0\u3067\u304d\u307e\u305b\u3093.
-EmbeddedFileExtractorIngestModule.ImageExtractor.getOutputFolderPath.exception.msg=\u6b21\u306e\u62bd\u8c61\u30d5\u30a1\u30a4\u30eb\u304b\u3089\u306e\u30a4\u30e1\u30fc\u30b8\u62bd\u51fa\u30d1\u30b9\u3092\u53d6\u5f97\u3067\u304d\u307e\u305b\u3093\u3067\u3057\u305f\: {0}
 EmbeddedFileExtractorIngestModule.ImageExtractor.pptContainer.init.err=\u6b21\u3092\u8aad\u307f\u53d6\u308a\u4e2d\u306bPPT\u30b3\u30f3\u30c6\u30ca\u30fc\u3092\u521d\u671f\u5316\u3067\u304d\u307e\u305b\u3093\u3067\u3057\u305f\: {0}
 EmbeddedFileExtractorIngestModule.ImageExtractor.pptxContainer.init.err=\u8aad\u307f\u53d6\u308a\u4e2d\u306bPPTX\u30b3\u30f3\u30c6\u30ca\u30fc\u3092\u521d\u671f\u5316\u3067\u304d\u307e\u305b\u3093\u3067\u3057\u305f\: {0}
 EmbeddedFileExtractorIngestModule.ImageExtractor.xlsContainer.init.err=\u6b21\u3092\u8aad\u307f\u53d6\u308a\u4e2d\u306bXLS\u30b3\u30f3\u30c6\u30ca\u30fc\u3092\u521d\u671f\u5316\u3067\u304d\u307e\u305b\u3093\u3067\u3057\u305f\: {0}

Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/DocumentEmbeddedContentExtractor.java

@@ -65,6 +65,7 @@ import org.sleuthkit.autopsy.coreutils.Logger;
 import org.sleuthkit.autopsy.ingest.IngestJobContext;
 import org.sleuthkit.autopsy.ingest.IngestServices;
 import org.sleuthkit.autopsy.ingest.ModuleContentEvent;
+import org.sleuthkit.autopsy.modules.embeddedfileextractor.FileTaskExecutor.FileTaskFailedException;
 import org.sleuthkit.autopsy.modules.filetypeid.FileTypeDetector;
 import org.sleuthkit.datamodel.AbstractFile;
 import org.sleuthkit.datamodel.EncodedFileOutputStream;
@@ -87,6 +88,7 @@ class DocumentEmbeddedContentExtractor {
     private String parentFileName;
     private final String UNKNOWN_IMAGE_NAME_PREFIX = "image_"; //NON-NLS
     private final FileTypeDetector fileTypeDetector;
+    private final FileTaskExecutor fileTaskExecutor;
 
     private String moduleDirRelative;
     private String moduleDirAbsolute;
@@ -121,7 +123,7 @@ class DocumentEmbeddedContentExtractor {
     }
     private SupportedExtractionFormats abstractFileExtractionFormat;
 
-    DocumentEmbeddedContentExtractor(IngestJobContext context, FileTypeDetector fileTypeDetector, String moduleDirRelative, String moduleDirAbsolute) throws NoCurrentCaseException {
+    DocumentEmbeddedContentExtractor(IngestJobContext context, FileTypeDetector fileTypeDetector, String moduleDirRelative, String moduleDirAbsolute, FileTaskExecutor fileTaskExecutor) throws NoCurrentCaseException {
 
         this.fileManager = Case.getCurrentCaseThrows().getServices().getFileManager();
         this.services = IngestServices.getInstance();
@@ -129,6 +131,7 @@ class DocumentEmbeddedContentExtractor {
         this.fileTypeDetector = fileTypeDetector;
         this.moduleDirRelative = moduleDirRelative;
         this.moduleDirAbsolute = moduleDirAbsolute;
+        this.fileTaskExecutor = fileTaskExecutor;
     }
 
     /**
@@ -167,16 +170,23 @@ class DocumentEmbeddedContentExtractor {
         this.parentFileName = utf8SanitizeFileName(EmbeddedFileExtractorIngestModule.getUniqueName(abstractFile));
 
         // Skip files that already have been unpacked.
+        /*
+         * TODO (Jira-7145): Is the logic of this check correct? Also note that
+         * this suspect code used to have a bug in that makeOutputFolder() was
+         * called, so the directory was always created here if it did not exist,
+         * making this check only a call to AbstractFile.hasChildren() in
+         * practice.
+         */
         try {
             if (abstractFile.hasChildren()) {
                 //check if local unpacked dir exists
-                if (new File(getOutputFolderPath(parentFileName)).exists()) {
+                File outputFolder = Paths.get(moduleDirAbsolute, parentFileName).toFile();
-                    LOGGER.log(Level.INFO, "File already has been processed as it has children and local unpacked file, skipping: {0}", abstractFile.getName()); //NON-NLS
+                if (fileTaskExecutor.exists(outputFolder)) {
                     return;
                 }
             }
-        } catch (TskCoreException e) {
+        } catch (TskCoreException | FileTaskExecutor.FileTaskFailedException | InterruptedException e) {
-            LOGGER.log(Level.SEVERE, String.format("Error checking if file already has been processed, skipping: %s", parentFileName), e); //NON-NLS
+            LOGGER.log(Level.SEVERE, String.format("Error checking if %s (objID = %d) has already has been processed, skipping", abstractFile.getName(), abstractFile.getId()), e); //NON-NLS
             return;
         }
 
@@ -299,7 +309,7 @@ class DocumentEmbeddedContentExtractor {
             return null;
         }
 
-        String outputFolderPath;
+        Path outputFolderPath;
         if (listOfAllPictures.isEmpty()) {
             return null;
         } else {
@@ -318,7 +328,7 @@ class DocumentEmbeddedContentExtractor {
             } catch (Exception ex) {
                 return null;
             }
-            writeExtractedImage(Paths.get(outputFolderPath, fileName).toString(), data);
+            writeExtractedImage(Paths.get(outputFolderPath.toString(), fileName).toString(), data);
             // TODO Extract more info from the Picture viz ctime, crtime, atime, mtime
             listOfExtractedImages.add(new ExtractedFile(fileName, getFileRelativePath(fileName), picture.getSize()));
             pictureNumber++;
@@ -359,7 +369,7 @@ class DocumentEmbeddedContentExtractor {
 
         // if no images are extracted from the PPT, return null, else initialize
         // the output folder for image extraction.
-        String outputFolderPath;
+        Path outputFolderPath;
         if (listOfAllPictures.isEmpty()) {
             return null;
         } else {
@@ -405,7 +415,7 @@ class DocumentEmbeddedContentExtractor {
             } catch (Exception ex) {
                 return null;
             }
-            writeExtractedImage(Paths.get(outputFolderPath, imageName).toString(), data);
+            writeExtractedImage(Paths.get(outputFolderPath.toString(), imageName).toString(), data);
             listOfExtractedImages.add(new ExtractedFile(imageName, getFileRelativePath(imageName), pictureData.getData().length));
             i++;
         }
@@ -451,7 +461,7 @@ class DocumentEmbeddedContentExtractor {
 
         // if no images are extracted from the PPT, return null, else initialize
         // the output folder for image extraction.
-        String outputFolderPath;
+        Path outputFolderPath;
         if (listOfAllPictures.isEmpty()) {
             return null;
         } else {
@@ -471,7 +481,7 @@ class DocumentEmbeddedContentExtractor {
             } catch (Exception ex) {
                 return null;
             }
-            writeExtractedImage(Paths.get(outputFolderPath, imageName).toString(), data);
+            writeExtractedImage(Paths.get(outputFolderPath.toString(), imageName).toString(), data);
             listOfExtractedImages.add(new ExtractedFile(imageName, getFileRelativePath(imageName), pictureData.getData().length));
             i++;
         }
@@ -487,9 +497,12 @@ class DocumentEmbeddedContentExtractor {
      * @return List of extracted files to be made into derived file instances.
      */
     private List<ExtractedFile> extractEmbeddedContentFromPDF(AbstractFile abstractFile) {
+        Path outputDirectory = getOutputFolderPath(parentFileName);
+        if (outputDirectory == null) {
+            return Collections.emptyList();
+        }
         PDFAttachmentExtractor pdfExtractor = new PDFAttachmentExtractor(parser);
         try {
-            Path outputDirectory = Paths.get(getOutputFolderPath(parentFileName));
             //Get map of attachment name -> location disk.
             Map<String, PDFAttachmentExtractor.NewResourceData> extractedAttachments = pdfExtractor.extract(
                     new ReadContentInputStream(abstractFile), abstractFile.getId(),
@@ -529,25 +542,29 @@ class DocumentEmbeddedContentExtractor {
     }
 
     /**
-     * Gets path to the output folder for file extraction. If the path does not
+     * Gets the path to an output folder for extraction of embedded content from
-     * exist, it is created.
+     * a file. The folder will have the same name as the file name passed in. If
+     * the folder does not exist, it is created.
      *
-     * @param parentFileName name of the abstract file being processed
+     * @param parentFileName The file name.
      *
-     * @return path to the file extraction folder for a given abstract file.
+     * @return The output folder path or null if the folder could not be found
+     *         or created.
      */
-    private String getOutputFolderPath(String parentFileName) {
+    private Path getOutputFolderPath(String parentFileName) {
-        String outputFolderPath = moduleDirAbsolute + File.separator + parentFileName;
+        Path outputFolderPath = Paths.get(moduleDirAbsolute, parentFileName);
-        File outputFilePath = new File(outputFolderPath);
+        try {
-        if (!outputFilePath.exists()) {
+            File outputFolder = outputFolderPath.toFile();
-            try {
+            if (!fileTaskExecutor.exists(outputFolder)) {
-                outputFilePath.mkdirs();
+                if (!fileTaskExecutor.mkdirs(outputFolder)) {
-            } catch (SecurityException ex) {
+                    outputFolderPath = null;
-                LOGGER.log(Level.WARNING, NbBundle.getMessage(this.getClass(), "EmbeddedFileExtractorIngestModule.ImageExtractor.getOutputFolderPath.exception.msg", parentFileName), ex);
+                }
-                return null;
             }
+            return outputFolderPath;
+        } catch (SecurityException | FileTaskFailedException | InterruptedException ex) {
+            LOGGER.log(Level.SEVERE, String.format("Failed to find or create %s", outputFolderPath), ex);
+            return null;
         }
-        return outputFolderPath;
     }
 
     /**
@@ -648,7 +665,7 @@ class DocumentEmbeddedContentExtractor {
         // plain old list after we upgrade to Tika 1.16 or above.
         private final Map<String, ExtractedFile> nameToExtractedFileMap = new HashMap<>();
 
-        public EmbeddedContentExtractor(ParseContext context) {
+        private EmbeddedContentExtractor(ParseContext context) {
             super(context);
         }
 
@@ -682,7 +699,8 @@ class DocumentEmbeddedContentExtractor {
             }
 
             if (name == null) {
-                name = UNKNOWN_IMAGE_NAME_PREFIX + fileCount++;
+                fileCount++;
+                name = UNKNOWN_IMAGE_NAME_PREFIX + fileCount;
             } else {
                 //make sure to select only the file name (not any directory paths
                 //that might be included in the name) and make sure
@@ -701,10 +719,13 @@ class DocumentEmbeddedContentExtractor {
                 }
             }
 
-            File extractedFile = new File(Paths.get(getOutputFolderPath(parentFileName), name).toString());
+            Path outputFolderPath = getOutputFolderPath(parentFileName);
-            byte[] fileData = IOUtils.toByteArray(stream);
+            if (outputFolderPath != null) {
-            writeExtractedImage(extractedFile.getAbsolutePath(), fileData);
+                File extractedFile = new File(Paths.get(outputFolderPath.toString(), name).toString());
-            nameToExtractedFileMap.put(name, new ExtractedFile(name, getFileRelativePath(name), fileData.length));
+                byte[] fileData = IOUtils.toByteArray(stream);
+                writeExtractedImage(extractedFile.getAbsolutePath(), fileData);
+                nameToExtractedFileMap.put(name, new ExtractedFile(name, getFileRelativePath(name), fileData.length));
+            }
         }
 
         /**

Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/EmbeddedFileExtractorIngestModule.java

@@ -1,7 +1,7 @@
 /*
  * Autopsy Forensic Browser
  *
- * Copyright 2015-2018 Basis Technology Corp.
+ * Copyright 2015-2020 Basis Technology Corp.
  * Contact: carrier <at> sleuthkit <dot> org
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
@@ -20,12 +20,11 @@ package org.sleuthkit.autopsy.modules.embeddedfileextractor;
 
 import java.io.File;
 import java.nio.file.Paths;
-import java.util.ArrayList;
+import java.util.HashMap;
-import java.util.Collections;
+import java.util.Map;
-import java.util.Iterator;
 import java.util.concurrent.ConcurrentHashMap;
-import javax.imageio.ImageIO;
+import java.util.logging.Level;
-import javax.imageio.spi.IIORegistry;
+import javax.annotation.concurrent.GuardedBy;
 import org.openide.util.NbBundle;
 import org.sleuthkit.autopsy.casemodule.Case;
 import org.sleuthkit.datamodel.AbstractFile;
@@ -34,10 +33,13 @@ import org.sleuthkit.autopsy.ingest.IngestModule.ProcessResult;
 import org.sleuthkit.autopsy.ingest.IngestJobContext;
 import org.sleuthkit.autopsy.modules.filetypeid.FileTypeDetector;
 import net.sf.sevenzipjbinding.SevenZipNativeInitializationException;
+import org.sleuthkit.autopsy.apputils.ApplicationLoggers;
 import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException;
+import java.util.logging.Logger;
 import org.sleuthkit.autopsy.ingest.FileIngestModuleAdapter;
 import org.sleuthkit.autopsy.ingest.IngestModuleReferenceCounter;
 import org.sleuthkit.autopsy.modules.embeddedfileextractor.SevenZipExtractor.Archive;
+import org.sleuthkit.autopsy.threadutils.TaskRetryUtil;
 
 /**
  * A file level ingest module that extracts embedded files from supported
@@ -52,6 +54,11 @@ import org.sleuthkit.autopsy.modules.embeddedfileextractor.SevenZipExtractor.Arc
 })
 public final class EmbeddedFileExtractorIngestModule extends FileIngestModuleAdapter {
 
+    private static final String TASK_RETRY_STATS_LOG_NAME = "task_retry_stats";
+    private static final Logger taskStatsLogger = ApplicationLoggers.getLogger(TASK_RETRY_STATS_LOG_NAME);
+    private static final Object execMapLock = new Object();
+    @GuardedBy("execMapLock")
+    private static final Map<Long, FileTaskExecutor> fileTaskExecsByJob = new HashMap<>();
     //Outer concurrent hashmap with keys of JobID, inner concurrentHashmap with keys of objectID
     private static final ConcurrentHashMap<Long, ConcurrentHashMap<Long, Archive>> mapOfDepthTrees = new ConcurrentHashMap<>();
     private static final IngestModuleReferenceCounter refCounter = new IngestModuleReferenceCounter();
@@ -68,110 +75,97 @@ public final class EmbeddedFileExtractorIngestModule extends FileIngestModuleAda
     }
 
     @Override
+    @NbBundle.Messages({
+        "EmbeddedFileExtractor_make_output_dir_err=Failed to create module output directory for Embedded File Extractor"
+    })
     public void startUp(IngestJobContext context) throws IngestModuleException {
+        jobId = context.getJobId();
+
         /*
          * Construct absolute and relative paths to the output directory. The
-         * relative path is relative to the case folder, and will be used in the
+         * output directory is a subdirectory of the ModuleOutput folder in the
-         * case database for extracted (derived) file paths. The absolute path
+         * case directory and is named for the module.
-         * is used to write the extracted (derived) files to local storage.
+         *
+         * The absolute path is used to write the extracted (derived) files to
+         * local storage.
+         *
+         * The relative path is relative to the case folder and is used in the
+         * case database for extracted (derived) file paths.
+         *
          */
-        jobId = context.getJobId();
+        Case currentCase = Case.getCurrentCase();
-        String moduleDirRelative = null;
+        String moduleDirAbsolute = Paths.get(currentCase.getModuleDirectory(), EmbeddedFileExtractorModuleFactory.getModuleName()).toString();
-        String moduleDirAbsolute = null;
+        String moduleDirRelative = Paths.get(currentCase.getModuleOutputDirectoryRelativePath(), EmbeddedFileExtractorModuleFactory.getModuleName()).toString();
 
-        try {
+        if (refCounter.incrementAndGet(jobId) == 1) {
-            final Case currentCase = Case.getCurrentCaseThrows();
+
-            moduleDirRelative = Paths.get(currentCase.getModuleOutputDirectoryRelativePath(), EmbeddedFileExtractorModuleFactory.getModuleName()).toString();
+            /*
-            moduleDirAbsolute = Paths.get(currentCase.getModuleDirectory(), EmbeddedFileExtractorModuleFactory.getModuleName()).toString();
+             * Construct a per ingest job executor that will be used for calling
-        } catch (NoCurrentCaseException ex) {
+             * java.io.File methods as tasks with retries. Retries are employed
-            throw new IngestModuleException(Bundle.EmbeddedFileExtractorIngestModule_NoOpenCase_errMsg(), ex);
+             * here due to observed issues with hangs when attempting these
-        }
+             * operations on case directories stored on a certain type of
-        /*
+             * network file system. See the FileTaskExecutor class header docs
-         * Create the output directory.
+             * for more details.
-         */
+             */
-        File extractionDirectory = new File(moduleDirAbsolute);
+            FileTaskExecutor fileTaskExecutor = new FileTaskExecutor(context);
-        if (!extractionDirectory.exists()) {
+            synchronized (execMapLock) {
-            try {
+                fileTaskExecsByJob.put(jobId, fileTaskExecutor);
-                extractionDirectory.mkdirs();
-            } catch (SecurityException ex) {
-                throw new IngestModuleException(Bundle.CannotCreateOutputFolder(), ex);
             }
+
+            try {
+                File extractionDirectory = new File(moduleDirAbsolute);
+                if (!fileTaskExecutor.exists(extractionDirectory)) {
+                    fileTaskExecutor.mkdirs(extractionDirectory);
+                }
+            } catch (FileTaskExecutor.FileTaskFailedException | InterruptedException ex) {
+                /*
+                 * The exception message is localized because ingest module
+                 * start up exceptions are displayed to the user when running
+                 * with the RCP GUI.
+                 */
+                throw new IngestModuleException(Bundle.EmbeddedFileExtractor_make_output_dir_err(), ex);
+            }
+
+            /*
+             * Construct a hash map to keep track of depth in archives while
+             * processing archive files.
+             *
+             * TODO (Jira-7119): A ConcurrentHashMap of ConcurrentHashMaps is
+             * almost certainly the wrong data structure here. ConcurrentHashMap
+             * is intended to efficiently provide snapshots to multiple threads.
+             * A thread may not see the current state.
+             */
+            mapOfDepthTrees.put(jobId, new ConcurrentHashMap<>());
         }
 
-        /*
-         * Construct a file type detector.
-         */
         try {
             fileTypeDetector = new FileTypeDetector();
         } catch (FileTypeDetector.FileTypeDetectorInitException ex) {
             throw new IngestModuleException(Bundle.CannotRunFileTypeDetection(), ex);
         }
+
         try {
-            this.archiveExtractor = new SevenZipExtractor(context, fileTypeDetector, moduleDirRelative, moduleDirAbsolute);
+            archiveExtractor = new SevenZipExtractor(context, fileTypeDetector, moduleDirRelative, moduleDirAbsolute, fileTaskExecsByJob.get(jobId));
         } catch (SevenZipNativeInitializationException ex) {
+            /*
+             * The exception message is localized because ingest module start up
+             * exceptions are displayed to the user when running with the RCP
+             * GUI.
+             */
             throw new IngestModuleException(Bundle.UnableToInitializeLibraries(), ex);
         }
-        if (refCounter.incrementAndGet(jobId) == 1) {
+
-            /*
-             * Construct a concurrentHashmap to keep track of depth in archives
-             * while processing archive files.
-             */
-            mapOfDepthTrees.put(jobId, new ConcurrentHashMap<>());
-            /**
-             * Initialize Java's Image I/O API so that image reading and writing
-             * (needed for image extraction) happens consistently through the
-             * same providers. See JIRA-6951 for more details.
-             */
-            initializeImageIO();
-        }
-        /*
-         * Construct an embedded content extractor for processing Microsoft
-         * Office documents and PDF documents.
-         */
         try {
-            this.documentExtractor = new DocumentEmbeddedContentExtractor(context, fileTypeDetector, moduleDirRelative, moduleDirAbsolute);
+            documentExtractor = new DocumentEmbeddedContentExtractor(context, fileTypeDetector, moduleDirRelative, moduleDirAbsolute, fileTaskExecsByJob.get(jobId));
         } catch (NoCurrentCaseException ex) {
+            /*
+             * The exception message is localized because ingest module start up
+             * exceptions are displayed to the user when running with the RCP
+             * GUI.
+             */
             throw new IngestModuleException(Bundle.EmbeddedFileExtractorIngestModule_UnableToGetMSOfficeExtractor_errMsg(), ex);
         }
     }
-    
-    /**
-     * Initializes the ImageIO API and sorts the providers for
-     * deterministic image reading and writing.
-     */
-    private void initializeImageIO() {
-        ImageIO.scanForPlugins();
-        
-        // Sift through each registry category and sort category providers by
-        // their canonical class name.
-        IIORegistry pluginRegistry = IIORegistry.getDefaultInstance();
-        Iterator<Class<?>> categories = pluginRegistry.getCategories();
-        while(categories.hasNext()) {
-            sortPluginsInCategory(pluginRegistry, categories.next());
-        }
-    }
-    
-    /**
-     * Sorts all ImageIO SPI providers by their class name.
-     */
-    private <T> void sortPluginsInCategory(IIORegistry pluginRegistry, Class<T> category) {
-        Iterator<T> serviceProviderIter = pluginRegistry.getServiceProviders(category, false);
-        ArrayList<T> providers = new ArrayList<>();
-        while (serviceProviderIter.hasNext()) {
-            providers.add(serviceProviderIter.next());
-        }
-        Collections.sort(providers, (first, second) -> {
-            return first.getClass().getCanonicalName().compareToIgnoreCase(second.getClass().getCanonicalName());
-        });
-        for(int i = 0; i < providers.size() - 1; i++) {
-            for(int j = i + 1; j < providers.size(); j++) {
-                // The registry only accepts pairwise orderings. To guarantee a 
-                // total order, all pairs need to be exhausted.
-                pluginRegistry.setOrdering(category, providers.get(i), 
-                        providers.get(j));
-            }
-        }
-    }
 
     @Override
     public ProcessResult process(AbstractFile abstractFile) {
@@ -213,6 +207,12 @@ public final class EmbeddedFileExtractorIngestModule extends FileIngestModuleAda
     public void shutDown() {
         if (refCounter.decrementAndGet(jobId) == 0) {
             mapOfDepthTrees.remove(jobId);
+            FileTaskExecutor fileTaskExecutor;
+            synchronized (execMapLock) {
+                fileTaskExecutor = fileTaskExecsByJob.remove(jobId);
+            }
+            fileTaskExecutor.shutDown();
+            taskStatsLogger.log(Level.INFO, String.format("total tasks: %d, total task timeouts: %d, total task retries: %d, total task failures: %d (ingest job ID = %d)", TaskRetryUtil.getTotalTasksCount(), TaskRetryUtil.getTotalTaskAttemptTimeOutsCount(), TaskRetryUtil.getTotalTaskRetriesCount(), TaskRetryUtil.getTotalFailedTasksCount(), jobId));
         }
     }
 

Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/ExtractArchiveWithPasswordAction.java

@@ -125,7 +125,7 @@ public class ExtractArchiveWithPasswordAction extends AbstractAction {
                     return false;
                 }
                 try {
-                    SevenZipExtractor extractor = new SevenZipExtractor(null, fileTypeDetector, moduleDirRelative, moduleDirAbsolute);
+                    SevenZipExtractor extractor = new SevenZipExtractor(null, fileTypeDetector, moduleDirRelative, moduleDirAbsolute, new FileTaskExecutor(null));
                     done = extractor.unpack(archive, new ConcurrentHashMap<>(), password);
                 } catch (SevenZipNativeInitializationException ex) {
                     IngestServices.getInstance().postMessage(IngestMessage.createWarningMessage(EmbeddedFileExtractorModuleFactory.getModuleName(), "Unable to extract file with password", password));

Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/FileTaskExecutor.java

@@ -0,0 +1,253 @@
+/*
+ * Autopsy Forensic Browser
+ *
+ * Copyright 2020 Basis Technology Corp.
+ * Contact: carrier <at> sleuthkit <dot> org
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.sleuthkit.autopsy.modules.embeddedfileextractor;
+
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import org.sleuthkit.autopsy.apputils.ApplicationLoggers;
+import java.util.logging.Logger;
+import org.sleuthkit.autopsy.ingest.IngestJobContext;
+import org.sleuthkit.autopsy.threadutils.TaskRetryUtil;
+
+/**
+ * An executor that will be used for calling java.io.File methods as tasks with
+ * retries. Retries are employed here due to observed issues with hangs when
+ * attempting these operations on case folders stored on a certain type of
+ * network file system. The problem was that calls to File.exists() and
+ * File.mkDirs() were never returning and ingest jobs on auto ingest nodes were
+ * getting stuck indefinitely (see Jira-6735).
+ *
+ * This solution is based on
+ * https://stackoverflow.com/questions/28279034/java-check-safely-if-a-file-exist-on-a-network-drive.
+ * We are presently limiting it to File methods and not using this technique for
+ * file writes.
+ *
+ * IMPORTANT: Stalled threads that have timed out and been cancelled may never
+ * complete and system resources may eventually be exhausted. However, this was
+ * deemed better than having an auto ingest node hang for nineteen days, as in
+ * the Jira story.
+ */
+class FileTaskExecutor {
+
+    private static final int MIN_THREADS_IN_POOL = 4;
+    private static final int MAX_THREADS_IN_POOL = Integer.MAX_VALUE; // Effectively no limit
+    private static final String FILE_IO_TASK_THREAD_NAME = "file-io-executor-task-%d";
+    private static final String FILE_EXISTS_TASK_DESC_FMT_STR = "Checking if %s already exists";
+    private static final String MKDIRS_TASK_DESC_FMT_STR = "Making directory %s";
+    private static final String NEW_FILE_TASK_DESC_FMT_STR = "Creating new file %s";
+    private static final String FILE_OPS_LOG_NAME = "efe_file_ops";
+    private static final Logger logger = ApplicationLoggers.getLogger(FILE_OPS_LOG_NAME);
+    private final ScheduledThreadPoolExecutor executor;
+    private final TaskTerminator terminator;
+
+    /**
+     * Constructs an executor that will be used for calling java.io.File methods
+     * as tasks with retries. Retries are employed here due to observed issues
+     * with hangs when attempting these operations on case folders stored on a
+     * certain type of network file system. The problem was that calls to
+     * File.exists() and File.mkDirs() were never returning and ingest jobs on
+     * auto ingest nodes were getting stuck indefinitely (see Jira-6735).
+     *
+     * This solution is based on
+     * https://stackoverflow.com/questions/28279034/java-check-safely-if-a-file-exist-on-a-network-drive.
+     * We are presently limiting it to File methods and not using this technique
+     * for file writes.
+     *
+     * IMPORTANT: Stalled threads that have timed out and been cancelled may
+     * never complete and system resources may eventually be exhausted. However,
+     * this was deemed better than having an auto ingest node hang for nineteen
+     * days, as in the Jira story.
+     *
+     * @param context An ingest job context that will be used in a
+     *                TaskRetryUtil.Terminator to cut off attempts to do a file
+     *                I/O operation if the ingest job is cancelled. Optional,
+     *                may be null.
+     */
+    FileTaskExecutor(IngestJobContext context) {
+        executor = new ScheduledThreadPoolExecutor(MIN_THREADS_IN_POOL, new ThreadFactoryBuilder().setNameFormat(FILE_IO_TASK_THREAD_NAME).build());
+        executor.setMaximumPoolSize(MAX_THREADS_IN_POOL);
+        if (context != null) {
+            terminator = new TaskTerminator(context);
+        } else {
+            terminator = null;
+        }
+    }
+
+    /**
+     * Attempts to check whether a given file exists, retrying several times if
+     * necessary.
+     *
+     * @param file The file.
+     *
+     * @return True or false.
+     *
+     * @throws FileTaskFailedException Thrown if the file I/O task could not be
+     *                                 completed.
+     * @throws InterruptedException    Thrown if the file I/O task is
+     *                                 interrupted.
+     */
+    boolean exists(final File file) throws FileTaskFailedException, InterruptedException {
+        Callable<Boolean> task = () -> {
+            return file.exists();
+        };
+        return attemptTask(task, String.format(FILE_EXISTS_TASK_DESC_FMT_STR, file.getPath()));
+    }
+
+    /**
+     * Attempts to create the parent directories for a given file, retrying
+     * several times if necessary.
+     *
+     * @param file The file.
+     *
+     * @return True on success or false on failure.
+     *
+     * @throws FileTaskFailedException Thrown if the file I/O task could not be
+     *                                 completed.
+     * @throws InterruptedException    Thrown if the file I/O task is
+     *                                 interrupted.
+     */
+    boolean mkdirs(final File file) throws FileTaskFailedException, InterruptedException {
+        Callable<Boolean> task = () -> {
+            return file.mkdirs();
+        };
+        return attemptTask(task, String.format(MKDIRS_TASK_DESC_FMT_STR, file.getPath()));
+    }
+
+    /**
+     * Attempts to create a new empty file, retrying several times if necessary.
+     *
+     * @param file The file.
+     *
+     * @return True on success or false on failure.
+     *
+     * @throws FileTaskFailedException Thrown if the file I/O task could not be
+     *                                 completed.
+     * @throws InterruptedException    Thrown if the file I/O task is
+     *                                 interrupted.
+     */
+    boolean createNewFile(final File file) throws FileTaskFailedException, InterruptedException {
+        Callable<Boolean> task = () -> {
+            return file.createNewFile();
+        };
+        return attemptTask(task, String.format(NEW_FILE_TASK_DESC_FMT_STR, file.getPath()));
+    }
+
+    /**
+     * Attempts a java.io.File task with retries.
+     *
+     * @param task     The task.
+     * @param taskDesc A description of the task, used for logging.
+     *
+     * @return True on success or false on failure.
+     *
+     * @throws FileTaskFailedException Thrown if the task could not be
+     *                                 completed.
+     * @throws InterruptedException    Thrown if the task is interrupted.
+     */
+    private boolean attemptTask(Callable<Boolean> task, String taskDesc) throws FileTaskFailedException, InterruptedException {
+        List<TaskRetryUtil.TaskAttempt> attempts = new ArrayList<>();
+        attempts.add(new TaskRetryUtil.TaskAttempt(0L, 10L, TimeUnit.MINUTES));
+        attempts.add(new TaskRetryUtil.TaskAttempt(5L, 10L, TimeUnit.MINUTES));
+        attempts.add(new TaskRetryUtil.TaskAttempt(10L, 10L, TimeUnit.MINUTES));
+        attempts.add(new TaskRetryUtil.TaskAttempt(15L, 10L, TimeUnit.MINUTES));
+        Boolean success = TaskRetryUtil.attemptTask(task, attempts, executor, terminator, logger, taskDesc);
+        if (success == null) {
+            throw new FileTaskFailedException(taskDesc + " failed");
+        }
+        return success;
+    }
+
+    /**
+     * Shuts down this executor.
+     *
+     * IMPORTANT: No attempt is made to wait for tasks to complete and stalled
+     * threads are possible. See class header documentation.
+     */
+    void shutDown() {
+        /*
+         * Not waiting for task terminaton because some tasks may never
+         * terminate. See class doc.
+         */
+        executor.shutdownNow();
+    }
+
+    /**
+     * A TaskRetryUtil.Terminator that uses an ingest job context to cut off
+     * attempts to do a java.io.File operation if the ingest job is cancelled.
+     */
+    private static class TaskTerminator implements TaskRetryUtil.Terminator {
+
+        private final IngestJobContext context;
+
+        /**
+         * Construct a TaskRetryUtil.Terminator that uses an ingest job context
+         * to cut off attempts to do a java.io.File operation if the ingest job
+         * is cancelled.
+         *
+         * @param context The ingest job context.
+         */
+        TaskTerminator(IngestJobContext context) {
+            this.context = context;
+        }
+
+        @Override
+        public boolean stopTaskAttempts() {
+            /*
+             * This works because IngestJobContext.fileIngestIsCancelled() calls
+             * IngestJobPipeline.isCancelled(), which returns the value of a
+             * volatile variable.
+             */
+            return context.fileIngestIsCancelled();
+        }
+    }
+
+    /**
+     * Exception thrown when a java.io.File task failed.
+     */
+    static final class FileTaskFailedException extends Exception {
+
+        private static final long serialVersionUID = 1L;
+
+        /**
+         * Constructs an exception thrown when a java.io.File task failed.
+         *
+         * @param message The exception message.
+         */
+        private FileTaskFailedException(String message) {
+            super(message);
+        }
+
+        /**
+         * Constructs an exception thrown when a java.io.File task failed.
+         *
+         * @param message The exception message.
+         * @param cause   The cause of the exception.
+         */
+        private FileTaskFailedException(String message, Throwable cause) {
+            super(message, cause);
+        }
+    }
+
+}

Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java

@@ -1,7 +1,7 @@
 /*
  * Autopsy Forensic Browser
  *
- * Copyright 2013-2019 Basis Technology Corp.
+ * Copyright 2015-2020 Basis Technology Corp.
  * Contact: carrier <at> sleuthkit <dot> org
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
@@ -22,6 +22,8 @@ import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.nio.charset.Charset;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -60,6 +62,7 @@ import org.sleuthkit.autopsy.ingest.IngestMessage;
 import org.sleuthkit.autopsy.ingest.IngestMonitor;
 import org.sleuthkit.autopsy.ingest.IngestServices;
 import org.sleuthkit.autopsy.ingest.ModuleContentEvent;
+import org.sleuthkit.autopsy.modules.embeddedfileextractor.FileTaskExecutor.FileTaskFailedException;
 import org.sleuthkit.autopsy.modules.filetypeid.FileTypeDetector;
 import org.sleuthkit.datamodel.AbstractFile;
 import org.sleuthkit.datamodel.Blackboard;
@@ -76,6 +79,10 @@ import org.sleuthkit.datamodel.ReadContentInputStream;
 import org.sleuthkit.datamodel.TskCoreException;
 import org.sleuthkit.datamodel.TskData;
 
+/**
+ * An embedded file extractor that uses 7Zip via Java bindings to extract the
+ * contents of an archive file to a directory named for the archive file.
+ */
 class SevenZipExtractor {
 
     private static final Logger logger = Logger.getLogger(SevenZipExtractor.class.getName());
@@ -97,6 +104,7 @@ class SevenZipExtractor {
     private IngestServices services = IngestServices.getInstance();
     private final IngestJobContext context;
     private final FileTypeDetector fileTypeDetector;
+    private final FileTaskExecutor fileTaskExecutor;
 
     private String moduleDirRelative;
     private String moduleDirAbsolute;
@@ -107,10 +115,6 @@ class SevenZipExtractor {
     private int numItems;
     private String currentArchiveName;
 
-    private String getLocalRootAbsPath(String uniqueArchiveFileName) {
-        return moduleDirAbsolute + File.separator + uniqueArchiveFileName;
-    }
-
     /**
      * Enum of mimetypes which support archive extraction
      */
@@ -138,15 +142,35 @@ class SevenZipExtractor {
         // TODO Expand to support more formats after upgrading Tika
     }
 
-    SevenZipExtractor(IngestJobContext context, FileTypeDetector fileTypeDetector, String moduleDirRelative, String moduleDirAbsolute) throws SevenZipNativeInitializationException {
+    /**
+     * Contructs an embedded file extractor that uses 7Zip via Java bindings to
+     * extract the contents of an archive file to a directory named for the
+     * archive file.
+     *
+     * @param context            The ingest job context, if being used as part
+     *                           of an ingest job. Optional, may be null.
+     * @param fileTypeDetector   A file type detector.
+     * @param moduleDirRelative  The relative path to the output directory for
+     *                           the extracted files. Used in the case database
+     *                           for extracted (derived) file paths for the
+     *                           extracted files.
+     * @param moduleDirAbsolute  The absolute path to the output directory for
+     *                           the extracted files.
+     * @param fileIoTaskExecutor A file I/O task executor.
+     *
+     * @throws SevenZipNativeInitializationException If there was an error
+     *                                               initializing the 7Zip Java
+     *                                               bindings.
+     */
+    SevenZipExtractor(IngestJobContext context, FileTypeDetector fileTypeDetector, String moduleDirRelative, String moduleDirAbsolute, FileTaskExecutor fileTaskExecutor) throws SevenZipNativeInitializationException {
         if (!SevenZip.isInitializedSuccessfully()) {
             throw new SevenZipNativeInitializationException("SevenZip has not been previously initialized.");
         }
-        
         this.context = context;
         this.fileTypeDetector = fileTypeDetector;
         this.moduleDirRelative = moduleDirRelative;
         this.moduleDirAbsolute = moduleDirAbsolute;
+        this.fileTaskExecutor = fileTaskExecutor;
     }
 
     /**
@@ -354,26 +378,36 @@ class SevenZipExtractor {
     }
 
     /**
-     * Query the database and get the list of files which exist for this archive
+     * Queries the case database to get any files already extracted from the
-     * which have already been added to the case database.
+     * given archive. The archive file path is used to find the files by parent
+     * path.
      *
-     * @param archiveFile     the archiveFile to get the files associated with
+     * @param archiveFile     The archive.
-     * @param archiveFilePath the archive file path that must be contained in
+     * @param archiveFilePath The archive file path.
-     *                        the parent_path of files
      *
-     * @return the list of files which already exist in the case database for
+     * @return A list of the files already extracted from the given archive.
-     *         this archive
      *
-     * @throws TskCoreException
+     * @throws TskCoreException          If there is an error querying the case
-     * @throws NoCurrentCaseException
+     *                                   database.
+     * @throws InterruptedException      If checking for the existence of the
+     *                                   extracted file directory is
+     *                                   interrupted.
+     * @throws FileIoTaskFailedException If there is an error checking for the
+     *                                   existence of the extracted file
+     *                                   directory.
      */
-    private List<AbstractFile> getAlreadyExtractedFiles(AbstractFile archiveFile, String archiveFilePath) throws TskCoreException, NoCurrentCaseException {
+    private List<AbstractFile> getAlreadyExtractedFiles(AbstractFile archiveFile, String archiveFilePath) throws TskCoreException, InterruptedException, FileTaskExecutor.FileTaskFailedException {
-        //check if already has derived files, skip
+        /*
-        //check if local unpacked dir exists 
+         * TODO (Jira-7145): Is this logic correct?
-        if (archiveFile.hasChildren() && new File(moduleDirAbsolute, EmbeddedFileExtractorIngestModule.getUniqueName(archiveFile)).exists()) {
+         */
-            return Case.getCurrentCaseThrows().getServices().getFileManager().findFilesByParentPath(getRootArchiveId(archiveFile), archiveFilePath);
+        List<AbstractFile> extractedFiles = new ArrayList<>();
+        File outputDirectory = new File(moduleDirAbsolute, EmbeddedFileExtractorIngestModule.getUniqueName(archiveFile));
+        if (archiveFile.hasChildren() && fileTaskExecutor.exists(outputDirectory)) {
+            Case currentCase = Case.getCurrentCase();
+            FileManager fileManager = currentCase.getServices().getFileManager();
+            extractedFiles.addAll(fileManager.findFilesByParentPath(getRootArchiveId(archiveFile), archiveFilePath));
         }
-        return new ArrayList<>();
+        return extractedFiles;
     }
 
     /**
@@ -388,24 +422,35 @@ class SevenZipExtractor {
     }
 
     /**
-     * Create the local directories if they do not exist for the archive
+     * Creates the root directory for files extracted from this archive. The
+     * directory name is a unique name derived from the archive file name and
+     * its object ID.
      *
-     * @param uniqueArchiveFileName the unique name which corresponds to the
+     * @param uniqueArchiveFileName The unique name of the archive file.
-     *                              archive file in this datasource
+     *
+     * @return True on success, false on failure.
      */
-    private void makeLocalDirectories(String uniqueArchiveFileName) {
+    private boolean makeExtractedFilesDirectory(String uniqueArchiveFileName) {
-        final String localRootAbsPath = getLocalRootAbsPath(uniqueArchiveFileName);
+        boolean success = true;
-        final File localRoot = new File(localRootAbsPath);
+        Path rootDirectoryPath = Paths.get(moduleDirAbsolute, uniqueArchiveFileName);
-        if (!localRoot.exists()) {
+        File rootDirectory = rootDirectoryPath.toFile();
-            localRoot.mkdirs();
+        try {
+            if (!fileTaskExecutor.exists(rootDirectory)) {
+                success = fileTaskExecutor.mkdirs(rootDirectory);
+            }
+        } catch (SecurityException | FileTaskFailedException | InterruptedException ex) {
+            logger.log(Level.SEVERE, String.format("Error creating root extracted files directory %s", rootDirectory), ex); //NON-NLS
+            success = false;
         }
+        return success;
     }
 
     /**
      * Get the path in the archive of the specified item
      *
      * @param archive            - the archive to get the path for
-     * @param inArchiveItemIndex - the item index to help provide uniqueness to the path
+     * @param inArchiveItemIndex - the item index to help provide uniqueness to
+     *                           the path
      * @param archiveFile        - the archive file the item exists in
      *
      * @return a string representing the path to the item in the archive
@@ -453,7 +498,7 @@ class SevenZipExtractor {
         }
         return pathInArchive;
     }
-    
+
     private byte[] getPathBytesInArchive(IInArchive archive, int inArchiveItemIndex, AbstractFile archiveFile) throws SevenZipException {
         return (byte[]) archive.getProperty(inArchiveItemIndex, PropID.PATH_BYTES);
     }
@@ -522,20 +567,18 @@ class SevenZipExtractor {
             unpackSuccessful = false;
             return unpackSuccessful;
         }
+
         try {
             List<AbstractFile> existingFiles = getAlreadyExtractedFiles(archiveFile, archiveFilePath);
             for (AbstractFile file : existingFiles) {
                 statusMap.put(getKeyAbstractFile(file), new ZipFileStatusWrapper(file, ZipFileStatus.EXISTS));
             }
-        } catch (TskCoreException e) {
+        } catch (TskCoreException | FileTaskFailedException | InterruptedException ex) {
-            logger.log(Level.INFO, "Error checking if file already has been processed, skipping: {0}", escapedArchiveFilePath); //NON-NLS
+            logger.log(Level.SEVERE, String.format("Error checking if %s has already been processed, skipping", escapedArchiveFilePath), ex); //NON-NLS
-            unpackSuccessful = false;
-            return unpackSuccessful;
-        } catch (NoCurrentCaseException ex) {
-            logger.log(Level.INFO, "No open case was found while trying to unpack the archive file {0}", escapedArchiveFilePath); //NON-NLS
             unpackSuccessful = false;
             return unpackSuccessful;
         }
+
         parentAr = depthMap.get(archiveFile.getId());
         if (parentAr == null) {
             parentAr = new Archive(0, archiveFile.getId(), archiveFile);
@@ -573,13 +616,8 @@ class SevenZipExtractor {
 
             //setup the archive local root folder
             final String uniqueArchiveFileName = FileUtil.escapeFileName(EmbeddedFileExtractorIngestModule.getUniqueName(archiveFile));
-            try {
+            if (!makeExtractedFilesDirectory(uniqueArchiveFileName)) {
-                makeLocalDirectories(uniqueArchiveFileName);
+                return false;
-            } catch (SecurityException e) {
-                logger.log(Level.SEVERE, "Error setting up output path for archive root: {0}", getLocalRootAbsPath(uniqueArchiveFileName)); //NON-NLS
-                //bail
-                unpackSuccessful = false;
-                return unpackSuccessful;
             }
 
             //initialize tree hierarchy to keep track of unpacked file structure
@@ -648,31 +686,24 @@ class SevenZipExtractor {
                 final String localRelPath = moduleDirRelative + File.separator + uniqueExtractedName;
 
                 //create local dirs and empty files before extracted
-                File localFile = new java.io.File(localAbsPath);
                 //cannot rely on files in top-bottom order
-                if (!localFile.exists()) {
+                File localFile = new File(localAbsPath);
-                    try {
+                boolean localFileExists;
-                        if ((Boolean) inArchive.getProperty(
+                try {
-                                inArchiveItemIndex, PropID.IS_FOLDER)) {
+                    if ((Boolean) inArchive.getProperty(inArchiveItemIndex, PropID.IS_FOLDER)) {
-                            localFile.mkdirs();
+                        localFileExists = findOrCreateDirectory(localFile);
-                        } else {
+                    } else {
-                            localFile.getParentFile().mkdirs();
+                        localFileExists = findOrCreateEmptyFile(localFile);
-                            try {
-                                localFile.createNewFile();
-                            } catch (IOException e) {
-                                logger.log(Level.SEVERE, "Error creating extracted file: "//NON-NLS
-                                                         + localFile.getAbsolutePath(), e);
-                            }
-                        }
-                    } catch (SecurityException e) {
-                        logger.log(Level.SEVERE, "Error setting up output path for unpacked file: {0}", //NON-NLS
-                                pathInArchive); //NON-NLS
-                        //TODO consider bail out / msg to the user
                     }
+                } catch (FileTaskFailedException | InterruptedException ex) {
+                    localFileExists = false;
+                    logger.log(Level.SEVERE, String.format("Error fiding or creating %s", localFile.getAbsolutePath()), ex); //NON-NLS
                 }
+
                 // skip the rest of this loop if we couldn't create the file
                 //continue will skip details from being added to the map
-                if (localFile.exists() == false) {
+                if (!localFileExists) {
+                    logger.log(Level.SEVERE, String.format("Skipping %s because it could not be created", localFile.getAbsolutePath())); //NON-NLS
                     continue;
                 }
 
@@ -802,9 +833,41 @@ class SevenZipExtractor {
                 context.addFilesToJob(unpackedFiles);
             }
         }
+
         return unpackSuccessful;
     }
 
+    /**
+     * Finds or creates a given directory.
+     *
+     * @param directory The directory.
+     *
+     * @return True on success, false on failure.
+     */
+    private boolean findOrCreateDirectory(File directory) throws FileTaskFailedException, InterruptedException {
+        if (!fileTaskExecutor.exists(directory)) {
+            return fileTaskExecutor.mkdirs(directory);
+        } else {
+            return true;
+        }
+    }
+
+    /**
+     * Finds or creates a given file. If the file is created, it will be empty.
+     *
+     * @param file The file.
+     *
+     * @return True on success, false on failure.
+     */
+    private boolean findOrCreateEmptyFile(File file) throws FileTaskFailedException, InterruptedException {
+        if (!fileTaskExecutor.exists(file)) {
+            fileTaskExecutor.mkdirs(file.getParentFile());
+            return fileTaskExecutor.createNewFile(file);
+        } else {
+            return true;
+        }
+    }
+
     private Charset detectFilenamesCharset(List<byte[]> byteDatas) {
         Charset detectedCharset = null;
         CharsetDetector charsetDetector = new CharsetDetector();
@@ -846,6 +909,7 @@ class SevenZipExtractor {
         return Arrays.stream(wrappedExtractionIndices)
                 .mapToInt(Integer::intValue)
                 .toArray();
+
     }
 
     /**
@@ -981,7 +1045,7 @@ class SevenZipExtractor {
          */
         @Override
         public ISequentialOutStream getStream(int inArchiveItemIndex,
-                                              ExtractAskMode mode) throws SevenZipException {
+                ExtractAskMode mode) throws SevenZipException {
 
             this.inArchiveItemIndex = inArchiveItemIndex;
 
@@ -1007,7 +1071,7 @@ class SevenZipExtractor {
                 }
             } catch (IOException ex) {
                 logger.log(Level.WARNING, String.format("Error opening or setting new stream " //NON-NLS
-                                                        + "for archive file at %s", localAbsPath), ex.getMessage()); //NON-NLS
+                        + "for archive file at %s", localAbsPath), ex.getMessage()); //NON-NLS
                 return null;
             }
 
@@ -1039,7 +1103,7 @@ class SevenZipExtractor {
                     : accessTime.getTime() / 1000;
 
             progressHandle.progress(archiveFile.getName() + ": "
-                                    + (String) inArchive.getProperty(inArchiveItemIndex, PropID.PATH),
+                    + (String) inArchive.getProperty(inArchiveItemIndex, PropID.PATH),
                     inArchiveItemIndex);
 
         }
@@ -1189,7 +1253,7 @@ class SevenZipExtractor {
                     return addNode(rootNode, tokens, null);
                 }
             }
-            
+
             return addNode(rootNode, tokens, byteTokens);
         }
 
@@ -1202,7 +1266,7 @@ class SevenZipExtractor {
          *
          * @return
          */
-        private UnpackedNode addNode(UnpackedNode parent, 
+        private UnpackedNode addNode(UnpackedNode parent,
                 List<String> tokenPath, List<byte[]> tokenPathBytes) {
             // we found all of the tokens
             if (tokenPath.isEmpty()) {
@@ -1432,8 +1496,8 @@ class SevenZipExtractor {
             }
 
             void addDerivedInfo(long size,
-                                boolean isFile,
+                    boolean isFile,
-                                long ctime, long crtime, long atime, long mtime, String relLocalPath) {
+                    long ctime, long crtime, long atime, long mtime, String relLocalPath) {
                 this.size = size;
                 this.isFile = isFile;
                 this.ctime = ctime;
@@ -1500,7 +1564,7 @@ class SevenZipExtractor {
                     this.fileNameBytes = Arrays.copyOf(fileNameBytes, fileNameBytes.length);
                 }
             }
-            
+
             byte[] getFileNameBytes() {
                 if (fileNameBytes == null) {
                     return null;

Core/src/org/sleuthkit/autopsy/modules/encryptiondetection/EncryptionDetectionFileIngestModule.java

@@ -324,7 +324,10 @@ final class EncryptionDetectionFileIngestModule extends FileIngestModuleAdapter
                     Database accessDatabase;
                     try {
                         accessDatabase = databaseBuilder.open();
-                    } catch (IOException | BufferUnderflowException | IndexOutOfBoundsException ignored) {
+                    } catch (Exception ex) { // Firewall, see JIRA-7097
+                        logger.log(Level.WARNING, String.format("Unexpected exception "
+                                + "trying to open msaccess database using Jackcess "
+                                + "(name: %s, id: %d)", file.getName(), file.getId()), ex);
                         return passwordProtected; 
                     }
                     /*

Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/ALeappAnalyzerIngestModule.java

@@ -318,7 +318,8 @@ public class ALeappAnalyzerIngestModule implements DataSourceIngestModule {
                 "\"" + aLeappExecutable + "\"", //NON-NLS
                 "-t", aLeappFileSystemType, //NON-NLS
                 "-i", sourceFilePath, //NON-NLS
-                "-o", moduleOutputPath.toString()
+                "-o", moduleOutputPath.toString(),
+                "-w"
         );
         processBuilder.redirectError(moduleOutputPath.resolve("aLeapp_err.txt").toFile());  //NON-NLS
         processBuilder.redirectOutput(moduleOutputPath.resolve("aLeapp_out.txt").toFile());  //NON-NLS

Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/LeappFileProcessor.java

@@ -30,7 +30,9 @@ import java.text.ParseException;
 import java.text.SimpleDateFormat;
 import java.util.List;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.Date;
 import java.util.HashMap;
 import static java.util.Locale.US;
@@ -239,7 +241,6 @@ public final class LeappFileProcessor {
                     Collection<BlackboardAttribute> bbattributes = processReadLine(line, columnNumberToProcess, fileName);
                     if (artifactType == null) {
                         logger.log(Level.SEVERE, "Error trying to process Leapp output files in directory . "); //NON-NLS
-                        
                     }
                     if (!bbattributes.isEmpty() && !blkBoard.artifactExists(dataSource, BlackboardArtifact.ARTIFACT_TYPE.fromID(artifactType.getTypeID()), bbattributes)) {
                         BlackboardArtifact bbartifact = createArtifactWithAttributes(artifactType.getTypeID(), dataSource, bbattributes);
@@ -264,7 +265,17 @@ public final class LeappFileProcessor {
      * @return
      */
     private Collection<BlackboardAttribute> processReadLine(String line, Map<Integer, String> columnNumberToProcess, String fileName) throws IngestModuleException {
-        String[] columnValues = line.split("\\t");
+         
+        String[] columnValues;
+        
+        // Check to see if the 2 values are equal, they may not be equal if there is no corresponding data in the line.
+        // If this happens then adding an empty value(s) for each columnValue where data does not exist
+        Integer maxColumnNumber = Collections.max(columnNumberToProcess.keySet());
+        if (maxColumnNumber > line.split("\\t").length) {
+            columnValues = Arrays.copyOf(line.split("\\t"), maxColumnNumber + 1);
+        } else {
+            columnValues = line.split("\\t");
+        }
 
         Collection<BlackboardAttribute> bbattributes = new ArrayList<BlackboardAttribute>();
 

Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/aleap-artifact-attribute-reference.xml

@@ -225,7 +225,7 @@
     </FileName>
 
     <FileName filename="google play searches.tsv" description="Google Play Searches">
-        <ArtifactName artifactname="TSK_WEB_SEARCH" comment="Google Play Search">
+        <ArtifactName artifactname="TSK_WEB_SEARCH_QUERY" comment="Google Play Search">
             <AttributeName attributename="TSK_DATETIME_ACCESSED" columnName="Timestamp" required="yes" />
             <AttributeName attributename="TSK_PROG_NAME" columnName="Display" required="yes" />
             <AttributeName attributename="TSK_TEXT" columnName="query" required="yes" />
@@ -233,7 +233,7 @@
     </FileName>
 
     <FileName filename="google quick search box.tsv" description="Google quick search box">
-        <ArtifactName artifactname="TSK_WEB_SEARCH" comment="Google Quick Search Search">
+        <ArtifactName artifactname="TSK_WEB_SEARCH_QUERY" comment="Google Quick Search Search">
             <AttributeName attributename="TSK_DATETIME" columnName="File Timestamp" required="yes" />
             <AttributeName attributename="null" columnName="Type" required="no" />
             <AttributeName attributename="TSK_TEXT" columnName="Queries	Response" required="yes" />
@@ -294,8 +294,8 @@
             <AttributeName attributename="TSK_DATETIME" columnName="Date" required="yes"/>
             <AttributeName attributename="null" columnName="MSG ID" required="no"/>
             <AttributeName attributename="TSK_THREAD_ID" columnName="Thread ID" required="yes"/>
-            <AttributeName attributename="null" columnName="Address" required="yes" />
+            <AttributeName attributename="TSK_PHONE_NUMBER_FROM" columnName="Address" required="yes" />
-            <AttributeName attributename="TSK_PHONE_NUMBER_FROM" columnName="Contact ID" required="yes"/>
+            <AttributeName attributename="null" columnName="Contact ID" required="yes"/>
             <AttributeName attributename="TSK_DATETIME_SENT" columnName="Date sent" required="yes"/>
             <AttributeName attributename="TSK_READ_STATUS" columnName="Read" required="yes"/>
             <AttributeName attributename="TSK_TEXT" columnName="Body" required="yes"/>            

Core/src/org/sleuthkit/autopsy/modules/yara/YaraIngestHelper.java

@@ -109,7 +109,10 @@ final class YaraIngestHelper {
     }
 
     /**
-     *
+     * Scan the given AbstractFile for yara rule matches from the rule sets in
+     * the given directory creating a blackboard artifact for each matching
+     * rule.
+     * 
      * @param file                 The Abstract File being processed.
      * @param baseRuleSetDirectory Base directory of the compiled rule sets.
      * @param localFile            Local copy of file.
@@ -138,8 +141,8 @@ final class YaraIngestHelper {
      * Scan the given file byte array for rule matches using the YaraJNIWrapper
      * API.
      *
-     * @param fileBytes
+     * @param fileBytes An array of the file data.
-     * @param ruleSetDirectory
+     * @param ruleSetDirectory Base directory of the compiled rule sets.
      *
      * @return List of rules that match from the given file from the given rule
      *         set. Empty list is returned if no matches where found.
@@ -158,6 +161,17 @@ final class YaraIngestHelper {
         return matchingRules;
     }
 
+    /**
+     * Scan the given file for rules that match from the given rule set directory.
+     * 
+     * @param scanFile Locally stored file to scan.
+     * @param ruleSetDirectory Base directory of the compiled rule sets.
+     * @param timeout YARA Scanner timeout value.
+     * 
+     * @return List of matching rules, if none were found the list will be empty.
+     * 
+     * @throws YaraWrapperException 
+     */
     private static List<String> scanFileForMatch(File scanFile, File ruleSetDirectory, int timeout) throws YaraWrapperException {
         List<String> matchingRules = new ArrayList<>();
 
@@ -228,7 +242,7 @@ final class YaraIngestHelper {
             ProcessBuilder builder = new ProcessBuilder(commandList);
             try {
                 int result = ExecUtil.execute(builder);
-                if(result != 0) {
+                if (result != 0) {
                     throw new IngestModuleException(String.format("Failed to compile Yara rules file %s. Compile error %d", file.toString(), result));
                 }
             } catch (SecurityException | IOException ex) {
@@ -249,7 +263,7 @@ final class YaraIngestHelper {
     private static List<RuleSet> getRuleSetsForNames(List<String> names) {
         List<RuleSet> ruleSetList = new ArrayList<>();
 
-        RuleSetManager manager = new RuleSetManager();
+        RuleSetManager manager = RuleSetManager.getInstance();
         for (RuleSet set : manager.getRuleSetList()) {
             if (names.contains(set.getName())) {
                 ruleSetList.add(set);

Core/src/org/sleuthkit/autopsy/modules/yara/YaraIngestModuleFactory.java

@@ -25,8 +25,10 @@ import org.sleuthkit.autopsy.coreutils.Version;
 import org.sleuthkit.autopsy.ingest.FileIngestModule;
 import org.sleuthkit.autopsy.ingest.IngestModuleFactory;
 import org.sleuthkit.autopsy.ingest.IngestModuleFactoryAdapter;
+import org.sleuthkit.autopsy.ingest.IngestModuleGlobalSettingsPanel;
 import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettings;
 import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettingsPanel;
+import org.sleuthkit.autopsy.modules.yara.ui.YaraGlobalSettingsPanel;
 import org.sleuthkit.autopsy.modules.yara.ui.YaraIngestSettingsPanel;
 
 /**
@@ -63,7 +65,7 @@ public class YaraIngestModuleFactory extends IngestModuleFactoryAdapter {
 
     @Override
     public IngestModuleIngestJobSettingsPanel getIngestJobSettingsPanel(IngestModuleIngestJobSettings settings) {
-        return new YaraIngestSettingsPanel((YaraIngestJobSettings)settings);
+        return new YaraIngestSettingsPanel((YaraIngestJobSettings) settings);
     }
 
     @Override
@@ -89,4 +91,16 @@ public class YaraIngestModuleFactory extends IngestModuleFactoryAdapter {
     static String getModuleName() {
         return Bundle.Yara_Module_Name();
     }
+
+    @Override
+    public boolean hasGlobalSettingsPanel() {
+        return true;
+    }
+
+    @Override
+    public IngestModuleGlobalSettingsPanel getGlobalSettingsPanel() {
+        YaraGlobalSettingsPanel globalOptionsPanel = new YaraGlobalSettingsPanel();
+        globalOptionsPanel.load();
+        return globalOptionsPanel;
+    }
 }

Core/src/org/sleuthkit/autopsy/modules/yara/rules/RuleSet.java

@@ -21,6 +21,7 @@ package org.sleuthkit.autopsy.modules.yara.rules;
 import java.io.File;
 import java.io.Serializable;
 import java.nio.file.Path;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 
@@ -69,7 +70,15 @@ public class RuleSet implements Comparable<RuleSet>, Serializable {
      * @return List of Files in current directory.
      */
     public List<File> getRuleFiles() {
-        return Arrays.asList(path.toFile().listFiles());
+        List<File> fileList = new ArrayList<>();
+        if(path.toFile().exists()) {
+            File[] fileArray = path.toFile().listFiles();
+            if(fileArray != null) {
+                fileList.addAll(Arrays.asList(fileArray));
+            }
+        }
+        
+        return fileList;
     }
 
     @Override

Core/src/org/sleuthkit/autopsy/modules/yara/rules/RuleSetManager.java

@@ -1,7 +1,7 @@
 /*
  * Autopsy Forensic Browser
  *
- * Copyright 2011-2020 Basis Technology Corp.
+ * Copyright 2020 Basis Technology Corp.
  * Contact: carrier <at> sleuthkit <dot> org
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
@@ -18,11 +18,14 @@
  */
 package org.sleuthkit.autopsy.modules.yara.rules;
 
+import java.beans.PropertyChangeListener;
+import java.beans.PropertyChangeSupport;
 import java.io.File;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.List;
+import javax.swing.SwingUtilities;
 import org.sleuthkit.autopsy.coreutils.PlatformUtil;
 
 /**
@@ -34,6 +37,54 @@ public class RuleSetManager {
     private final static String BASE_FOLDER = "yara";
     private final static String RULE_SET_FOLDER = "ruleSets";
 
+    /**
+     * Rule Set Property names.
+     */
+    public final static String RULE_SET_ADDED = "YARARuleSetAdded";
+    public final static String RULE_SET_DELETED = "YARARuleSetDeleted";
+
+    private final PropertyChangeSupport changeSupport;
+
+    private static RuleSetManager instance;
+
+    /**
+     * Private constructor for this singleton.
+     */
+    private RuleSetManager() {
+        changeSupport = new PropertyChangeSupport(this);
+    }
+
+    /**
+     * Returns the instance of this manager class.
+     *
+     * @return
+     */
+    public synchronized static RuleSetManager getInstance() {
+        if (instance == null) {
+            instance = new RuleSetManager();
+        }
+
+        return instance;
+    }
+
+    /**
+     * Adds a property change listener to the manager.
+     *
+     * @param listener Listener to be added.
+     */
+    public static void addPropertyChangeListener(PropertyChangeListener listener) {
+        getInstance().getChangeSupport().addPropertyChangeListener(listener);
+    }
+
+    /**
+     * Remove a property change listener from this manager.
+     *
+     * @param listener Listener to be added.
+     */
+    public void removePropertyChangeListener(PropertyChangeListener listener) {
+        getInstance().getChangeSupport().removePropertyChangeListener(listener);
+    }
+
     /**
      * Create a new Yara rule set with the given set name.
      *
@@ -43,12 +94,11 @@ public class RuleSetManager {
      *
      * @throws RuleSetException RuleSet with given name already exists.
      */
-    public RuleSet createRuleSet(String name) throws RuleSetException {
+    public synchronized RuleSet createRuleSet(String name) throws RuleSetException {
-
+        if (name == null || name.isEmpty()) {
-        if(name == null || name.isEmpty()) {
+            throw new RuleSetException("YARA rule set name cannot be null or empty string");
-            throw new RuleSetException("YARA rule set name cannot be null or empty string" );
         }
-        
+
         if (isRuleSetExists(name)) {
             throw new RuleSetException(String.format("Yara rule set with name %s already exits.", name));
         }
@@ -58,7 +108,42 @@ public class RuleSetManager {
 
         setPath.toFile().mkdir();
 
-        return new RuleSet(name, setPath);
+        RuleSet newSet = new RuleSet(name, setPath);
+
+        SwingUtilities.invokeLater(new Runnable() {
+            @Override
+            public void run() {
+                getChangeSupport().firePropertyChange(RULE_SET_ADDED, null, newSet);
+            }
+        });
+
+        return newSet;
+    }
+
+    /**
+     * Deletes an existing RuleSet.
+     *
+     * @param ruleSet RuleSet to be deleted.
+     *
+     * @throws RuleSetException
+     */
+    public synchronized void deleteRuleSet(RuleSet ruleSet) throws RuleSetException {
+        if (ruleSet == null) {
+            throw new RuleSetException("YARA rule set name cannot be null or empty string");
+        }
+
+        if (!isRuleSetExists(ruleSet.getName())) {
+            throw new RuleSetException(String.format("A YARA rule set with name %s does not exits.", ruleSet.getName()));
+        }
+
+        deleteDirectory(ruleSet.getPath().toFile());
+
+        SwingUtilities.invokeLater(new Runnable() {
+            @Override
+            public void run() {
+                getChangeSupport().firePropertyChange(RULE_SET_DELETED, ruleSet, null);
+            }
+        });
     }
 
     /**
@@ -66,7 +151,7 @@ public class RuleSetManager {
      *
      * @return
      */
-    public List<RuleSet> getRuleSetList() {
+    public synchronized List<RuleSet> getRuleSetList() {
         List<RuleSet> ruleSets = new ArrayList<>();
         Path basePath = getRuleSetPath();
 
@@ -86,7 +171,7 @@ public class RuleSetManager {
      *
      * @return True if the rule set exist.
      */
-    public boolean isRuleSetExists(String name) {
+    public synchronized boolean isRuleSetExists(String name) {
         Path basePath = getRuleSetPath();
         Path setPath = Paths.get(basePath.toString(), name);
 
@@ -110,4 +195,30 @@ public class RuleSetManager {
         return basePath;
     }
 
+    /**
+     * Returns the PropertyChangeSupport instance.
+     *
+     * @return PropertyChangeSupport instance.
+     */
+    private PropertyChangeSupport getChangeSupport() {
+        return changeSupport;
+    }
+
+    /**
+     * Recursively delete the given directory and its children.
+     *
+     * @param directoryToBeDeleted
+     *
+     * @return True if the delete was successful.
+     */
+    private boolean deleteDirectory(File directoryToBeDeleted) {
+        File[] allContents = directoryToBeDeleted.listFiles();
+        if (allContents != null) {
+            for (File file : allContents) {
+                deleteDirectory(file);
+            }
+        }
+        return directoryToBeDeleted.delete();
+    }
+
 }

Core/src/org/sleuthkit/autopsy/modules/yara/ui/Bundle.properties-MERGED

@@ -10,6 +10,7 @@ RuleSetDetailsPanel.setDetailsLabel.text=Set Details
 RuleSetDetailsPanel.openFolderButton.text=Open Folder
 RuleSetPanel.descriptionField.text=This module allows you to find files the match Yara rules. Each set has a list of Yara rule files.  A file need only match one rule in the set to be found.
 RuleSetDetailsPanel.openLabel.text=Place rule files in the set's folder. They will be compiled before use.
+YARA_Global_Settings_Panel_Title=YARA Options
 YaraIngestSettingsPanel.border.title=Select YARA rule sets to enable during ingest:
 YaraIngestSettingsPanel.allFilesButton.text=All Files
 YaraIngestSettingsPanel.allFilesButton.toolTipText=
@@ -21,3 +22,8 @@ YaraRuleSetOptionPanel_badName_msg=Rule set name {0} already exists.\nRule set n
 YaraRuleSetOptionPanel_badName_title=Create Rule Set
 YaraRuleSetOptionPanel_new_rule_set_name_msg=Supply a new unique rule set name:
 YaraRuleSetOptionPanel_new_rule_set_name_title=Rule Set Name
+# {0} - rule set name
+YaraRuleSetOptionPanel_rule_set_delete=Unable to delete the selected YARA rule set {0}.\nRule set may have already been removed.
+# {0} - rule set name
+YaraRuleSetOptionPanel_RuleSet_Missing=The folder for the selected YARA rule set, {0}, no longer exists.
+YaraRuleSetOptionPanel_RuleSet_Missing_title=Folder removed

Core/src/org/sleuthkit/autopsy/modules/yara/ui/RuleSetDetailsPanel.form

@@ -99,10 +99,6 @@
       <Events>
         <EventHandler event="actionPerformed" listener="java.awt.event.ActionListener" parameters="java.awt.event.ActionEvent" handler="refreshButtonActionPerformed"/>
       </Events>
-      <AuxValues>
-        <AuxValue name="JavaCodeGenerator_VariableLocal" type="java.lang.Boolean" value="true"/>
-        <AuxValue name="JavaCodeGenerator_VariableModifier" type="java.lang.Integer" value="0"/>
-      </AuxValues>
       <Constraints>
         <Constraint layoutClass="org.netbeans.modules.form.compat2.layouts.DesignGridBagLayout" value="org.netbeans.modules.form.compat2.layouts.DesignGridBagLayout$GridBagConstraintsDescription">
           <GridBagConstraints gridX="2" gridY="5" gridWidth="1" gridHeight="1" fill="0" ipadX="0" ipadY="0" insetsTop="0" insetsLeft="0" insetsBottom="0" insetsRight="0" anchor="14" weightX="0.0" weightY="0.0"/>

Core/src/org/sleuthkit/autopsy/modules/yara/ui/RuleSetDetailsPanel.java

@@ -20,7 +20,6 @@ package org.sleuthkit.autopsy.modules.yara.ui;
 
 import java.awt.Component;
 import java.awt.Desktop;
-import java.awt.Graphics;
 import java.io.File;
 import java.io.IOException;
 import java.util.List;
@@ -58,7 +57,7 @@ public class RuleSetDetailsPanel extends javax.swing.JPanel {
         fileList.setCellRenderer(new FileRenderer());
         openFolderButton.setEnabled(false);
         scrollPane.setViewportView(fileList);
-        
+        refreshButton.setEnabled(false);
     }
 
     /**
@@ -82,6 +81,7 @@ public class RuleSetDetailsPanel extends javax.swing.JPanel {
         }
 
         openFolderButton.setEnabled(ruleSet != null);
+        refreshButton.setEnabled(ruleSet != null);
     }
     
     /**
@@ -120,7 +120,7 @@ public class RuleSetDetailsPanel extends javax.swing.JPanel {
         openFolderButton = new javax.swing.JButton();
         openLabel = new javax.swing.JLabel();
         scrollPane = new javax.swing.JScrollPane();
-        javax.swing.JButton refreshButton = new javax.swing.JButton();
+        refreshButton = new javax.swing.JButton();
 
         setLayout(new java.awt.GridBagLayout());
 
@@ -223,6 +223,7 @@ public class RuleSetDetailsPanel extends javax.swing.JPanel {
     // Variables declaration - do not modify//GEN-BEGIN:variables
     private javax.swing.JButton openFolderButton;
     private javax.swing.JLabel openLabel;
+    private javax.swing.JButton refreshButton;
     private javax.swing.JScrollPane scrollPane;
     // End of variables declaration//GEN-END:variables
 }

Core/src/org/sleuthkit/autopsy/modules/yara/ui/YaraGlobalSettingsPanel.form

@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+
+<Form version="1.4" maxVersion="1.9" type="org.netbeans.modules.form.forminfo.JPanelFormInfo">
+  <AuxValues>
+    <AuxValue name="FormSettings_autoResourcing" type="java.lang.Integer" value="1"/>
+    <AuxValue name="FormSettings_autoSetComponentName" type="java.lang.Boolean" value="false"/>
+    <AuxValue name="FormSettings_generateFQN" type="java.lang.Boolean" value="true"/>
+    <AuxValue name="FormSettings_generateMnemonicsCode" type="java.lang.Boolean" value="true"/>
+    <AuxValue name="FormSettings_i18nAutoMode" type="java.lang.Boolean" value="true"/>
+    <AuxValue name="FormSettings_layoutCodeTarget" type="java.lang.Integer" value="1"/>
+    <AuxValue name="FormSettings_listenerGenerationStyle" type="java.lang.Integer" value="0"/>
+    <AuxValue name="FormSettings_variablesLocal" type="java.lang.Boolean" value="false"/>
+    <AuxValue name="FormSettings_variablesModifier" type="java.lang.Integer" value="2"/>
+    <AuxValue name="designerSize" type="java.awt.Dimension" value="-84,-19,0,5,115,114,0,18,106,97,118,97,46,97,119,116,46,68,105,109,101,110,115,105,111,110,65,-114,-39,-41,-84,95,68,20,2,0,2,73,0,6,104,101,105,103,104,116,73,0,5,119,105,100,116,104,120,112,0,0,1,44,0,0,1,-112"/>
+  </AuxValues>
+
+  <Layout class="org.netbeans.modules.form.compat2.layouts.DesignBorderLayout"/>
+</Form>

Core/src/org/sleuthkit/autopsy/modules/yara/ui/YaraGlobalSettingsPanel.java

@@ -0,0 +1,85 @@
+/*
+ * Autopsy Forensic Browser
+ *
+ * Copyright 2020 Basis Technology Corp.
+ * Contact: carrier <at> sleuthkit <dot> org
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.sleuthkit.autopsy.modules.yara.ui;
+
+import java.awt.BorderLayout;
+import org.openide.util.NbBundle.Messages;
+import org.sleuthkit.autopsy.corecomponents.OptionsPanel;
+import org.sleuthkit.autopsy.ingest.IngestModuleGlobalSettingsPanel;
+
+/**
+ * YARA global settings panel.
+ */
+public class YaraGlobalSettingsPanel extends IngestModuleGlobalSettingsPanel implements OptionsPanel {
+
+    private static final long serialVersionUID = 1L;
+
+    private final YaraRuleSetOptionPanel panel = new YaraRuleSetOptionPanel();
+
+    @Messages({
+        "YARA_Global_Settings_Panel_Title=YARA Options"
+    })
+    /**
+     * Creates new form YaraGlobalSettingsPanel
+     */
+    public YaraGlobalSettingsPanel() {
+        initComponents();
+        addOptionPanel();
+    }
+
+    @Override
+    public void saveSettings() {
+
+    }
+
+    @Override
+    public void store() {
+
+    }
+
+    @Override
+    public void load() {
+        if (panel != null) {
+            panel.updatePanel();
+        }
+    }
+
+    /**
+     * Add the YaraRuleSetOptionPanel to this panel.
+     */
+    private void addOptionPanel() {
+        add(panel, BorderLayout.CENTER);
+        setName(Bundle.YARA_Global_Settings_Panel_Title());
+    }
+
+    /**
+     * This method is called from within the constructor to initialize the form.
+     * WARNING: Do NOT modify this code. The content of this method is always
+     * regenerated by the Form Editor.
+     */
+    @SuppressWarnings("unchecked")
+    // <editor-fold defaultstate="collapsed" desc="Generated Code">//GEN-BEGIN:initComponents
+    private void initComponents() {
+
+        setLayout(new java.awt.BorderLayout());
+    }// </editor-fold>//GEN-END:initComponents
+
+    // Variables declaration - do not modify//GEN-BEGIN:variables
+    // End of variables declaration//GEN-END:variables
+}

Core/src/org/sleuthkit/autopsy/modules/yara/ui/YaraIngestSettingsPanel.java

@@ -18,6 +18,8 @@
  */
 package org.sleuthkit.autopsy.modules.yara.ui;
 
+import java.beans.PropertyChangeEvent;
+import java.beans.PropertyChangeListener;
 import java.util.ArrayList;
 import java.util.Enumeration;
 import java.util.List;
@@ -49,24 +51,42 @@ public class YaraIngestSettingsPanel extends IngestModuleIngestJobSettingsPanel
         checkboxList = new CheckBoxJList<>();
         scrollPane.setViewportView(checkboxList);
     }
-    
+
+    /**
+     * Constructs a new panel with the given JobSetting objects.
+     * 
+     * @param settings Ingest job settings.
+     */
     public YaraIngestSettingsPanel(YaraIngestJobSettings settings) {
         this();
-        
+
         List<String> setNames = settings.getSelectedRuleSetNames();
-        
+
         checkboxList.setModel(listModel);
         checkboxList.setOpaque(false);
-        RuleSetManager manager = new RuleSetManager();
+        List<RuleSet> ruleSetList = RuleSetManager.getInstance().getRuleSetList();
-        List<RuleSet> ruleSetList = manager.getRuleSetList();
         for (RuleSet set : ruleSetList) {
             RuleSetListItem item = new RuleSetListItem(set);
             item.setChecked(setNames.contains(set.getName()));
             listModel.addElement(item);
         }
-        
+
         allFilesButton.setSelected(!settings.onlyExecutableFiles());
         executableFilesButton.setSelected(settings.onlyExecutableFiles());
+
+        RuleSetManager.addPropertyChangeListener(new PropertyChangeListener() {
+            @Override
+            public void propertyChange(PropertyChangeEvent evt) {
+                switch (evt.getPropertyName()) {
+                    case RuleSetManager.RULE_SET_ADDED:
+                        handleRuleSetAdded((RuleSet) evt.getNewValue());
+                        break;
+                    case RuleSetManager.RULE_SET_DELETED:
+                        handleRuleSetDeleted((RuleSet) evt.getOldValue());
+                        break;
+                }
+            }
+        });
     }
 
     @Override
@@ -84,6 +104,36 @@ public class YaraIngestSettingsPanel extends IngestModuleIngestJobSettingsPanel
         return new YaraIngestJobSettings(selectedRules, executableFilesButton.isSelected());
     }
 
+    /**
+     * Handle the addition of a new Rule Set.
+     * 
+     * @param ruleSet 
+     */
+    private void handleRuleSetAdded(RuleSet ruleSet) {
+        if (ruleSet == null) {
+            return;
+        }
+
+        RuleSetListItem item = new RuleSetListItem(ruleSet);
+        listModel.addElement(item);
+    }
+
+    /**
+     * Handle the removal of the rule set.
+     * 
+     * @param ruleSet 
+     */
+    private void handleRuleSetDeleted(RuleSet ruleSet) {
+        Enumeration<RuleSetListItem> enumeration = listModel.elements();
+        while (enumeration.hasMoreElements()) {
+            RuleSetListItem item = enumeration.nextElement();
+            if (item.getDisplayName().equals(ruleSet.getName())) {
+                listModel.removeElement(item);
+                return;
+            }
+        }
+    }
+
     /**
      * RuleSet wrapper class for Checkbox JList model.
      */

Core/src/org/sleuthkit/autopsy/modules/yara/ui/YaraRuleSetOptionPanel.form

@@ -42,6 +42,14 @@
       <Layout class="org.netbeans.modules.form.compat2.layouts.support.JScrollPaneSupportLayout"/>
       <SubComponents>
         <Container class="javax.swing.JPanel" name="viewportPanel">
+          <Properties>
+            <Property name="minimumSize" type="java.awt.Dimension" editor="org.netbeans.beaninfo.editors.DimensionEditor">
+              <Dimension value="[1000, 127]"/>
+            </Property>
+            <Property name="preferredSize" type="java.awt.Dimension" editor="org.netbeans.beaninfo.editors.DimensionEditor">
+              <Dimension value="[1020, 400]"/>
+            </Property>
+          </Properties>
           <AuxValues>
             <AuxValue name="JavaCodeGenerator_VariableLocal" type="java.lang.Boolean" value="true"/>
             <AuxValue name="JavaCodeGenerator_VariableModifier" type="java.lang.Integer" value="0"/>

Core/src/org/sleuthkit/autopsy/modules/yara/ui/YaraRuleSetOptionPanel.java

@@ -20,13 +20,12 @@ package org.sleuthkit.autopsy.modules.yara.ui;
 
 import java.awt.event.ActionEvent;
 import java.awt.event.ActionListener;
-import java.io.File;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 import javax.swing.JOptionPane;
 import javax.swing.event.ListSelectionEvent;
 import javax.swing.event.ListSelectionListener;
-import org.openide.util.NbBundle;
+import org.openide.util.NbBundle.Messages;
 import org.sleuthkit.autopsy.modules.yara.rules.RuleSet;
 import org.sleuthkit.autopsy.modules.yara.rules.RuleSetException;
 import org.sleuthkit.autopsy.modules.yara.rules.RuleSetManager;
@@ -41,16 +40,12 @@ public class YaraRuleSetOptionPanel extends javax.swing.JPanel {
 
     private static final Logger logger = Logger.getLogger(YaraRuleSetOptionPanel.class.getName());
 
-    private final RuleSetManager manager;
-
     /**
      * Creates new form YaraRuleSetOptionPanel
      */
     public YaraRuleSetOptionPanel() {
         initComponents();
 
-        manager = new RuleSetManager();
-
         ruleSetPanel.addListSelectionListener(new ListSelectionListener() {
             @Override
             public void valueChanged(ListSelectionEvent e) {
@@ -77,25 +72,41 @@ public class YaraRuleSetOptionPanel extends javax.swing.JPanel {
      * Update the panel with the current rule set.
      */
     void updatePanel() {
-        ruleSetPanel.addSetList(manager.getRuleSetList());
+        ruleSetPanel.addSetList(RuleSetManager.getInstance().getRuleSetList());
     }
+    
+    @Messages({
+        "# {0} - rule set name",
+        "YaraRuleSetOptionPanel_RuleSet_Missing=The folder for the selected YARA rule set, {0}, no longer exists.",
+        "YaraRuleSetOptionPanel_RuleSet_Missing_title=Folder removed",
+    })
 
     /**
      * Handle the change in rule set selection. Update the detail panel with the
      * selected rule.
      */
     private void handleSelectionChange() {
-        ruleSetDetailsPanel.setRuleSet(ruleSetPanel.getSelectedRule());
+        RuleSet ruleSet = ruleSetPanel.getSelectedRule();
+
+        if(ruleSet != null && !ruleSet.getPath().toFile().exists()) {
+            ruleSetDetailsPanel.setRuleSet(null);
+            ruleSetPanel.removeRuleSet(ruleSet);
+            JOptionPane.showMessageDialog(this,
+                    Bundle.YaraRuleSetOptionPanel_RuleSet_Missing(ruleSet.getName()),
+                    Bundle.YaraRuleSetOptionPanel_RuleSet_Missing_title(),
+                    JOptionPane.ERROR_MESSAGE);
+        } else {
+            ruleSetDetailsPanel.setRuleSet(ruleSet);
+        }
     }
 
-    @NbBundle.Messages({
+    @Messages({
         "YaraRuleSetOptionPanel_new_rule_set_name_msg=Supply a new unique rule set name:",
         "YaraRuleSetOptionPanel_new_rule_set_name_title=Rule Set Name",
         "# {0} - rule set name",
         "YaraRuleSetOptionPanel_badName_msg=Rule set name {0} already exists.\nRule set names must be unique.",
         "YaraRuleSetOptionPanel_badName_title=Create Rule Set",
-        "YaraRuleSetOptionPanel_badName2_msg=Rule set is invalid.\nRule set names must be non-empty string and unique.",
+        "YaraRuleSetOptionPanel_badName2_msg=Rule set is invalid.\nRule set names must be non-empty string and unique.",})
-    })
     /**
      * Handle the new rule set action. Prompt the user for a rule set name,
      * create the new set and update the rule set list.
@@ -105,16 +116,21 @@ public class YaraRuleSetOptionPanel extends javax.swing.JPanel {
                 Bundle.YaraRuleSetOptionPanel_new_rule_set_name_msg(),
                 Bundle.YaraRuleSetOptionPanel_new_rule_set_name_title());
         
-        if(value == null || value.isEmpty()) {
+        // User hit cancel.
+        if(value == null) {
+            return;
+        }
+
+        if (value.isEmpty()) {
             JOptionPane.showMessageDialog(this,
                     Bundle.YaraRuleSetOptionPanel_badName2_msg(),
                     Bundle.YaraRuleSetOptionPanel_badName_title(),
                     JOptionPane.ERROR_MESSAGE);
             return;
         }
-        
+
         try {
-            ruleSetPanel.addRuleSet(manager.createRuleSet(value));
+            ruleSetPanel.addRuleSet(RuleSetManager.getInstance().createRuleSet(value));
         } catch (RuleSetException ex) {
             JOptionPane.showMessageDialog(this,
                     Bundle.YaraRuleSetOptionPanel_badName_msg(value),
@@ -124,31 +140,29 @@ public class YaraRuleSetOptionPanel extends javax.swing.JPanel {
         }
     }
 
+    @Messages({
+        "# {0} - rule set name",
+        "YaraRuleSetOptionPanel_rule_set_delete=Unable to delete the selected YARA rule set {0}.\nRule set may have already been removed."
+    })
+
     /**
      * Handle the delete rule action. Delete the rule set and update the the
      * rule set list.
      */
     private void handleDeleteRuleSet() {
         RuleSet ruleSet = ruleSetPanel.getSelectedRule();
-        ruleSetPanel.removeRuleSet(ruleSet);
+        if (ruleSet != null) {
-        deleteDirectory(ruleSet.getPath().toFile());
+            try {
-    }
+                RuleSetManager.getInstance().deleteRuleSet(ruleSet);
-
+            } catch (RuleSetException ex) {
-    /**
+                JOptionPane.showMessageDialog(this,
-     * Recursively delete the given directory and its children.
+                        Bundle.YaraRuleSetOptionPanel_rule_set_delete(ruleSet.getName()),
-     *
+                        Bundle.YaraRuleSetOptionPanel_badName_title(),
-     * @param directoryToBeDeleted
+                        JOptionPane.ERROR_MESSAGE);
-     *
+                logger.log(Level.WARNING, String.format("Failed to delete YARA rule set %s", ruleSet.getName()), ex);
-     * @return True if the delete was successful.
-     */
-    private boolean deleteDirectory(File directoryToBeDeleted) {
-        File[] allContents = directoryToBeDeleted.listFiles();
-        if (allContents != null) {
-            for (File file : allContents) {
-                deleteDirectory(file);
             }
+            ruleSetPanel.removeRuleSet(ruleSet);
         }
-        return directoryToBeDeleted.delete();
     }
 
     /**
@@ -172,6 +186,8 @@ public class YaraRuleSetOptionPanel extends javax.swing.JPanel {
 
         scrollPane.setBorder(null);
 
+        viewportPanel.setMinimumSize(new java.awt.Dimension(1000, 127));
+        viewportPanel.setPreferredSize(new java.awt.Dimension(1020, 400));
         viewportPanel.setLayout(new java.awt.GridBagLayout());
 
         separator.setOrientation(javax.swing.SwingConstants.VERTICAL);

Core/src/org/sleuthkit/autopsy/report/modules/html/HTMLReport.java

@@ -390,6 +390,9 @@ public class HTMLReport implements TableReportModule {
                 case TSK_WEB_CATEGORIZATION:
                     in = getClass().getResourceAsStream("/org/sleuthkit/autopsy/images/domain-16.png"); //NON-NLS
                     break;
+                case TSK_YARA_HIT:
+                    in = getClass().getResourceAsStream("/org/sleuthkit/autopsy/images/yara_16.png"); //NON-NLS
+                    break;
                 default:
                     logger.log(Level.WARNING, "useDataTypeIcon: unhandled artifact type = {0}", dataType); //NON-NLS
                     in = getClass().getResourceAsStream("/org/sleuthkit/autopsy/report/images/star.png"); //NON-NLS

Core/src/org/sleuthkit/autopsy/threadutils/TaskRetryUtil.java

@@ -0,0 +1,276 @@
+/*
+ * Autopsy Forensic Browser
+ *
+ * Copyright 2020 Basis Technology Corp.
+ * Contact: carrier <at> sleuthkit <dot> org
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.sleuthkit.autopsy.threadutils;
+
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * A utility that attempts a task a specified number of times with a specified
+ * delay before each attempt and an optional timeout for each attempt. If an
+ * attempt times out, the attempt will be cancelled and the next attempt, if
+ * any, will begin.
+ */
+public class TaskRetryUtil {
+
+    private static final AtomicLong totalTasks = new AtomicLong();
+    private static final AtomicLong totalTaskRetries = new AtomicLong();
+    private static final AtomicLong totalTaskAttemptTimeOuts = new AtomicLong();
+    private static final AtomicLong totalFailedTasks = new AtomicLong();
+
+    /**
+     * Encapsulates the specification of a task attempt for the attemptTask()
+     * utility.
+     */
+    public static class TaskAttempt {
+
+        private final Long delay;
+        private final Long timeOut;
+        private final TimeUnit timeUnit;
+
+        /**
+         * Constructs an object that encapsulates the specification of a task
+         * attempt for the attemptTask() utility. The attempt will have neither
+         * a delay nor a time out.
+         */
+        public TaskAttempt() {
+            this.delay = 0L;
+            this.timeOut = 0L;
+            this.timeUnit = TimeUnit.SECONDS;
+        }
+
+        /**
+         * Constructs an object that encapsulates the specification of a task
+         * attempt for the attemptTask() utility.
+         *
+         * @param delay    The delay before the task should be attempted, may be
+         *                 zero or any positive integer.
+         * @param timeUnit The time unit for the delay before the task should be
+         *                 attempted.
+         */
+        public TaskAttempt(Long delay, TimeUnit timeUnit) {
+            if (delay == null || delay < 0) {
+                throw new IllegalArgumentException(String.format("Argument for delay parameter = %d, must be zero or any positive integer", delay));
+            }
+            if (timeUnit == null) {
+                throw new IllegalArgumentException("Argument for timeUnit parameter is null");
+            }
+            this.delay = delay;
+            this.timeOut = 0L;
+            this.timeUnit = timeUnit;
+        }
+
+        /**
+         * Constructs an object that encapsulates the specification of a task
+         * attempt for the attemptTask() utility.
+         *
+         * @param delay    The delay before the task should be attempted, must
+         *                 be zero or any positive integer.
+         * @param timeOut  The timeout for the task attempt, must be zero or any
+         *                 positive integer.
+         * @param timeUnit The time unit for the delay before the task should be
+         *                 attempted and the time out.
+         */
+        public TaskAttempt(Long delay, Long timeOut, TimeUnit timeUnit) {
+            if (delay == null || delay < 0) {
+                throw new IllegalArgumentException(String.format("Argument for delay parameter = %d, must be zero or any positive integer", delay));
+            }
+            if (timeOut == null || timeOut < 0) {
+                throw new IllegalArgumentException(String.format("Argument for timeOut parameter = %d, must be zero or any positive integer", delay));
+            }
+            if (timeUnit == null) {
+                throw new IllegalArgumentException("Argument for timeUnit parameter is null");
+            }
+            this.delay = delay;
+            this.timeOut = timeOut;
+            this.timeUnit = timeUnit;
+        }
+
+        /**
+         * Gets the optional delay before the task should be attempted, may be
+         * zero. Call getTimeUnit() to get the time unit for the delay.
+         *
+         * @return The delay.
+         */
+        public Long getDelay() {
+            return delay;
+        }
+
+        /**
+         * Gets the the optional timeout for the task attempt, may be zero. Call
+         * getTimeUnit() to get the time unit for the delay.
+         *
+         * @return The timeout.
+         */
+        public Long getTimeout() {
+            return timeOut;
+        }
+
+        /**
+         * Gets the time unit for the optional delay before the task should be
+         * attempted and/or the optional time out.
+         *
+         * @return The time unit.
+         */
+        public TimeUnit getTimeUnit() {
+            return timeUnit;
+        }
+
+    }
+
+    /**
+     * A Terminator can be supplied to the attemptTask() utility. The utility
+     * will query the RetryTerminator before starting each task attempt
+     */
+    public interface Terminator {
+
+        /**
+         * Indicates whether or not the task should be abandoned with no further
+         * attempts.
+         *
+         * @return True or false.
+         */
+        boolean stopTaskAttempts();
+
+    }
+
+    /**
+     * Attempts a task a specified number of times with a specified delay before
+     * each attempt and an optional timeout for each attempt. If an attempt
+     * times out, that particular attempt task will be cancelled.
+     *
+     * @tparam T         The return type of the task.
+     * @param task       The task.
+     * @param attempts   The defining details for each attempt of the task.
+     * @param executor   The scheduled task executor to be used to attempt the
+     *                   task.
+     * @param terminator A task terminator that can be used to stop the task
+     *                   attempts between attempts. Optional, may be null.
+     * @param logger     A logger that will be used to log info messages about
+     *                   each task attempt and for error messages. Optional, may
+     *                   be null.
+     * @param taskDesc   A description of the task for log messages. Optional,
+     *                   may be null.
+     *
+     * @return The task result if the task was completed, null otherwise.
+     *
+     * @throws InterruptedException
+     */
+    public static <T> T attemptTask(Callable<T> task, List<TaskAttempt> attempts, ScheduledThreadPoolExecutor executor, Terminator terminator, Logger logger, String taskDesc) throws InterruptedException {
+        T result = null;
+        String taskDescForLog = taskDesc != null ? taskDesc : "Task";
+        int attemptCounter = 0;
+        if (attempts.size() > 0) {
+            totalTasks.incrementAndGet();
+        }
+        while (result == null && attemptCounter < attempts.size()) {
+            if (terminator != null && terminator.stopTaskAttempts()) {
+                if (logger != null) {
+                    logger.log(Level.WARNING, String.format("Attempts to execute '%s' terminated ", taskDescForLog));
+                }
+                break;
+            }
+            TaskAttempt attempt = attempts.get(attemptCounter);
+            if (logger != null) {
+                logger.log(Level.INFO, String.format("SCHEDULING '%s' (attempt = %d, delay = %d %s, timeout = %d %s)", taskDescForLog, attemptCounter + 1, attempt.getDelay(), attempt.getTimeUnit(), attempt.getTimeout(), attempt.getTimeUnit()));
+            }
+            if (attemptCounter > 0) {
+                totalTaskRetries.incrementAndGet();
+            }
+            ScheduledFuture<T> future = executor.schedule(task, attempt.getDelay(), attempt.getTimeUnit());
+            try {
+                result = future.get(attempt.getDelay() + attempt.getTimeout(), attempt.getTimeUnit());
+            } catch (InterruptedException ex) {
+                if (logger != null) {
+                    logger.log(Level.SEVERE, String.format("Interrupted executing '%s'", taskDescForLog), ex);
+                }
+                throw ex;
+            } catch (ExecutionException ex) {
+                if (logger != null) {
+                    logger.log(Level.SEVERE, String.format("Error executing '%s'", taskDescForLog), ex);
+                }
+            } catch (TimeoutException ex) {
+                if (logger != null) {
+                    logger.log(Level.SEVERE, String.format("Time out executing '%s'", taskDescForLog), ex);
+                }
+                totalTaskAttemptTimeOuts.incrementAndGet();
+                future.cancel(true);
+            }
+            ++attemptCounter;
+        }
+        if (result == null) {
+            if (terminator == null || !terminator.stopTaskAttempts()) {
+                totalFailedTasks.incrementAndGet();
+            }
+        }
+        return result;
+    }
+
+    /**
+     * Returns a count of the total number of tasks submitted to this utility.
+     *
+     * @return The tasks count.
+     */
+    public static long getTotalTasksCount() {
+        return totalTasks.get();
+    }
+
+    /**
+     * Returns a count of the total number of task retry attempts.
+     *
+     * @return The task retries count.
+     */
+    public static long getTotalTaskRetriesCount() {
+        return totalTaskRetries.get();
+    }
+
+    /**
+     * Returns a count of the total number of task attempts that timed out.
+     *
+     * @return The timed out task attempts count.
+     */
+    public static long getTotalTaskAttemptTimeOutsCount() {
+        return totalTaskAttemptTimeOuts.get();
+    }
+
+    /**
+     * Returns a count of the total number of tasks submitted to this utility
+     * that were not able to be completed despite retry attempts.
+     *
+     * @return The failed tasks count.
+     */
+    public static long getTotalFailedTasksCount() {
+        return totalFailedTasks.get();
+    }
+
+    /**
+     * Private contructor to prevent TaskRetryUtil object instantiation.
+     */
+    private TaskRetryUtil() {
+    }
+
+}

Core/src/org/sleuthkit/autopsy/url/analytics/DomainCategorizer.java

@@ -26,7 +26,7 @@ import com.google.common.annotations.Beta;
  * and should have a class annotation of '(at)ServiceProvider(service =
  * DomainCategoryProvider.class)'.
  *
- * NOTE: The @SuppressWarnings("try") on the class is to suppress warnings
+ * NOTE: The (at)SuppressWarnings("try") on the class is to suppress warnings
  * relating to the fact that the close method can throw an InterruptedException
  * since Exception can encompass the InterruptedException. See the following
  * github issue and bugs for more information:

KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java

@@ -1013,16 +1013,34 @@ public class Server {
         "# {0} - colelction name", "Server.deleteCore.exception.msg=Failed to delete Solr colelction {0}",})
     void deleteCollection(String coreName, CaseMetadata metadata) throws KeywordSearchServiceException, KeywordSearchModuleException {
         try {
-            IndexingServerProperties properties = getMultiUserServerProperties(metadata.getCaseDirectory());
+            HttpSolrClient solrServer;
-            HttpSolrClient solrServer = getSolrClient("http://" + properties.getHost() + ":" + properties.getPort() + "/solr");
+            if (metadata.getCaseType() == CaseType.SINGLE_USER_CASE) {
-            connectToSolrServer(solrServer);
+                solrServer = getSolrClient("http://localhost:" + localSolrServerPort + "/solr"); //NON-NLS
-            
+                CoreAdminResponse response = CoreAdminRequest.getStatus(coreName, solrServer);
-            CollectionAdminRequest.Delete deleteCollectionRequest = CollectionAdminRequest.deleteCollection(coreName);
+                if (null != response.getCoreStatus(coreName).get("instanceDir")) {             //NON-NLS
-            CollectionAdminResponse response = deleteCollectionRequest.process(solrServer);
+                    /*
-            if (response.isSuccess()) {
+                     * Send a core unload request to the Solr server, with the
-                logger.log(Level.INFO, "Deleted collection {0}", coreName); //NON-NLS
+                     * parameter set that request deleting the index and the
+                     * instance directory (deleteInstanceDir = true). Note that
+                     * this removes everything related to the core on the server
+                     * (the index directory, the configuration files, etc.), but
+                     * does not delete the actual Solr text index because it is
+                     * currently stored in the case directory.
+                     */
+                    org.apache.solr.client.solrj.request.CoreAdminRequest.unloadCore(coreName, true, true, solrServer);
+                }
             } else {
-                logger.log(Level.WARNING, "Unable to delete collection {0}", coreName); //NON-NLS
+                IndexingServerProperties properties = getMultiUserServerProperties(metadata.getCaseDirectory());
+                solrServer = getSolrClient("http://" + properties.getHost() + ":" + properties.getPort() + "/solr");
+                connectToSolrServer(solrServer);
+
+                CollectionAdminRequest.Delete deleteCollectionRequest = CollectionAdminRequest.deleteCollection(coreName);
+                CollectionAdminResponse response = deleteCollectionRequest.process(solrServer);
+                if (response.isSuccess()) {
+                    logger.log(Level.INFO, "Deleted collection {0}", coreName); //NON-NLS
+                } else {
+                    logger.log(Level.WARNING, "Unable to delete collection {0}", coreName); //NON-NLS
+                }
             }
         } catch (SolrServerException | IOException ex) {
             // We will get a RemoteSolrException with cause == null and detailsMessage

KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java

@@ -238,8 +238,11 @@ public class SolrSearchService implements KeywordSearchService, AutopsyService {
             } catch (KeywordSearchModuleException ex) {
                 throw new KeywordSearchServiceException(Bundle.SolrSearchService_exceptionMessage_unableToDeleteCollection(index.getIndexName()), ex);
             }
-            if (!FileUtil.deleteDir(new File(index.getIndexPath()).getParentFile())) {
+            File indexDir = new File(index.getIndexPath()).getParentFile();
-                throw new KeywordSearchServiceException(Bundle.SolrSearchService_exceptionMessage_failedToDeleteIndexFiles(index.getIndexPath()));
+            if (indexDir.exists()) {
+                if (!FileUtil.deleteDir(indexDir)) {
+                    throw new KeywordSearchServiceException(Bundle.SolrSearchService_exceptionMessage_failedToDeleteIndexFiles(index.getIndexPath()));
+                }
             }
         }
     }

NEWS.txt

@@ -1,36 +1,76 @@
----------------- VERSION 4.17.0  --------------
+---------------- VERSION 4.18.0  --------------
-GUI:
+Keyword Search:
-Expanded the Data Source Summary panel to show recent activity, past cases, analysis results, etc.  Also made this available from the main UI when a data source is selected.
+- A major upgrade from Solr 4 to Solr 8.6.3.  Single user cases continue to use the embedded server.  
-Expanded Discovery UI to support searching for and basic display of web domains.  It collapses the various web artifacts into a single view. 
+Multi-user clusters need to install a new Solr 8 server and can now create a Solr cloud with multiple servers. 
+-- NOTE: Cases created with Autopsy 4.18 cannot be opened by previous versions of Autopsy.  Autopsy 4.18 can open older cases though. 
+- Improved text indexing speed by not doing language detection on unknown file formats and unallocated space. 
+
+Domain Discovery:
+- Added details view to Domain Discovery to show what web-based artifacts are associated with the selected domain. 
+- Updated the Domain Discovery grouping and sorting by options. 
+- Added basic domain categorization for webmail-based domains.
+
+Content Viewers:
+- Built more specialized viewers for web-based artifacts.
+
+Data Source Summary: 
+- Added a “Geolocations” tab that shows what cities the data source was near (based on geolocation data).
+- Added a “Timeline” tab that shows counts of events from the last 30 days the data source was used. 
+- Added navigation buttons to jump from the summary view to the main Autopsy UI (for example to go to the map). 
 
 Ingest Modules:
-Added iOS Analyzer module based on iLEAPP and a subset of its artifacts. 
+- New YARA ingest module to flag files based on regular expression patterns. 
-New Picture Analyzer module that does EXIF extraction and HEIC conversion. HEIC/HEIF images are converted to JPEGs that retain EXIF using ImageMagick (replaces the previous EXIF ingest module).
+- New “Android Analyzer (aLEAPP)” module based on aLEAPP.  Previous “Android Analyzer” also still exists. 
-Added support for the latest version of Edge browser that is based on Chromium into Recent Activity. Other Chromium-based browsers are also supported. 
+- Updated “iOS Analyzer (iLEAPP)” module to create more artifacts and work on disk images. 
-Updated the rules that search Web History artifacts for search queries.  Expanded module to support multiple search engines for ambiguous URLs.
+- Hash Database module will calculate SHA-256 hash in addition to MD5.
-Bluetooth pairing artifacts are created based on RegRipper output.
+- Removed Interesting Item rule that flagged existence of Bitlocker (since it ships with Windows). 
-Prefetch artifacts record the full path of exes.
+- Fixed a major bug in the PhotoRec module that could result in an incorrect file layout if the carved file spanned non-contiguous sectors. 
-PhotoRec module allows you to include or exclude specific file types.
+- Fixed MBOX detection bug in Email module. 
-Upgraded to Tika 1.23.
 
-Performance:
+Reporting:
-Documents are added to Solr in batches instead of one by one. 
+- Attachments from tagged messages are now included in a Portable Case. 
-More efficient queries to find WAL files for SQLite databases. 
-Use a local drive for temp files for multi-user cases instead of the shared folder. 
-
-Command Line
-Command line support for report profiles.
-Restored support for Windows file type association for opening a case in Autopsy by double clicking case metadata (.aut) file.
-Better feedback for command line argument errors.
 
 Misc:
-Updated versions of libvmdk, libvhdi, and libewf.
+- Added support for Ext4 inline data and sparse blocks (via TSK fix).
-Persona UI fixes: Pre-populate account and changed order of New Persona dialog.
+- Updated PostgreSQL JDBC driver to support any recent version of PostgreSQL for multi-user cases and PostgreSQL Central Repository.
-Streaming ingest support added to auto ingest.
+- Added personas to the summary viewer in CVT.
-Recent Activity module processes now use the global timeout.
+- Handling of bad characters in auto ingest manifest files.
-Option to include Autopsy executable in portable case (Windows only.)
+- Assorted small bug fixes.
-Upgraded to NetBeans 11 Rich Client Platform.
+
-Added debug feature to save the stack trace on all threads. 
+
+---------------- VERSION 4.17.0  --------------
+GUI:
+- Expanded the Data Source Summary panel to show recent activity, past cases, analysis results, etc.  Also made this available from the main UI when a data source is selected.
+- Expanded Discovery UI to support searching for and basic display of web domains.  It collapses the various web artifacts into a single view. 
+
+Ingest Modules:
+- Added iOS Analyzer module based on iLEAPP and a subset of its artifacts. 
+- New Picture Analyzer module that does EXIF extraction and HEIC conversion. HEIC/HEIF images are converted to JPEGs that retain EXIF using ImageMagick (replaces the previous EXIF ingest module).
+- Added support for the latest version of Edge browser that is based on Chromium into Recent Activity. Other Chromium-based browsers are also supported. 
+- Updated the rules that search Web History artifacts for search queries.  Expanded module to support multiple search engines for ambiguous URLs.
+- Bluetooth pairing artifacts are created based on RegRipper output.
+- Prefetch artifacts record the full path of exes.
+- PhotoRec module allows you to include or exclude specific file types.
+- Upgraded to Tika 1.23.
+
+Performance:
+- Documents are added to Solr in batches instead of one by one. 
+- More efficient queries to find WAL files for SQLite databases. 
+- Use a local drive for temp files for multi-user cases instead of the shared folder. 
+
+Command Line
+- Command line support for report profiles.
+- Restored support for Windows file type association for opening a case in Autopsy by double clicking case metadata (.aut) file.
+- Better feedback for command line argument errors.
+
+Misc:
+- Updated versions of libvmdk, libvhdi, and libewf.
+- Persona UI fixes: Pre-populate account and changed order of New Persona dialog.
+- Streaming ingest support added to auto ingest.
+- Recent Activity module processes now use the global timeout.
+- Option to include Autopsy executable in portable case (Windows only.)
+- Upgraded to NetBeans 11 Rich Client Platform.
+- Added debug feature to save the stack trace on all threads. 
 
 
 ---------------- VERSION 4.16.0  --------------

RecentActivity/src/org/sleuthkit/autopsy/recentactivity/DefaultDomainCategorizer.java

@@ -45,7 +45,7 @@ import org.sleuthkit.autopsy.url.analytics.DomainCategory;
  * messaging:
  * https://www.raymond.cc/blog/list-of-web-messengers-for-your-convenience/
  *
- * NOTE: The @SuppressWarnings("try") on the class is to suppress warnings
+ * NOTE: The (at)SuppressWarnings("try") on the class is to suppress warnings
  * relating to the fact that the close method can throw an InterruptedException
  * since Exception can encompass the InterruptedException. See the following
  * github issue and bugs for more information:

RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java

@@ -866,6 +866,8 @@ class ExtractRegistry extends Extract {
                                                     parentModuleName, sid));
                                             bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PATH,
                                                     parentModuleName, homeDir));
+                                            
+                                            newArtifacts.add(bbart);
                                         } else {
                                             //add attributes to existing artifact
                                             BlackboardAttribute bbattr = bbart.getAttribute(new BlackboardAttribute.Type(ATTRIBUTE_TYPE.TSK_USER_NAME));
@@ -878,10 +880,10 @@ class ExtractRegistry extends Extract {
                                             if (bbattr == null) {
                                                 bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PATH,
                                                         parentModuleName, homeDir));
-                                            }
+                                            }                                          
                                         }
                                         bbart.addAttributes(bbattributes);
-                                        newArtifacts.add(bbart);
+                                       
                                     } catch (TskCoreException ex) {
                                         logger.log(Level.SEVERE, "Error adding account artifact to blackboard.", ex); //NON-NLS
                                     }

RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractZoneIdentifier.java

@@ -294,6 +294,7 @@ final class ExtractZoneIdentifier extends Extract {
         private static final String REFERRER_URL = "ReferrerUrl"; //NON-NLS
         private static final String HOST_URL = "HostUrl"; //NON-NLS
         private static final String FAMILY_NAME = "LastWriterPackageFamilyName"; //NON-NLS
+        private static String fileName;
 
         private final Properties properties = new Properties(null);
 
@@ -307,6 +308,7 @@ final class ExtractZoneIdentifier extends Extract {
          * @throws IOException
          */
         ZoneIdentifierInfo(AbstractFile zoneFile) throws IOException {
+            fileName = zoneFile.getName();
             properties.load(new ReadContentInputStream(zoneFile));
         }
 
@@ -318,8 +320,13 @@ final class ExtractZoneIdentifier extends Extract {
         private int getZoneId() {
             int zoneValue = -1;
             String value = properties.getProperty(ZONE_ID);
-            if (value != null) {
+            try {
-                zoneValue = Integer.parseInt(value);
+                if (value != null) {
+                    zoneValue = Integer.parseInt(value);
+                }
+            } catch (NumberFormatException ex) {
+               String message = String.format("Unable to parse Zone Id for File %s", fileName); //NON-NLS
+               LOG.log(Level.WARNING, message); 
             }
 
             return zoneValue;

docs/doxygen-dev/footer.html

@@ -1,5 +1,5 @@
 <hr/>
-<p><i>Copyright &#169; 2012-2020 Basis Technology. Generated on $date<br/> 
+<p><i>Copyright &#169; 2012-2021 Basis Technology. Generated on $date<br/> 
 This work is licensed under a
 <a rel="license" href="http://creativecommons.org/licenses/by-sa/3.0/us/">Creative Commons Attribution-Share Alike 3.0 United States License</a>.
 </i></p>

docs/doxygen-user/aleapp.dox

@@ -0,0 +1,20 @@
+/*! \page aleapp_page Android Analyzer (aLEAPP)
+
+[TOC]
+
+\section aleapp_overview Overview
+
+The Android Analyzer ingest module runs aLEAPP (https://github.com/abrignoni/aLEAPP) and converts the results into results that can be viewed in Autopsy.
+
+\section aleapp_config Using the Module
+
+Select the checkbox in the Ingest Modules settings screen to enable the Android Analzyer (ALEAPP) module. The module will run on .tar/.zip files found in a \ref ds_log "logical files data source" or a \ref ds_img "disk image".
+
+\section aleapp_results Seeing Results
+
+Results from the Android Analyzer module will appear in the \ref tree_viewer_page under Results->Extracted Content.
+
+\image html aleapp_main.jpg
+
+
+*/

docs/doxygen-user/communications.dox

@@ -24,7 +24,7 @@ The middle column displays each account, its device and type, and the number of
 Selecting an account in the middle column will bring up the data for that account in the right hand column. There are four tabs that show information about the selected account.
 
 <ul>
-<li> The <b>Summary</b> tab displays counts of how many times the account has appeared in different data types in the top section. In the middle it displays the files this account was found in. If the \ref central_repo_page is enabled, the bottom section will show any other cases that contained this account.
+<li> The <b>Summary</b> tab displays counts of how many times the account has appeared in different data types in the top section. In the middle it displays the files this account was found in. If the \ref central_repo_page is enabled, you can see if any \ref personas_page "personas" are associated with this account and whether any other cases that contained this account.
 
 \image html cvt_summary_tab.png
 

docs/doxygen-user/drone.dox

@@ -1,13 +1,13 @@
-/*! \page drone_page Drone Analyzer
+/*! \page drone_page DJI Drone Analyzer
 
 [TOC]
 
 
 \section drone_overview Overview
 
-The Drone Analyzer module allows you to analyze files from a drone.
+The DJI Drone Analyzer module allows you to analyze files from a drone.
 
-Currently, the Drone Analyzer module works on images obtained from the internal SD card found in the following DJI drone models:
+Currently, the DJI Drone Analyzer module works on images obtained from the internal SD card found in the following DJI drone models:
 - Phantom 3
 - Phantom 4
 - Phantom 4 Pro
@@ -20,7 +20,7 @@ The module will find DAT files and process them using DatCon (https://datfile.ne
 
 \section drone_config Running the Module
 
-To enable the Drone Analyzer ingest module select the checkbox in the \ref ingest_configure "Ingest Modules configuration screen". 
+To enable the DJI Drone Analyzer ingest module select the checkbox in the \ref ingest_configure "Ingest Modules configuration screen". 
 
 \section drone_results Viewing Results
 

docs/doxygen-user/footer.html

@@ -1,5 +1,5 @@
 <hr/>
-<p><i>Copyright &#169; 2012-2020 Basis Technology. Generated on $date<br/> 
+<p><i>Copyright &#169; 2012-2021 Basis Technology. Generated on $date<br/> 
 This work is licensed under a
 <a rel="license" href="http://creativecommons.org/licenses/by-sa/3.0/us/">Creative Commons Attribution-Share Alike 3.0 United States License</a>.
 </i></p>

docs/doxygen-user/hashdb_lookup.dox

@@ -6,7 +6,7 @@
 What Does It Do
 ========
 
-The Hash Lookup Module calculates MD5 hash values for files and looks up hash values in a database to determine if the file is notable, known (in general), included in a specific set of files, or unknown.
+The Hash Lookup Module calculates MD5 hash values for files and looks up hash values in a database to determine if the file is notable, known (in general), included in a specific set of files, or unknown. SHA-256 hashes are also calculated, though these will not be used in hash set lookups.
 
 
 Configuration

docs/doxygen-user/ileapp.dox

@@ -8,7 +8,7 @@ The iOS Analyzer ingest module runs iLEAPP (https://github.com/abrignoni/iLEAPP)
 
 \section ileapp_config Using the Module
 
-Select the checkbox in the Ingest Modules settings screen to enable the IOS Analzyer (iLEAPP) module. In Autopsy 4.17.0 the module only runs on .tar/.zip files found in a \ref ds_log "logical files data source".
+Select the checkbox in the Ingest Modules settings screen to enable the IOS Analzyer (iLEAPP) module. The module will run on .tar/.zip files found in a \ref ds_log "logical files data source" or a \ref ds_img "disk image".
 
 \section ileapp_results Seeing Results
 

docs/doxygen-user/main.dox

@@ -51,6 +51,8 @@ The following topics are available here:
  - \subpage drone_page
  - \subpage gpx_page
  - \subpage ileapp_page
+ - \subpage aleapp_page
+ - \subpage yara_page
 
 - Reviewing the Results
  - \subpage uilayout_page

docs/doxygen-user/yara.dox

@@ -0,0 +1,50 @@
+/*! \page yara_page YARA Analyzer
+
+[TOC]
+
+
+\section yara_overview Overview
+
+The YARA Analyzer module uses a set of rules to search files for textual or binary patterns. YARA was designed for malware analysis but can be used to search for any type of files. For more information on YARA see <a href="https://virustotal.github.io/yara/">https://virustotal.github.io/yara/</a>.
+
+\section yara_config Configuration
+
+To create and edit your rule sets, go to "Tools", "Options" and then select the "YARA" tab. 
+
+\image html yara_options.png
+
+YARA rule sets are stored in folders in the user's Autopsy folder. To create a new rule set, click the "New Set" button in the lower left and enter the name for your new set.
+
+\image html yara_new_rule_set.png
+
+With your new rule set selected, click the "Open Folder" button to go to the newly created rules folder. You can now copy existing YARA files into this folder to include them in the rule set. Information on writing YARA rules can be found <a href="https://yara.readthedocs.io/en/stable/writingrules.html">here</a> and many existing YARA rules can be found through a web search. As a very simple example, we will add this rule to the sample rule set to find files that contain the words "hello" and "world":
+
+\verbatim
+rule HelloWorldRule
+{
+    strings:
+        $part1 = "hello" nocase 
+        $part2 = "world" nocase 
+
+    condition:
+        $part1 and $part2
+}
+\endverbatim
+
+Once you've added your rules to the folder, click the "Refresh File List" button to show them in the options panel.
+
+\section yara_running Running the Module
+
+To enable the YARA Analyzer ingest module select the checkbox in the \ref ingest_configure "Ingest Modules configuration screen". 
+
+\image html yara_ingest_settings.png
+
+Make sure all rule sets you want to run are checked. You can also choose between running on all files or only running on executable files. 
+
+\section yara_results Viewing Results
+
+Results are show in the Results tree under "Extracted Content". 
+
+\image html yara_results.png
+
+*/

docs/doxygen/footer.html

@@ -1,5 +1,5 @@
 <hr/>
-<p><i>Copyright &#169; 2012-2020 Basis Technology. Generated on: $date<br/> 
+<p><i>Copyright &#169; 2012-2021 Basis Technology. Generated on: $date<br/> 
 This work is licensed under a
 <a rel="license" href="http://creativecommons.org/licenses/by-sa/3.0/us/">Creative Commons Attribution-Share Alike 3.0 United States License</a>.
 </i></p>

docs/doxygen/modDev.dox

@@ -44,21 +44,24 @@ If this is your first module, then  you will need to make a NetBeans module. If
 
 To make a NetBeans module:
 - Open the NetBeans IDE and go to File -> New Project.
-- From the list of categories, choose "NetBeans Modules" and then "Module" from the list of "Projects".  Click Next. 
+- From the list of categories on the left, choose "Java with Ant" and then "NetBeans Modules". Next select "Module" from the list of projects on the right and click Next. 
 - In the next panel of the wizard, give the module a name and directory. Select Standalone Module (the default is typically "Add to Suite") so that you build the module as an external module against Autopsy.  You will need to tell NetBeans about the Autopsy platform, so choose the "Manage" button.  Choose the "Add Platform" button and browse to the location of the platform discussed in the previous sections (as a reminder this will either be the location that you installed Autopsy into or where you opened up the ZIP file you created from source).  Click Next.
 - Finally, enter the code base name.  We use the same naming convention as used for naming packages (http://docs.oracle.com/javase/tutorial/java/package/namingpkgs.html).  Press Finish. 
 
 \subsubsection mod_dev_mod_nb_config Configuring the NetBeans Module
 
 After the module is created, you will need to do some further configuration. 
-- Right click on the newly created module and choose "Properties".
+<ul>
-- You will need to configure the module to be dependent on modules from within the Autopsy platform. Go to the "Libraries" area and choose "Add" in the "Module Dependencies" section.  Choose the:
+<li>Right click on the newly created module and choose "Properties".
--- "Autopsy-core" library to get access to the Autopsy services.
+<li>You will need to configure the module to be dependent on modules from within the Autopsy platform. Go to the "Libraries" area and choose "Add" in the "Module Dependencies" section.  Choose the:
--- NetBeans "Lookup API" library so that your module can be discovered by Autopsy. 
+<ul>
-- If you later determine that you need to pull in external JAR files, then you will use the "Wrapped Jar" section to add them in.  
+<li>"Autopsy-Core" library to get access to the Autopsy services.
-- Note, you will also need to come back to this section if you update the platform.  You may need to add a new dependency for the version of the Autopsy-core that comes with the updated platform. 
+<li>NetBeans "Lookup API" library so that your module can be discovered by Autopsy. 
-- Autopsy requires that all modules restart Autopsy after they are installed. Configure your module this way under Build -> Packaging.  Check the box that says Needs Restart on Install.
+</ul>
-
+<li>If you later determine that you need to pull in external JAR files, then you will use the "Wrapped Jar" section to add them in.  
+<li>Note, you will also need to come back to this section if you update the platform.  You may need to add a new dependency for the version of the Autopsy-Core that comes with the updated platform. 
+<li>Autopsy requires that all modules restart Autopsy after they are installed. Configure your module this way under Build -> Packaging.  Check the box that says Needs Restart on Install.
+</ul>
 You now have a NetBeans module that is using Autopsy as its build platform.  That means you will have access to all of the services and utilities that Autopsy provides. 
 
 
@@ -76,7 +79,7 @@ For general NetBeans module information, refer to <a href="http://bits.netbeans.
 \section mod_dev_aut Creating Autopsy Modules
 
 You can now add Autopsy modules into the NetBeans container module.  There are other pages that focus on that and are listed on the main page.  The rest of this document contains info that you will eventually want to come back to though. 
-As you will read in the later sections about the different module types, each Autopsy Module is a java class that extends an interface (the interface depends on the type of module). 
+As you will read in the later sections about the different module types, each Autopsy Module is a Java class that extends an interface (the interface depends on the type of module). 
 
 
 \subsection mod_dev_aut_run1 Running Your Module During Development