Merge branch 'develop' of https://github.com/sleuthkit/autopsy into 5061-BingTranslator

Core/src/org/sleuthkit/autopsy/casemodule/LogicalImagerDSProcessor.java

@@ -186,13 +186,16 @@ public class LogicalImagerDSProcessor implements DataSourceProcessor {
      *                             associated with the data source that is
      *                             intended to be unique across multiple cases
      *                             (e.g., a UUID).
-     * @param imageFilePath        Path to the image file.
+     * @param imagePath            Path to the image file.
+     * @param sectorSize           The sector size (use '0' for autodetect).
      * @param timeZone             The time zone to use when processing dates
      *                             and times for the image, obtained from
      *                             java.util.TimeZone.getID.
-     * @param chunkSize            The maximum size of each chunk of the raw
+     * @param ignoreFatOrphanFiles Whether to parse orphans if the image has a
-     *                             data source as it is divided up into virtual
+     *                             FAT filesystem.
-     *                             unallocated space files.
+     * @param md5                  The MD5 hash of the image, may be null.
+     * @param sha1                 The SHA-1 hash of the image, may be null.
+     * @param sha256               The SHA-256 hash of the image, may be null.
      * @param src                  The source directory of image.
      * @param dest                 The destination directory to copy the source.
      * @param progressMonitor      Progress monitor for reporting progress

Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeNormalizer.java

@@ -2,7 +2,7 @@
  *
  * Autopsy Forensic Browser
  *
- * Copyright 2018 Basis Technology Corp.
+ * Copyright 2019 Basis Technology Corp.
  * Contact: carrier <at> sleuthkit <dot> org
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
@@ -33,12 +33,6 @@ final public class CorrelationAttributeNormalizer {
     //common seperators that may be removed for normalizing
     private static final String SEPERATORS_REGEX = "[\\s-:]";
 
-    /**
-     * This is a utility class - no need for constructing or subclassing, etc...
-     */
-    private CorrelationAttributeNormalizer() {
-    }
-
     /**
      * Normalize the data. Converts text to lower case, and ensures that the
      * data is a valid string of the format expected given the attributeType.
@@ -54,7 +48,7 @@ final public class CorrelationAttributeNormalizer {
             throw new CorrelationAttributeNormalizationException("Attribute type was null.");
         }
         if (data == null) {
-            throw new CorrelationAttributeNormalizationException("Data was null.");
+            throw new CorrelationAttributeNormalizationException("Correlation value was null.");
         }
 
         String trimmedData = data.trim();
@@ -310,4 +304,11 @@ final public class CorrelationAttributeNormalizer {
             throw new CorrelationAttributeNormalizationException("Data provided was not a valid Imsi. : " + data);
         }
     }
+
+    /**
+     * This is a utility class - no need for constructing or subclassing, etc...
+     */
+    private CorrelationAttributeNormalizer() {
+        //Empty constructor
+    }
 }

Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactUtil.java

@@ -1,7 +1,7 @@
 /*
  * Central Repository
  *
- * Copyright 2015-2018 Basis Technology Corp.
+ * Copyright 2015-2019 Basis Technology Corp.
  * Contact: carrier <at> sleuthkit <dot> org
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
@@ -36,15 +36,12 @@ import org.sleuthkit.datamodel.TskCoreException;
 import org.sleuthkit.datamodel.TskData;
 
 /**
- *
+ * Utility class for correlation attributes in the central repository
  */
 public class EamArtifactUtil {
 
     private static final Logger logger = Logger.getLogger(EamArtifactUtil.class.getName());
 
-    public EamArtifactUtil() {
-    }
-
     @Messages({"EamArtifactUtil.emailaddresses.text=Email Addresses"})
     public static String getEmailAddressAttrString() {
         return Bundle.EamArtifactUtil_emailaddresses_text();
@@ -153,13 +150,13 @@ public class EamArtifactUtil {
 
     /**
      * Add a CorrelationAttributeInstance of the specified type to the provided
- list if the artifactForInstance has an Attribute of the given type with a non empty
+     * list if the artifactForInstance has an Attribute of the given type with a
- value.
+     * non empty value.
      *
      * @param eamArtifacts    the list of CorrelationAttributeInstance objects
      *                        which should be added to
-     * @param artifact        the blackboard artifactForInstance which we are creating a
+     * @param artifact        the blackboard artifactForInstance which we are
-                        CorrelationAttributeInstance for
+     *                        creating a CorrelationAttributeInstance for
      * @param bbAttributeType the type of BlackboardAttribute we expect to exist
      *                        for a CorrelationAttributeInstance of this type
      *                        generated from this Blackboard Artifact
@@ -190,7 +187,8 @@ public class EamArtifactUtil {
      * @param correlationType the given type
      * @param value           the artifactForInstance value
      *
-     * @return CorrelationAttributeInstance from details, or null if validation failed or another error occurred
+     * @return CorrelationAttributeInstance from details, or null if validation
+     *         failed or another error occurred
      */
     private static CorrelationAttributeInstance makeCorrelationAttributeInstanceUsingTypeValue(BlackboardArtifact bbArtifact, CorrelationAttributeInstance.Type correlationType, String value) {
         try {
@@ -266,20 +264,19 @@ public class EamArtifactUtil {
             correlationAttributeInstance = EamDb.getInstance().getCorrelationAttributeInstance(type, correlationCase, correlationDataSource, file.getId());
         } catch (EamDbException | CorrelationAttributeNormalizationException ex) {
             logger.log(Level.WARNING, String.format(
-                    "Correlation attribute could not be retrieved for '%s' (id=%d): %s",
+                    "Correlation attribute could not be retrieved for '%s' (id=%d): ",
-                    content.getName(), content.getId(), ex.getMessage()));
+                    content.getName(), content.getId()), ex);
             return null;
         }
         //if there was no correlation attribute found for the item using object_id then check for attributes added with schema 1,1 which lack object_id  
-        if (correlationAttributeInstance == null) {
+        if (correlationAttributeInstance == null && file.getMd5Hash() != null) {
-            String value = file.getMd5Hash();
             String filePath = (file.getParentPath() + file.getName()).toLowerCase();
             try {
-                correlationAttributeInstance = EamDb.getInstance().getCorrelationAttributeInstance(type, correlationCase, correlationDataSource, value, filePath);
+                correlationAttributeInstance = EamDb.getInstance().getCorrelationAttributeInstance(type, correlationCase, correlationDataSource, file.getMd5Hash(), filePath);
             } catch (EamDbException | CorrelationAttributeNormalizationException ex) {
                 logger.log(Level.WARNING, String.format(
-                        "Correlation attribute could not be retrieved for '%s' (id=%d): %s",
+                        "Correlation attribute could not be retrieved for '%s' (id=%d): ",
-                        content.getName(), content.getId(), ex.getMessage()));
+                        content.getName(), content.getId()), ex);
                 return null;
             }
         }
@@ -289,12 +286,12 @@ public class EamArtifactUtil {
 
     /**
      * Create an EamArtifact from the given Content. Will return null if an
- artifactForInstance can not be created - this is not necessarily an error case, it
+     * artifactForInstance can not be created - this is not necessarily an error
- just means an artifactForInstance can't be made. If creation fails due to an error
+     * case, it just means an artifactForInstance can't be made. If creation
- (and not that the file is the wrong type or it has no hash), the error
+     * fails due to an error (and not that the file is the wrong type or it has
- will be logged before returning.
+     * no hash), the error will be logged before returning.
-
+     *
- Does not add the artifactForInstance to the database.
+     * Does not add the artifactForInstance to the database.
      *
      * @param content The content object
      *
@@ -354,7 +351,6 @@ public class EamArtifactUtil {
         if (file == null) {
             return false;
         }
-
         switch (file.getType()) {
             case UNALLOC_BLOCKS:
             case UNUSED_BLOCKS:
@@ -374,4 +370,11 @@ public class EamArtifactUtil {
                 return false;
         }
     }
+
+    /**
+     * Constructs a new EamArtifactUtil
+     */
+    private EamArtifactUtil() {
+        //empty constructor
+    }
 }

Core/src/org/sleuthkit/autopsy/communications/FiltersPanel.java

@@ -275,6 +275,8 @@ final public class FiltersPanel extends JPanel {
     
     /**
      * Populate the devices filter widgets
+     * 
+     * @param initialState 
      */
     private void updateDeviceFilter(boolean initialState) {
         try {
@@ -301,7 +303,7 @@ final public class FiltersPanel extends JPanel {
      * Given a list of subFilters, set the states of the panel controls 
      * accordingly.
      * 
-     * @param subFilters A list of subFilters
+     * @param commFilter Contains a list of subFilters
      */
     public void setFilters(CommunicationsFilter commFilter) {
         List<CommunicationsFilter.SubFilter> subFilters = commFilter.getAndFilters();

Core/src/org/sleuthkit/autopsy/communications/relationships/RelationshipBrowser.java

@@ -61,7 +61,7 @@ public final class RelationshipBrowser extends JPanel implements Lookup.Provider
 
     /**
      * Sets the value of currentSelection and passes the SelectionInfo onto the
-     * currently selected\visible tab.
+     * currently selected or visible tab.
      *
      * @param info Currently selected account nodes
      */

Core/src/org/sleuthkit/autopsy/contentviewers/Bundle.properties

@@ -83,7 +83,7 @@ MediaViewImagePanel.zoomResetButton.text=Reset
 MediaViewImagePanel.zoomTextField.text=
 MediaViewImagePanel.rotationTextField.text=
 MediaViewImagePanel.rotateLeftButton.toolTipText=
-HtmlPanel.showImagesToggleButton.text=Show Images
+HtmlPanel.showImagesToggleButton.text=Download Images
 MediaPlayerPanel.audioSlider.toolTipText=
 MediaPlayerPanel.VolumeIcon.text=\ \ \ \ \ Volume
 MediaPlayerPanel.progressLabel.text=00:00:00/00:00:00

Core/src/org/sleuthkit/autopsy/contentviewers/Bundle.properties-MERGED

@@ -22,7 +22,7 @@ GstVideoPanel.cannotProcFile.err=The media player cannot process this file.
 GstVideoPanel.noOpenCase.errMsg=No open case available.
 Html_text_display_error=The HTML text cannot be displayed, it may not be correctly formed HTML.
 HtmlPanel_showImagesToggleButton_hide=Hide Images
-HtmlPanel_showImagesToggleButton_show=Show Images
+HtmlPanel_showImagesToggleButton_show=Download Images
 HtmlViewer_file_error=This file is missing or unreadable.
 MediaFileViewer.initGst.gstException.msg=Error initializing gstreamer for audio/video viewing and frame extraction capabilities.  Video and audio viewing will be disabled.
 GstVideoPanel.setupVideo.infoLabel.text=Playback of deleted videos is not supported, use an external player.
@@ -145,7 +145,7 @@ MediaViewImagePanel.zoomResetButton.text=Reset
 MediaViewImagePanel.zoomTextField.text=
 MediaViewImagePanel.rotationTextField.text=
 MediaViewImagePanel.rotateLeftButton.toolTipText=
-HtmlPanel.showImagesToggleButton.text=Show Images
+HtmlPanel.showImagesToggleButton.text=Download Images
 MediaPlayerPanel.audioSlider.toolTipText=
 MediaPlayerPanel.VolumeIcon.text=\ \ \ \ \ Volume
 MediaPlayerPanel.progressLabel.text=00:00:00/00:00:00

Core/src/org/sleuthkit/autopsy/contentviewers/FileViewer.java

@@ -77,12 +77,12 @@ public class FileViewer extends javax.swing.JPanel implements DataContentViewer
     }
 
     /**
-     * Get the FileTypeViewer for a given mimetype
+     * Get the FileTypeViewer for a given file
      *
-     * @param mimeType
+     * @param file
      *
      * @return FileTypeViewer, null if no known content viewer supports the
-     *         mimetype
+     *         file
      */
     private FileTypeViewer getSupportingViewer(AbstractFile file) {
         FileTypeViewer viewer = mimeTypeToViewerMap.get(file.getMIMEType());

Core/src/org/sleuthkit/autopsy/contentviewers/HtmlPanel.form

@@ -18,7 +18,7 @@
       <Group type="103" groupAlignment="0" attributes="0">
           <Group type="102" attributes="0">
               <Component id="showImagesToggleButton" min="-2" max="-2" attributes="0"/>
-              <EmptySpace min="0" pref="95" max="32767" attributes="0"/>
+              <EmptySpace min="0" pref="75" max="32767" attributes="0"/>
           </Group>
           <Component id="htmlJPanel" max="32767" attributes="0"/>
       </Group>

Core/src/org/sleuthkit/autopsy/contentviewers/HtmlPanel.java

@@ -18,6 +18,11 @@
  */
 package org.sleuthkit.autopsy.contentviewers;
 
+import java.io.IOException;
+import java.io.StringReader;
+import java.util.List;
+import java.util.logging.Level;
+import java.util.logging.Logger;
 import javafx.application.Platform;
 import javafx.beans.value.ChangeListener;
 import javafx.beans.value.ObservableValue;
@@ -25,8 +30,9 @@ import javafx.concurrent.Worker;
 import javafx.scene.web.WebView;
 import javafx.embed.swing.JFXPanel;
 import javafx.scene.Scene;
-import org.jsoup.Jsoup;
+import net.htmlparser.jericho.Attribute;
-import org.jsoup.nodes.Node;
+import net.htmlparser.jericho.OutputDocument;
+import net.htmlparser.jericho.Source;
 import org.openide.util.NbBundle.Messages;
 import org.w3c.dom.Document;
 import org.w3c.dom.NodeList;
@@ -38,6 +44,7 @@ import org.w3c.dom.events.EventTarget;
 @SuppressWarnings("PMD.SingularField") // UI widgets cause lots of false positives
 final class HtmlPanel extends javax.swing.JPanel {
 
+    private static final Logger logger = Logger.getLogger(HtmlPanel.class.getName());
     private static final long serialVersionUID = 1L;
     private static final String TEXT_TYPE = "text/plain";
     private final JFXPanel jfxPanel = new JFXPanel();
@@ -92,26 +99,76 @@ final class HtmlPanel extends javax.swing.JPanel {
     }
 
     /**
-     * Cleans out input HTML string
+     * Cleans out input HTML string so it will not access resources over the internet
      *
      * @param htmlInString The HTML string to cleanse
      *
      * @return The cleansed HTML String
      */
     private String cleanseHTML(String htmlInString) {
-        org.jsoup.nodes.Document doc = Jsoup.parse(htmlInString);
+        String returnString = "";
-        // remove all 'img' tags.
+        try {
-        doc.select("img").stream().forEach(Node::remove);
+            Source source = new Source(new StringReader(htmlInString));
-        // remove all 'span' tags, these are often images which are ads
+            OutputDocument document = new OutputDocument(source);
-        doc.select("span").stream().forEach(Node::remove);
+            //remove background images
-        return doc.html();
+            source.getAllTags().stream().filter((tag) -> (tag.toString().contains("background-image"))).forEachOrdered((tag) -> {
+                document.remove(tag);
+            });
+            //remove images
+            source.getAllElements("img").forEach((element) -> {
+                document.remove(element.getAllTags());
+            });
+            //remove frames
+            source.getAllElements("frame").forEach((element) -> {
+                document.remove(element.getAllTags());
+            });
+            //remove iframes
+            source.getAllElements("iframe").forEach((element) -> {
+                document.remove(element.getAllTags());
+            });
+            //remove pictures
+            source.getAllElements("picture").forEach((element) -> {
+                document.remove(element.getAllTags());
+            });
+            //remove svg
+            source.getAllElements("svg").forEach((element) -> {
+                document.remove(element.getAllTags());
+            });
+            //remove audio
+            source.getAllElements("audio").forEach((element) -> {
+                document.remove(element.getAllTags());
+            });
+            //remove video
+            source.getAllElements("video").forEach((element) -> {
+                document.remove(element.getAllTags());
+            });
+            //remove tracks
+            source.getAllElements("track").forEach((element) -> {
+                document.remove(element.getAllTags());
+            });
+            //remove embeded external elements
+            source.getAllElements("embed").forEach((element) -> {
+                document.remove(element.getAllTags());
+            });
+            //remove linked elements
+            source.getAllElements("link").forEach((element) -> {
+                document.remove(element.getAllTags());
+            });
+            //remove other URI elements such as input boxes
+            List<Attribute> attributesToRemove = source.getURIAttributes();
+            document.remove(attributesToRemove);
+            returnString = document.toString();
+        } catch (IOException ex) {
+            logger.log(Level.WARNING, "Unable to read html for cleaning out URI elements with Jericho", ex);
+        }
+        return returnString;
     }
 
     /**
      * Refresh the panel to reflect the current show/hide images setting.
      */
     @Messages({
-        "HtmlPanel_showImagesToggleButton_show=Show Images",
+        "HtmlPanel_showImagesToggleButton_show=Download Images",
         "HtmlPanel_showImagesToggleButton_hide=Hide Images",
         "Html_text_display_error=The HTML text cannot be displayed, it may not be correctly formed HTML.",})
     private void refresh() {
@@ -164,7 +221,7 @@ final class HtmlPanel extends javax.swing.JPanel {
             layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
             .addGroup(layout.createSequentialGroup()
                 .addComponent(showImagesToggleButton)
-                .addGap(0, 95, Short.MAX_VALUE))
+                .addGap(0, 75, Short.MAX_VALUE))
             .addComponent(htmlJPanel, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)
         );
         layout.setVerticalGroup(

Core/src/org/sleuthkit/autopsy/core/UserPreferences.java

@@ -504,10 +504,10 @@ public final class UserPreferences {
     /**
      * Get the maximum number of results to display in a result table.
      *
-     * @return Saved value or default (0) which indicates no max.
+     * @return Saved value or default (10,000).
      */
     public static int getResultsTablePageSize() {
-        return preferences.getInt(RESULTS_TABLE_PAGE_SIZE, 0);
+        return preferences.getInt(RESULTS_TABLE_PAGE_SIZE, 10_000);
     }
 
     /**

Core/src/org/sleuthkit/autopsy/corecomponents/Bundle.properties

@@ -216,4 +216,4 @@ DataResultViewerTable.pagesLabel.text=Pages:
 DataResultViewerTable.pageNumLabel.text=
 DataResultViewerTable.pageLabel.text=Page:
 ViewPreferencesPanel.maxResultsLabel.text=Maximum number of Results to show in table:
-ViewPreferencesPanel.maxResultsLabel.toolTipText=<html>\nAll results are shown in the results table if this value is 0 (default).\n<br>You may want to consider setting this value if you have large numbers of results that are taking a long time  to display.\n</html>
+ViewPreferencesPanel.maxResultsLabel.toolTipText=<html>\nSetting this value to 0 will display all results in the results table.\n<br>Note that setting this value to 0 may result in poor UI responsiveness when there are large numbers of results.\n</html>

Core/src/org/sleuthkit/autopsy/corecomponents/Bundle.properties-MERGED

@@ -269,4 +269,4 @@ DataResultViewerTable.pagesLabel.text=Pages:
 DataResultViewerTable.pageNumLabel.text=
 DataResultViewerTable.pageLabel.text=Page:
 ViewPreferencesPanel.maxResultsLabel.text=Maximum number of Results to show in table:
-ViewPreferencesPanel.maxResultsLabel.toolTipText=<html>\nAll results are shown in the results table if this value is 0 (default).\n<br>You may want to consider setting this value if you have large numbers of results that are taking a long time  to display.\n</html>
+ViewPreferencesPanel.maxResultsLabel.toolTipText=<html>\nSetting this value to 0 will display all results in the results table.\n<br>Note that setting this value to 0 may result in poor UI responsiveness when there are large numbers of results.\n</html>

Core/src/org/sleuthkit/autopsy/datamodel/BaseChildFactory.java

@@ -286,7 +286,12 @@ public abstract class BaseChildFactory<T extends Content> extends ChildFactory.D
              * If pageSize is set split keys into pages, otherwise create a
              * single page containing all keys.
              */
+            if (keys.isEmpty()) {
+                pages.clear();
+            } else {
                 pages = Lists.partition(keys, pageSize > 0 ? pageSize : keys.size());
+            }
+
             if (pages.size() != oldPageCount) {
                 try {
                     // Number of pages has changed so we need to send out a notification.

Core/src/org/sleuthkit/autopsy/datamodel/FileTypesByExtension.java

@@ -390,6 +390,7 @@ public final class FileTypesByExtension implements AutopsyVisitableItem {
          * @param skCase
          * @param o      Observable that will notify when there could be new
          *               data to display
+         * @param nodeName
          */
         private FileExtensionNodeChildren(FileTypesByExtension.SearchFilterInterface filter, SleuthkitCase skCase, Observable o, String nodeName) {
             super(nodeName, new ViewsKnownAndSlackFilter<>());

Core/src/org/sleuthkit/autopsy/rejview/HexView.java

@@ -251,10 +251,10 @@ final class HexView extends JPanel {
             this.setHighlight(startByte, endByte);
 
             if (startByte != endByte) {
-                /**
+                /*
-                 * @param 1 Start
+                 * param 1 Start
-                 * @param 2 End
+                 * param 2 End
-                 * @param 3 Len
+                 * param 3 Len
                  */
                 int length = endByte - startByte;
                 String text = Bundle.HexView_statusTemplate_nonZeroLength(
@@ -266,8 +266,8 @@ final class HexView extends JPanel {
                         String.format("0x%1$x", length));
                 statusLabel.setText(text);
             } else {
-                /**
+                /*
-                 * @param 1 Start
+                 * param 1 Start
                  */
                 String text = Bundle.HexView_statusTemplate_zeroLength(startByte, String.format("0x%1$x", startByte));
                 statusLabel.setText(text);

Core/src/org/sleuthkit/autopsy/rejview/RejTreeKeyView.java

@@ -58,10 +58,10 @@ public final class RejTreeKeyView extends RejTreeNodeView {
     public RejTreeKeyView(RejTreeKeyNode node) {
         super(new BorderLayout());
 
-        /**
+        /*
-         * @param 1 Name
+         * param 1 Name
-         * @param 2 Number of subkeys
+         * param 2 Number of subkeys
-         * @param 3 Number of values
+         * param 3 Number of values
          */
         String metadataTemplate = "<html><i>"
                 + Bundle.RejTreeKeyView_template_name()

Core/src/org/sleuthkit/autopsy/rejview/RejTreeValueView.java

@@ -51,9 +51,9 @@ public final class RejTreeValueView extends RejTreeNodeView {
         "RejTreeValueView.valueBorder.title=Value",})
     public RejTreeValueView(RejTreeValueNode node) {
         super(new BorderLayout());
-        /**
+        /*
-         * @param 1 Name
+         * param 1 Name
-         * @param 2 Type
+         * param 2 Type
          */
         String metadataTemplate = "<html><i>"
                 + Bundle.RejTreeValueView_template_name()
@@ -63,8 +63,8 @@ public final class RejTreeValueView extends RejTreeNodeView {
         String valueName;
         String valueType;
 
-        /**
+        /*
-         * @param 1 Value
+         * param 1 Value
          */
         String valueTemplate = "<html>%1$s</html>";
         try {

Core/src/org/sleuthkit/autopsy/report/FileReportText.java

@@ -24,6 +24,7 @@ import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.OutputStreamWriter;
 import java.io.Writer;
+import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
@@ -63,9 +64,12 @@ class FileReportText implements FileReportModule {
     public void startReport(String baseReportDir) {
         this.reportPath = baseReportDir + FILE_NAME;
         try {
-            out = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(this.reportPath)));
+            out = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(this.reportPath), StandardCharsets.UTF_8));
+            out.write('\ufeff');
         } catch (FileNotFoundException ex) {
             logger.log(Level.WARNING, "Failed to create report text file", ex); //NON-NLS
+        } catch (IOException ex) {
+            logger.log(Level.WARNING, "Failed to write BOM to report text file", ex); //NON-NLS
         }
     }
 

Core/src/org/sleuthkit/autopsy/texttranslation/translators/GoogleTranslator.java

@@ -28,10 +28,13 @@ import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStream;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 import org.openide.util.NbBundle.Messages;
 import org.openide.util.lookup.ServiceProvider;
+import org.sleuthkit.autopsy.coreutils.EscapeUtil;
 import org.sleuthkit.autopsy.texttranslation.TextTranslator;
 import org.sleuthkit.autopsy.texttranslation.TranslationException;
 
@@ -57,8 +60,25 @@ public final class GoogleTranslator implements TextTranslator {
         loadTranslator();
     }
     
+    private static boolean googleIsReachable() {
+        String host = "www.google.com";
+        InetAddress address;
+        try {
+            address = InetAddress.getByName(host);
+            return address.isReachable(1500);
+        }catch (UnknownHostException ex) {
+            return false;
+        } catch (IOException ex) {
+            return false;
+        }
+    }
+    
     @Override
     public String translate(String string) throws TranslationException {
+        if (!googleIsReachable()) {
+            throw new TranslationException("Failure translating using GoogleTranslator: Cannot connect to Google");
+        }
+        
         if (googleTranslate != null) {
             try {
                 // Translates some text into English, without specifying the source language.
@@ -66,8 +86,8 @@ public final class GoogleTranslator implements TextTranslator {
                 // HTML files were producing lots of white space at the end
                 String substring = string.trim();
 
-                // WE can't currently set parameters, so we are using the default behavior of
+                // We can't currently set parameters, so we are using the default behavior of 
-                // asuming the input is HTML. We need to replace newlines with <br> for Google to preserve them
+                // assuming the input is HTML. We need to replace newlines with <br> for Google to preserve them
                 substring = substring.replaceAll("(\r\n|\n)", "<br />");
                 
                 // The API complains if the "Payload" is over 204800 bytes. I'm assuming that 
@@ -84,6 +104,10 @@ public final class GoogleTranslator implements TextTranslator {
                 
                 // put back the newlines
                 translatedString = translatedString.replaceAll("<br />", "\n");
+                
+                // With our current settings, Google Translate outputs HTML
+                // so we need to undo the escape characters.
+                translatedString = EscapeUtil.unEscapeHtml(translatedString);
                 return translatedString;
             } catch (Throwable ex) {  
                 //Catching throwables because some of this Google Translate code throws throwables

ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryModule.java

@@ -1,7 +1,7 @@
 /*
  * Autopsy Forensic Browser
  *
- * Copyright 2013-2018 Basis Technology Corp.
+ * Copyright 2013-2019 Basis Technology Corp.
  * Contact: carrier <at> sleuthkit <dot> org
  *
  * Licensed under the Apache License, Version 2.0 (the "License");

docs/doxygen-user/interesting_files.dox

@@ -1,81 +1,114 @@
 /*! \page interesting_files_identifier_page Interesting Files Identifier Module
 
-What Does It Do
+\section interesting_files_overview Overview
-========
 
-The Interesting Files module allows you to search for files or directories in a data source and generate alerts when they are found.  You configure rules for the files that you want to find. 
+The Interesting Files module allows you to automatically flag files and directories that match a set of rules. This can be useful if you always need to check whether files with a given name or path are in the data source, or if you are always interested in files with a certain type.
 
-Use this to be notified when certain things are found. There are examples below that generate alerts when VMWare images are found or when iPhone backup files are found.  This module is useful for file types that will frequently have a consistent name and that may not be part of the standard checklist that you look for, or if you simply want to automate your checklist. 
+This module allows you to make sets of rules that will be run against each file as it is processed. If a file matches any of the rules, you will see an entry for it in the \ref tree_viewer_page. You can share your rules with other users, and import sets made by others into your copy of Autopsy.
 
-Configuration
+\section interesting_files_terminology Terminology
-=======
 
-Add rules using "Tools", "Options", "Interesting Files".
+<ul>
+<li>A <b>rule</b> is a set of conditions that must be true about a file for it to match the rule. All conditions in the rule must be true. For example, if a rule has conditions "file size > 1 MB" and "file extension = .txt", only files that match both conditions will be considered a match.
+<li>A <b>rule set</b> is a collection of rules. If a file matches any rule in the rule set it will be flagged as a match for this rule set. Rule sets can be enabled and disabled at ingest time.
+</ul>
 
-All rules need to be part of a set. Select "New set" on the left side panel to create a new set. Sets need to have the following defined:
+\section interesting_files_config Configuration
 
-- Set Name (required)
+To create and edit your rule sets, go to "Tools", "Options" and then select the "Interesting Files" tab. The area on the left side will show you a list of all the rule sets that are currently available. Selecting a rule set will display its description and information about each of its rules on the right side of the panel.
-- Set Description (optional)
 
-Sets can be renamed, edited, copied, and imported and exported from the left side panel.
+\image html InterestingFiles/main.png
 
-Rules specify what to look for in a data source.  Each rule specifies:
+The buttons on the bottom of the left side of the panel control the rule sets.
-- Type: If the rule should be applied to only files, only directories, or both files and directories. 
-- Name Pattern: String to match the file name against. Note that you can enter multiple extensions in a comma-separated list.
-- Name Pattern Type: Should the pattern be matched against the full file type or just the extension. 
-- Path Pattern: A substring of the parent path that must be matched. This allows you to restrict generic names to a specific structure (such as an application name). A substring match is performed. 
-- Rule Name: Additional details that are displayed in the UI when that rule is matched. This allows you to determine which rule in the set matched.
 
-\image html interesting_files_configuration.PNG
+<ul>
+<li><b>New Set</b> - Allows you to create a new rule set (rules will be added later). You will see a new window asking for the name of the new rule set, an optional description, and whether known files should be ignored (i.e., if a file is in the NSRL, then it won't show up on the list of matches even if it satisfies the conditions of one of the rules in the set).
+\image html InterestingFiles/new_rule_set.png 
+<li><b>Edit Set</b> - Brings up the same window as "New Set" and allows you to change any of the fields.
+<li><b>Delete Set</b> - Removes the selected rule set
+<li><b>Copy Set</b> - Makes a copy of the selected rule set. It will bring up the same window as "New Set". You must change the rule set name in order to save the copy.
+<li><b>Import Set</b> - Imports a previously exported rule set. Once imported, you will not need the original copy.
+<li><b>Export Set</b> - Exports the selected rule set in a format that can be shared with other Autopsy users.
+</ul>
 
+Selecting a rule set will display its description, whether it ignores known files, and the rules contained in the set. Selecting a rule will display the conditions for that rule in the "Rule Details" section.
 
-VMWare Example
+The buttons under the list of rules allow you to create new rules and edit or delete existing rules. Selecting "New Rule" will bring up a new window to create the rule.
---------
-This set of rules is to detect VMWare Player or vmdk files.  This would help to make sure you look into the virtual machines for additional evidence. 
 
-NOTE: This is not extensive and is simply a minimal example:
+\image html InterestingFiles/new_rule.png 
 
+The top line allows you to choose whether you want to match only files, only directories, or both. If you select directories or both, some of the condition types will be unavailable since they only apply to files.
 
-- Set Name: VMWare
+Each rule must have at least one condition. To create conditions, check the box to the left of the condition you want to enable. The following is a description of each condition, with some full examples after.
-- Rule 1: 
- - Type: Files
- - Full Name: vmplayer.exe
- - Name: Program EXE
-- Rule 2:
- - Type: Files
- - Extension: vmdk
- - Name: VMDK File
 
-iPhone Backups Example
+<ul>
--------------
+<li><b>Name</b> - Enter either the full file name or one or more extensions, and select whether this is an exact match or a substring/regex match. If substring/regex match is enabled, it will automatically add wildcards to the beginning and end of the text. If you're only matching directories, this will match the directory name. If you're using a comma-separated list of extensions, make sure the regex checkbox is disabled - the two features do not work together. The following table shows some examples of what the different combinations can be used for.
-This set of rules is to detect a folder for iPhone Backups.  These are typically in a folder such as "%AppData%\Roaming\Apple Computer\MobileSync\Backup" on Windows. Here is a rule that you could use for that.
 
-- Set Name: iPhone Backups
+<table>
-- Rule 1:
+<tr><th>Type</th><th>Substring/Regex</th><th>Text</th><th>Description</th><th>Sample match</th></tr>
- - Type: Directory
+<tr><td>Full Name</td><td>false</td><td>\verbatim test.txt \endverbatim</td><td>Will match files named "test.txt"</td><td>text.txt</tr>
- - Name: Backup
+<tr><td>Full Name</td><td>true</td><td>\verbatim bomb \endverbatim</td><td>Will match files with "bomb" anywhere their name</td><td>Pipe bomb.png</td></tr>
- - Path: Apple Computer/MobileSync
+<tr><td>Full Name</td><td>true</td><td>\verbatim virus.*\.exe \endverbatim</td><td>Will match files with "virus" followed by ".exe" anywhere their name</td><td>bad_virus.exe</td></tr>
+<tr><td>Extension Only</td><td>false</td><td>\verbatim zip \endverbatim</td><td>Will match .zip files</td><td>myArchive.zip</td></tr>
+<tr><td>Extension Only</td><td>false</td><td>\verbatim zip,rar,7z \endverbatim</td><td>Will match .zip, .rar, and .7z files</td><td>anotherArchive.rar</td></tr>
+<tr><td>Extension Only</td><td>true</td><td>\verbatim jp \endverbatim</td><td>Will match .jpg, .jpeg files, and any others with "jp" in the extension</td><td>myImage.jpg</td></tr>
+</table>
 
+<li><b>Path Substring</b> - Enter a folder name that must be part of file's path for it to be a match. If you only want to specify that a word appears somewhere in the path, use the regex option.
+<table>
+<tr><th>Regex</th><th>Text</th><th>Description</th><th>Sample match</th></tr>
+<tr><td>false</td><td>\verbatim Documents \endverbatim</td><td>Match any file that has a folder named "Documents" in its path</td><td>/folder1/Documents/fileA.doc</td></tr>
+<tr><td>true</td><td>\verbatim bomb \endverbatim</td><td>Match any file with "bomb" in the path</td><td>/folder1/bomb making/file2.doc</td></tr>
+<tr><td>true</td><td>\verbatim Users/.*/Downloads \endverbatim</td><td>Match any file with "Users" and "Downloads" in the path</td><td>C:/Users/user1/Downloads/myFile.txt</td></tr>
+</table>
 
-Using the Module
+<li><b>MIME Type</b> - Use the pull-down list to select a MIME type. Only a single MIME type can be selected.
-======
 
-When you enable the Interesting Files module, you can choose what rule sets to enable.  To add rules, use the "Advanced" button from the ingest module panel. 
+<li><b>File Size</b> - Select whether you want to match files equal to, smaller than, or larger than a given size. 
 
-When files are found, they will be in the Interesting Files area of the tree.  You should see the set and rule names with the match.
+<li><b>Modified Within</b> - Select how recently a file must have been modified to match the rule.
+</ul>
 
+Finally you can optionally enter a name for the rule. This will be displayed in the UI for each match.
 
-Ingest Settings
+\subsection interesting_files_examples Examples
-------
+Here are a few examples of rules being created.
 
-When running the ingest modules, the user can choose which interesting file rules to enable .
+This is a rule that matches any file with "bomb" in the name that also has an "image/png" MIME type.
-<br>
-\image html interesting_files_ingest_settings.PNG
 
-Seeing Results
+\image html InterestingFiles/bomb_png.png 
-------
-The results show up in the tree under "Results", "Interesting Items".
 
-\image html interesting_files_results.PNG
+This is a rule that matches folders named "Private".
+
+\image html InterestingFiles/private_folder.png
+ 
+This rule is looking for archives in the user download directory. It requires "Users" and "Downloads" in the file's path, and an extension of .zip, .rar, or .7z.
+ 
+\image html InterestingFiles/download_archive.png
+
+This is a rule that matches files with size at least 50MB that have been modified in the last week.
+
+\image html InterestingFiles/new_large_files.png
+
+\section interesting_files_running Running the Module
+
+At runtime, you can select which rule sets you would like to run on your data source.
+
+\image html InterestingFiles/ingest.png
+
+\section interesting_files_results Viewing Results
+
+Files that match any of the rules in the enabled rule sets will be shown in the Results section of the \ref tree_viewer_page under "Interesting Items" and then the name of the rule set that matched. Note that other modules besides Interesting Files put results in this section of the tree, so there may be more than just what matched your rule sets. Selecting the "Interesting Files" node under one of your rule sets will display all matching files in the \ref result_viewer_page.
+
+\image html InterestingFiles/results.png
+
+You can see which rule matched in the "Category" column. You can export some or all of the files for further analysis. To do this, first use the standard Windows file
+selection methods to highlight the files you want to export in the \ref result_viewer_page :
+<ul>
+<li>Hold down Ctrl and click on each file you want to export
+<li>Hold down Shift to select a range of files
+<li>Click on any file in the Result Viewer and then hit Ctrl+A to select all the files
+</ul>
+Once you have your desired files selected, right click and select “Extract Files” to save copies of them.
 
 */