Merge remote-tracking branch 'upstream/develop' into custom_artifact_addition

Core/src/org/sleuthkit/autopsy/corecomponents/MediaViewImagePanel.java

@@ -29,9 +29,7 @@ import java.util.concurrent.ExecutionException;
 import java.util.stream.Collectors;
 import javafx.application.Platform;
 import javafx.concurrent.Task;
-import javafx.concurrent.WorkerStateEvent;
 import javafx.embed.swing.JFXPanel;
-import javafx.event.EventHandler;
 import javafx.geometry.Pos;
 import javafx.scene.Cursor;
 import javafx.scene.Scene;
@@ -58,8 +56,13 @@ import org.sleuthkit.datamodel.AbstractFile;
  * Image viewer part of the Media View layered pane. Uses JavaFX to display the
  * image.
  */
+@NbBundle.Messages({"MediaViewImagePanel.externalViewerButton.text=Open in External Viewer",
+    "MediaViewImagePanel.errorLabel.text=Could not load file into Media View.",
+    "MediaViewImagePanel.errorLabel.OOMText=Could not load file into Media View: insufficent memory."})
 public class MediaViewImagePanel extends JPanel implements DataContentViewerMedia.MediaViewPanel {
 
+    private static final Image EXTERNAL = new Image(MediaViewImagePanel.class.getResource("/org/sleuthkit/autopsy/images/external.png").toExternalForm());
+
     private static final Logger LOGGER = Logger.getLogger(MediaViewImagePanel.class.getName());
 
     private final boolean fxInited;
@@ -70,17 +73,6 @@ public class MediaViewImagePanel extends JPanel implements DataContentViewerMedi
     private final ProgressBar progressBar = new ProgressBar();
     private final MaskerPane maskerPane = new MaskerPane();
 
-    @NbBundle.Messages({"MediaViewImagePanel.errorLabel.text=Could not load file into Media view."})
-    private final Label errorLabel = new Label(Bundle.MediaViewImagePanel_errorLabel_text());
-
-    /**
-     * TODO: why is this passed to the action? it means we duplciate this string
-     * all over the place -jm
-     */
-    @NbBundle.Messages({"MediaViewImagePanel.externalViewerButton.text=Open in External Viewer"})
-    private final Button externalViewerButton = new Button(Bundle.MediaViewImagePanel_externalViewerButton_text());
-    private final VBox errorNode = new VBox(10, errorLabel, externalViewerButton);
-
     static {
         ImageIO.scanForPlugins();
     }
@@ -109,8 +101,6 @@ public class MediaViewImagePanel extends JPanel implements DataContentViewerMedi
         if (fxInited) {
             Platform.runLater(() -> {
 
-                errorNode.setAlignment(Pos.CENTER);
-
                 // build jfx ui (we could do this in FXML?)
                 fxImageView = new ImageView();  // will hold image
                 borderpane = new BorderPane(fxImageView); // centers and sizes imageview
@@ -148,11 +138,19 @@ public class MediaViewImagePanel extends JPanel implements DataContentViewerMedi
         });
     }
 
-    private void showErrorNode(AbstractFile file) {
+    private void showErrorNode(String errorMessage, AbstractFile file) {
+        final Button externalViewerButton = new Button(Bundle.MediaViewImagePanel_externalViewerButton_text(), new ImageView(EXTERNAL));
         externalViewerButton.setOnAction(actionEvent -> //fx ActionEvent
+                /*
+                 * TODO: why is the name passed into the action constructor? it
+                 * means we duplicate this string all over the place -jm
+                 */
                 new ExternalViewerAction(Bundle.MediaViewImagePanel_externalViewerButton_text(), new FileNode(file))
-                .actionPerformed(new ActionEvent(this, ActionEvent.ACTION_PERFORMED, "")) //Swing ActionEvent //NOI18N
+                .actionPerformed(new ActionEvent(this, ActionEvent.ACTION_PERFORMED, "")) //Swing ActionEvent 
         );
+
+        final VBox errorNode = new VBox(10, new Label(errorMessage), externalViewerButton);
+        errorNode.setAlignment(Pos.CENTER);
         borderpane.setCenter(errorNode);
     }
 
@@ -172,12 +170,14 @@ public class MediaViewImagePanel extends JPanel implements DataContentViewerMedi
                 readImageTask.cancel();
             }
             readImageTask = ImageUtils.newReadImageTask(file);
-            readImageTask.setOnSucceeded((WorkerStateEvent event) -> {
+            readImageTask.setOnSucceeded(succeeded -> {
                 //Note that all error conditions are allready logged in readImageTask.succeeded()
                 if (!Case.isCaseOpen()) {
                     /*
                      * handle in-between condition when case is being closed and
                      * an image was previously selected
+                     *
+                     * NOTE: I think this is unnecessary -jm
                      */
                     reset();
                     return;
@@ -190,29 +190,33 @@ public class MediaViewImagePanel extends JPanel implements DataContentViewerMedi
                         fxImageView.setImage(fxImage);
                         borderpane.setCenter(fxImageView);
                     } else {
-                        showErrorNode(file);
+                        showErrorNode(Bundle.MediaViewImagePanel_errorLabel_text(), file);
                     }
                 } catch (InterruptedException | ExecutionException ex) {
-                    showErrorNode(file);
+                    showErrorNode(Bundle.MediaViewImagePanel_errorLabel_text(), file);
                 }
                 borderpane.setCursor(Cursor.DEFAULT);
             });
-            readImageTask.setOnFailed(new EventHandler<WorkerStateEvent>() {
+            readImageTask.setOnFailed(failed -> {
-
+                if (!Case.isCaseOpen()) {
-                @Override
+                    /*
-                public void handle(WorkerStateEvent event) {
+                     * handle in-between condition when case is being closed and
-                    if (!Case.isCaseOpen()) {
+                     * an image was previously selected
-                        /*
+                     *
-                         * handle in-between condition when case is being closed
+                     * NOTE: I think this is unnecessary -jm
-                         * and an image was previously selected
+                     */
-                         */
+                    reset();
-                        reset();
+                    return;
-                        return;
-                    }
-
-                    showErrorNode(file);
-                    borderpane.setCursor(Cursor.DEFAULT);
                 }
+                Throwable exception = readImageTask.getException();
+                if (exception instanceof OutOfMemoryError
+                        && exception.getMessage().contains("Java heap space")) {
+                    showErrorNode(Bundle.MediaViewImagePanel_errorLabel_OOMText(), file);
+                } else {
+                    showErrorNode(Bundle.MediaViewImagePanel_errorLabel_text(), file);
+                }
+
+                borderpane.setCursor(Cursor.DEFAULT);
             });
 
             maskerPane.setProgressNode(progressBar);

Core/src/org/sleuthkit/autopsy/coreutils/ImageUtils.java

@@ -421,10 +421,17 @@ public class ImageUtils {
      * @param fileID
      *
      * @return a File object representing the location of the cached thumbnail.
-     *         This file may not actually exist(yet).
+     *         This file may not actually exist(yet). Returns null if there was
+     *         any problem getting the file, such as no case was open.
      */
     private static File getCachedThumbnailLocation(long fileID) {
-        return Paths.get(Case.getCurrentCase().getCacheDirectory(), "thumbnails", fileID + ".png").toFile(); //NOI18N
+        try {
+            String cacheDirectory = Case.getCurrentCase().getCacheDirectory();
+            return Paths.get(cacheDirectory, "thumbnails", fileID + ".png").toFile(); //NOI18N
+        } catch (IllegalStateException e) {
+            return null;
+        }
+
     }
 
     /**
@@ -650,7 +657,7 @@ public class ImageUtils {
         @Override
         protected javafx.scene.image.Image call() throws Exception {
             // If a thumbnail file is already saved locally, just read that.
-            if (cacheFile.exists()) {
+            if (cacheFile != null && cacheFile.exists()) {
                 try {
                     BufferedImage cachedThumbnail = ImageIO.read(cacheFile);
                     if (nonNull(cachedThumbnail) && cachedThumbnail.getWidth() == iconSize) {
@@ -710,7 +717,7 @@ public class ImageUtils {
             updateProgress(-1, 1);
 
             //if we got a valid thumbnail save it
-            if (nonNull(thumbnail) && DEFAULT_THUMBNAIL != thumbnail) {
+            if (cacheFile != null && nonNull(thumbnail) && DEFAULT_THUMBNAIL != thumbnail) {
                 saveThumbnail(thumbnail);
             }
             return SwingFXUtils.toFXImage(thumbnail, null);

ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java

@@ -928,7 +928,15 @@ public final class DrawableDB {
                 try (Statement stmt = con.createStatement();
                         ResultSet valsResults = stmt.executeQuery(query.toString())) {
                     while (valsResults.next()) {
-                        vals.add((A) valsResults.getObject(groupBy.attrName.toString()));
+                        /*
+                         * I don't like that we have to do this cast here, but
+                         * can't think of a better alternative at the momment
+                         * unless something has gone seriously wrong, we know
+                         * this should be of type A even if JAVA doesn't
+                         */
+                        @SuppressWarnings("unchecked")
+                        A value = (A) valsResults.getObject(groupBy.attrName.toString());
+                        vals.add(value);
                     }
                 } catch (SQLException ex) {
                     LOGGER.log(Level.WARNING, "Unable to get values for attribute", ex);
@@ -1209,8 +1217,7 @@ public final class DrawableDB {
      * @param f check if this file is a video. will return false for null file.
      *
      * @return returns true if this file is a video as determined by {@link ImageGalleryModule#isVideoFile(org.sleuthkit.datamodel.AbstractFile)
-     *         } but caches the result.
+     *         } but caches the result. returns false if passed a null AbstractFile
-     *         returns false if passed a null AbstractFile
      */
     public boolean isVideoFile(AbstractFile f) {
         return isNull(f) ? false

ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/drawableviews/DrawableUIBase.java

@@ -50,12 +50,13 @@ import org.sleuthkit.datamodel.AbstractFile;
 import org.sleuthkit.datamodel.TskCoreException;
 
 /**
- *
+ * Abstract base class for views of a single drawable file.
  */
-@NbBundle.Messages({"MediaViewImagePanel.errorLabel.text=Could not read file."})
+@NbBundle.Messages({"DrawableUIBase.errorLabel.text=Could not read file",
+    "DrawableUIBase.errorLabel.OOMText=Insufficent memory"})
 abstract public class DrawableUIBase extends AnchorPane implements DrawableView {
 
-    static final Executor exec = Executors.newWorkStealingPool();
+    static final Executor exec = Executors.newSingleThreadExecutor();
 
     private static final Logger LOGGER = Logger.getLogger(DrawableUIBase.class.getName());
 
@@ -145,7 +146,13 @@ abstract public class DrawableUIBase extends AnchorPane implements DrawableView
             }
         });
         myTask.setOnFailed(failed -> {
-            showErrorNode(Bundle.MediaViewImagePanel_errorLabel_text(), file);
+            Throwable exception = myTask.getException();
+            if (exception instanceof OutOfMemoryError
+                    && exception.getMessage().contains("Java heap space")) {
+                showErrorNode(Bundle.DrawableUIBase_errorLabel_OOMText(), file);
+            } else {
+                showErrorNode(Bundle.DrawableUIBase_errorLabel_OOMText(), file);
+            }
             synchronized (DrawableUIBase.this) {
                 imageTask = null;
             }
@@ -190,12 +197,12 @@ abstract public class DrawableUIBase extends AnchorPane implements DrawableView
                 imageView.setImage(fxImage);
                 imageBorder.setCenter(imageView);
             } else {
-                showErrorNode(Bundle.MediaViewImagePanel_errorLabel_text(), file);
+                showErrorNode(Bundle.DrawableUIBase_errorLabel_text(), file);
             }
         } catch (CancellationException ex) {
 
         } catch (InterruptedException | ExecutionException ex) {
-            showErrorNode(Bundle.MediaViewImagePanel_errorLabel_text(), file);
+            showErrorNode(Bundle.DrawableUIBase_errorLabel_text(), file);
         }
     }
 
@@ -203,9 +210,7 @@ abstract public class DrawableUIBase extends AnchorPane implements DrawableView
     void showErrorNode(String errorMessage, AbstractFile file) {
         Button createButton = ActionUtils.createButton(new OpenExternalViewerAction(file));
 
-        VBox vBox = new VBox(10,
+        VBox vBox = new VBox(10, new Label(errorMessage), createButton);
-                new Label(errorMessage), createButton);
-
         vBox.setAlignment(Pos.CENTER);
         imageBorder.setCenter(vBox);
     }

ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/drawableviews/MetaDataPane.java

@@ -214,6 +214,7 @@ public class MetaDataPane extends DrawableUIBase {
     public void handleCategoryChanged(CategoryManager.CategoryChangeEvent evt) {
         getFileID().ifPresent(fileID -> {
             if (evt.getFileIDs().contains(fileID)) {
+                updateCategory();
                 updateAttributesTable();
             }
         });

ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/drawableviews/SlideShowView.java

@@ -191,19 +191,19 @@ public class SlideShowView extends DrawableTileBase {
         Platform.runLater(() -> imageBorder.setCenter(progressNode));
 
         //called on fx thread
-        mediaTask.setOnSucceeded(succeedded -> {
+        myTask.setOnSucceeded(succeedded -> {
             showMedia(file, myTask);
             synchronized (SlideShowView.this) {
                 mediaTask = null;
             }
         });
-        mediaTask.setOnFailed(failed -> {
+        myTask.setOnFailed(failed -> {
             showErrorNode(getMediaLoadErrorLabel(myTask), file);
             synchronized (SlideShowView.this) {
                 mediaTask = null;
             }
         });
-        mediaTask.setOnCancelled(cancelled -> {
+        myTask.setOnCancelled(cancelled -> {
             disposeContent();
         });
 
@@ -228,7 +228,7 @@ public class SlideShowView extends DrawableTileBase {
     }
 
     private String getMediaLoadErrorLabel(Task<Node> mediaTask) {
-        return Bundle.MediaViewImagePanel_errorLabel_text() + ": " + mediaTask.getException().getLocalizedMessage();
+        return Bundle.DrawableUIBase_errorLabel_text() + ": " + mediaTask.getException().getLocalizedMessage();
     }
 
     /**

KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AbstractFileChunk.java

@@ -55,10 +55,7 @@ class AbstractFileChunk {
     }
 
     void index(Ingester ingester, byte[] content, long contentSize, Charset indexCharset) throws IngesterException {
-        // We are currently only passing utf-8 as indexCharset. If other charsets were to be used in the future, 
+        ByteContentStream bcs = new ByteContentStream(content, contentSize, parent.getSourceFile(), indexCharset);
-        // this might need to be changed to accommodate.
-        byte[] saitizedContent = sanitize(content, indexCharset);
-        ByteContentStream bcs = new ByteContentStream(saitizedContent, contentSize, parent.getSourceFile(), indexCharset);
         try {
             ingester.ingest(this, bcs, content.length);
         } catch (Exception ingEx) {
@@ -66,35 +63,4 @@ class AbstractFileChunk {
                     parent.getSourceFile().getId(), chunkID), ingEx);
         }
     }
-
-    // Given a byte array, filter out all occurances non-characters 
-    // http://unicode.org/cldr/utility/list-unicodeset.jsp?a=[:Noncharacter_Code_Point=True:]
-    // and non-printable control characters except tabulator, new line and carriage return
-    // and replace them with the character (^)
-    private static byte[] sanitize(byte[] input, Charset indexCharset) {
-        String inputString = new String(input, indexCharset);
-        StringBuilder sanitized = new StringBuilder(inputString.length());
-        char ch;
-        for (int i = 0; i < inputString.length(); i++) {
-            ch = inputString.charAt(i);
-            if (charIsValidSolrUTF8(ch)) {
-                sanitized.append(ch);
-            } else {
-                sanitized.append('^'); // NON-NLS
-            }
-        }
-
-        byte[] output = sanitized.toString().getBytes(indexCharset);
-        return output;
-    }
-
-    // Is the given character a valid UTF-8 character
-    // return true if it is, false otherwise
-    private static boolean charIsValidSolrUTF8(char ch) {
-        return (ch % 0x10000 != 0xffff && // 0xffff - 0x10ffff range step 0x10000
-                ch % 0x10000 != 0xfffe && // 0xfffe - 0x10fffe range
-                (ch <= 0xfdd0 || ch >= 0xfdef) && // 0xfdd0 - 0xfdef
-                (ch > 0x1F || ch == 0x9 || ch == 0xa || ch == 0xd));
-    }
-
 }

KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TikaTextExtractor.java

@@ -64,9 +64,8 @@ class TikaTextExtractor implements TextExtractor {
     private static final int MAX_EXTR_TEXT_CHARS = 512 * 1024;
     private static final int SINGLE_READ_CHARS = 1024;
     private static final int EXTRA_CHARS = 128; //for whitespace
-    //private static final String UTF16BOM = "\uFEFF"; disabled prepending of BOM
     private final char[] textChunkBuf = new char[MAX_EXTR_TEXT_CHARS];
-    private KeywordSearchIngestModule module;
+    private final KeywordSearchIngestModule module;
     private AbstractFile sourceFile; //currently processed file
     private int numChunks = 0;
     private final ExecutorService tikaParseExecutor = Executors.newSingleThreadExecutor();
@@ -187,20 +186,16 @@ class TikaTextExtractor implements TextExtractor {
                     }
                 }
 
-                //logger.log(Level.INFO, "TOTAL READ SIZE: " + totalRead + " file: " + sourceFile.getName());
+                // Sanitize by replacing non-UTF-8 characters with caret '^'
-                //encode to bytes to index as byte stream
+                for (int i = 0; i < totalRead; ++i) {
-                String extracted;
+                    if (!isValidSolrUTF8(textChunkBuf[i])) {
-                //add BOM and trim the 0 bytes
+                        textChunkBuf[i] = '^';
-                //set initial size to chars read + bom + metadata (roughly) - try to prevent from resizing
+                    }
-                StringBuilder sb = new StringBuilder((int) totalRead + 1000);
-                //inject BOM here (saves byte buffer realloc later), will be converted to specific encoding BOM
-                //sb.append(UTF16BOM); disabled prepending of BOM
-                if (totalRead < MAX_EXTR_TEXT_CHARS) {
-                    sb.append(textChunkBuf, 0, (int) totalRead);
-                } else {
-                    sb.append(textChunkBuf);
                 }
 
+                StringBuilder sb = new StringBuilder((int) totalRead + 1000);
+                sb.append(textChunkBuf, 0, (int) totalRead);
+
                 //reset for next chunk
                 totalRead = 0;
 
@@ -216,10 +211,8 @@ class TikaTextExtractor implements TextExtractor {
                     }
                 }
 
-                extracted = sb.toString();
+                // Encode from UTF-8 charset to bytes
-
+                byte[] encodedBytes = sb.toString().getBytes(OUTPUT_CHARSET);
-                //converts BOM automatically to charSet encoding
-                byte[] encodedBytes = extracted.getBytes(OUTPUT_CHARSET);
                 AbstractFileChunk chunk = new AbstractFileChunk(this, this.numChunks + 1);
                 try {
                     chunk.index(ingester, encodedBytes, encodedBytes.length, OUTPUT_CHARSET);
@@ -262,6 +255,18 @@ class TikaTextExtractor implements TextExtractor {
         return success;
     }
 
+    /**
+     * This method determines if a passed-in Java char (16 bits) is a valid
+     * UTF-8 printable character, returning true if so, false if not.
+     *
+     * @param ch the character to test
+     *
+     * @return Returns true if the character is valid UTF-8, false if not.
+     */
+    private static boolean isValidSolrUTF8(char ch) {
+        return ((ch <= 0xFDD0 || ch >= 0xFDEF) && (ch > 0x1F || ch == 0x9 || ch == 0xA || ch == 0xD) && (ch != 0xFFFF) && (ch != 0xFFFE));
+    }
+
     @Override
     public boolean isContentTypeSpecific() {
         return true;

docs/doxygen/modReport.dox

@@ -1,14 +1,14 @@
 /*! \page mod_report_page Developing Report Modules
 
 \section report_summary Overview
-Report modules allow Autopsy users to create different report types.  Autopsy comes with modules to generate HTML and Excel artifact reports, a tab delimited File report, a Keyhole Markup Language (KML) report for Google Earth data, and a body file for timeline creation.  You can made additional modules to create custom output formats. 
+Report modules allow Autopsy users to create different report types.  Autopsy comes with modules to generate HTML and Excel artifact reports, a tab delimited File report, a Keyhole Markup Language (KML) report for Google Earth data, and a body file for timeline creation.  You can make additional modules to create custom output formats. 
 
 There are three types of reporting modules that differ in how the data is organized. 
+- Table report modules organize the data into tables.  If your output is in table format, this type of module will be easier to make because Autopsy does a lot of the organizing work for you. 
+- File report modules are also table-based, but they specifically deal with reporting on the Files in the case, not artifacts.
 - General report modules are free form and you are allowed to organize the output however you want.  
-- Table report modules organize the data into tables.  If your output is in table format, this type of module will be easier to make the module because Autopsy does a lot of the organizing work for you. 
-- File report modules are also table based, but they specifically deal with reporting on the Files in the case, not artifacts.
 
-Table report modules require their sub-classes to override methods to start and end tables, and add rows to those tables. These methods are provided data, generated from a default configuration panel, for the module to report on. Because of this, when creating a table report module one only needs to focus on how to display the data, not how to find it.
+Table report modules require their subclasses to override methods to start and end tables, and add rows to those tables. These methods are provided data, generated from a default configuration panel, for the module to report on. Because of this, when creating a table report module one only needs to focus on how to display the data, not how to find it.
 
 File report modules are similar to table report modules, but only require their sub-classes to start and end a single table, and add rows to that table. The methods are given an AbstractFile and a list of FileReportDataTypes, which specify what information about the file should be added to the report. The data can be extracted from the file by calling the FileReportDataTypes getValue method with the file as it's argument.
 
@@ -17,7 +17,10 @@ On the other hand, general report modules have a single method to generate the r
 General modules are also given the responsibility of updating their report's progress bar and processing label in the UI. A progress panel is given to every general report module. It contains basic API to start, stop, and add to the progress bar, as well as update the processing label. The module is also expected to check the progress bar's status occasionally to see if the user has manually canceled the report.
 
 \section report_create_module Creating a Report Module
-To create a report module, start off by creating a new Java or Python (Jython) class and implementing (Java) or inheriting (Jython) one of the org.sleuthkit.autopsy.report.TableReportModule interface, the org.sleuthkit.autopsy.report.FileReportModule interface, or the org.sleuthkit.autopsy.report.GeneralReportModule interface. 
+To create a report module, start off by creating a new Java or Python (Jython) class and implementing (Java) or inheriting (Jython) the appropriate interface:
+- org.sleuthkit.autopsy.report.TableReportModule 
+- org.sleuthkit.autopsy.report.FileReportModule 
+- org.sleuthkit.autopsy.report.GeneralReportModule 
 
 All three of these interfaces extend the org.sleuthkit.autopsy.report.ReportModule interface that defines the following methods:
 - org.sleuthkit.autopsy.report.ReportModule.getName()