Merge pull request #4915 from dannysmyda/5202-Export-Tags-Saving-Bug

5202 export tags saving bug

Core/src/org/sleuthkit/autopsy/contentviewers/MediaViewImagePanel.java

@@ -25,7 +25,6 @@ import java.beans.PropertyChangeEvent;
 import java.beans.PropertyChangeListener;
 import java.beans.PropertyChangeSupport;
 import java.io.File;
-import java.io.IOException;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.ArrayList;
@@ -78,7 +77,7 @@ import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException;
 import org.sleuthkit.autopsy.casemodule.services.contentviewertags.ContentViewerTagManager;
 import org.sleuthkit.autopsy.casemodule.services.contentviewertags.ContentViewerTagManager.ContentViewerTag;
 import org.sleuthkit.autopsy.casemodule.services.contentviewertags.ContentViewerTagManager.SerializationException;
-import org.sleuthkit.autopsy.contentviewers.imagetagging.ImageTagsUtility;
+import org.sleuthkit.autopsy.contentviewers.imagetagging.ImageTagsUtil;
 import org.sleuthkit.autopsy.contentviewers.imagetagging.ImageTagControls;
 import org.sleuthkit.autopsy.contentviewers.imagetagging.ImageTagRegion;
 import org.sleuthkit.autopsy.contentviewers.imagetagging.ImageTagCreator;
@@ -876,13 +875,14 @@ class MediaViewImagePanel extends JPanel implements MediaFileViewer.MediaViewPan
                                 .map(cvTag -> cvTag.getDetails()).collect(Collectors.toList());
                         
                         //Apply tags to image and write to file
-                        BufferedImage pngImage = ImageTagsUtility.writeTags(file, regions, "png");
+                        BufferedImage taggedImage = ImageTagsUtil.getImageWithTags(file, regions);
                         Path output = Paths.get(exportChooser.getSelectedFile().getPath(),
                                 FilenameUtils.getBaseName(file.getName()) + "-with_tags.png"); //NON-NLS
-                        ImageIO.write(pngImage, "png", output.toFile());
+                        ImageIO.write(taggedImage, "png", output.toFile());
                         
                         JOptionPane.showMessageDialog(null, Bundle.MediaViewImagePanel_successfulExport());
-                    } catch (TskCoreException | NoCurrentCaseException | IOException ex) {
+                    } catch (Exception ex) { //Runtime exceptions may spill out of ImageTagsUtil from JavaFX.
+                        //This ensures we (devs and users) have something when it doesn't work.
                         LOGGER.log(Level.WARNING, "Unable to export tagged image to disk", ex); //NON-NLS
                         JOptionPane.showMessageDialog(null, Bundle.MediaViewImagePanel_unsuccessfulExport());
                     }

Core/src/org/sleuthkit/autopsy/contentviewers/imagetagging/ImageTagsUtil.java

@@ -0,0 +1,239 @@
+/*
+ * Autopsy Forensic Browser
+ *
+ * Copyright 2019 Basis Technology Corp.
+ * Contact: carrier <at> sleuthkit <dot> org
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.sleuthkit.autopsy.contentviewers.imagetagging;
+
+import java.awt.image.BufferedImage;
+import java.io.BufferedInputStream;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.concurrent.ExecutionException;
+import javafx.concurrent.Task;
+import javafx.embed.swing.SwingFXUtils;
+import javafx.scene.image.Image;
+import javax.imageio.ImageIO;
+import org.opencv.core.Core;
+import org.opencv.core.Mat;
+import org.opencv.core.MatOfByte;
+import org.opencv.core.Point;
+import org.opencv.core.Scalar;
+import org.opencv.core.Size;
+import org.opencv.highgui.Highgui;
+import org.opencv.imgproc.Imgproc;
+import org.sleuthkit.autopsy.coreutils.ImageUtils;
+import org.sleuthkit.datamodel.AbstractFile;
+import org.sleuthkit.datamodel.ReadContentInputStream;
+
+/**
+ * Utility for drawing rectangles on image files.
+ */
+public final class ImageTagsUtil {
+    
+    //String constant for writing PNG in ImageIO
+    private final static String AWT_PNG = "png";
+    
+    //String constant for encoding PNG in OpenCV
+    private final static String OPENCV_PNG = ".png";
+
+    /**
+     * Creates an image with tags applied.
+     *
+     * @param file Source image.
+     * @param tagRegions Tags to apply.
+     * @return Tagged image.
+     *
+     * @throws IOException
+     * @throws InterruptedException Calling thread was interrupted
+     * @throws ExecutionException Error while reading image from AbstractFile
+     */
+    public static BufferedImage getImageWithTags(AbstractFile file,
+            Collection<ImageTagRegion> tagRegions) throws IOException, InterruptedException, ExecutionException {
+
+        //The raw image in OpenCV terms
+        Mat sourceImage = getImageMatFromFile(file);
+        //Image with tags in OpenCV terms
+        MatOfByte taggedMatrix = getTaggedImageMatrix(sourceImage, tagRegions);
+
+        try (ByteArrayInputStream taggedStream = new ByteArrayInputStream(taggedMatrix.toArray())) {
+            return ImageIO.read(taggedStream);
+        } finally {
+            sourceImage.release();
+            taggedMatrix.release();
+        }
+    }
+    
+    /**
+     * Get the image from file.
+     * 
+     * @param file
+     * @return
+     * @throws IOException
+     * @throws InterruptedException
+     * @throws ExecutionException 
+     */
+    private static BufferedImage getImageFromFile(AbstractFile file) throws IOException, InterruptedException, ExecutionException {
+        if (ImageUtils.isGIF(file)) {
+            //Grab the first frame.
+            try (BufferedInputStream bufferedReadContentStream = 
+                    new BufferedInputStream(new ReadContentInputStream(file))) {
+                return ImageIO.read(bufferedReadContentStream);
+            }
+        } else {
+            //Otherwise, read the full image.
+            Task<Image> readImageTask = ImageUtils.newReadImageTask(file);
+            readImageTask.run();
+            Image fxResult = readImageTask.get();
+            return SwingFXUtils.fromFXImage(fxResult, null);
+        }
+    }
+
+    /**
+     * Reads the image and converts it into an OpenCV equivalent.
+     *
+     * @param file Image to read
+     * @return raw image bytes
+     *
+     * @throws IOException
+     * @throws InterruptedException Calling thread was interrupted.
+     * @throws ExecutionException Error while reading image from AbstractFile
+     */
+    private static Mat getImageMatFromFile(AbstractFile file) throws InterruptedException, ExecutionException, IOException {
+        //Get image from file
+        BufferedImage buffImage = getImageFromFile(file);
+
+        //Convert it to OpenCV Mat.
+        try (ByteArrayOutputStream outStream = new ByteArrayOutputStream()) {
+            ImageIO.write(buffImage, AWT_PNG, outStream);
+
+            byte[] imageBytes = outStream.toByteArray();
+            MatOfByte rawSourceBytes = new MatOfByte(imageBytes);
+            Mat sourceImage = Highgui.imdecode(rawSourceBytes, Highgui.IMREAD_COLOR);
+            rawSourceBytes.release();
+
+            return sourceImage;
+        }
+    }
+
+    /**
+     * Adds tags to an image matrix.
+     *
+     * @param sourceImage
+     * @param tagRegions
+     * @param outputEncoding
+     * @return
+     */
+    private static MatOfByte getTaggedImageMatrix(Mat sourceImage, Collection<ImageTagRegion> tagRegions) {
+
+        //Apply all tags to source image
+        for (ImageTagRegion region : tagRegions) {
+            Point topLeft = new Point(region.getX(), region.getY());
+            Point bottomRight = new Point(topLeft.x + region.getWidth(),
+                    topLeft.y + region.getHeight());
+            //Red
+            Scalar rectangleBorderColor = new Scalar(0, 0, 255);
+
+            int rectangleBorderWidth = (int) Math.rint(region.getStrokeThickness());
+
+            Core.rectangle(sourceImage, topLeft, bottomRight,
+                    rectangleBorderColor, rectangleBorderWidth);
+        }
+
+        MatOfByte taggedMatrix = new MatOfByte();
+        Highgui.imencode(OPENCV_PNG, sourceImage, taggedMatrix);
+
+        return taggedMatrix;
+    }
+
+    /**
+     * Creates a thumbnail with tags applied.
+     *
+     * @param file Input file to apply tags & produce thumbnail from
+     * @param tagRegions Tags to apply
+     * @param iconSize Size of the output thumbnail
+     * @return BufferedImage Thumbnail image
+     *
+     * @throws InterruptedException Calling thread was interrupted.
+     * @throws ExecutionException Error while reading image from file.
+     */
+    public static BufferedImage getThumbnailWithTags(AbstractFile file, Collection<ImageTagRegion> tagRegions,
+            IconSize iconSize) throws IOException, InterruptedException, ExecutionException {
+
+        //Raw image
+        Mat sourceImage = getImageMatFromFile(file);
+        //Full size image with tags
+        MatOfByte taggedMatrix = getTaggedImageMatrix(sourceImage, tagRegions);
+        //Resized to produce thumbnail
+        MatOfByte thumbnailMatrix = getResizedMatrix(taggedMatrix, iconSize);
+
+        try (ByteArrayInputStream thumbnailStream = new ByteArrayInputStream(thumbnailMatrix.toArray())) {
+            return ImageIO.read(thumbnailStream);
+        } finally {
+            sourceImage.release();
+            taggedMatrix.release();
+            thumbnailMatrix.release();
+        }
+    }
+
+    /**
+     * Resizes the image matrix.
+     *
+     * @param taggedMatrix Image to resize.
+     * @param size Size of thumbnail.
+     *
+     * @return A new resized image matrix.
+     */
+    private static MatOfByte getResizedMatrix(MatOfByte taggedMatrix, IconSize size) {
+        Size resizeDimensions = new Size(size.getSize(), size.getSize());
+        Mat taggedImage = Highgui.imdecode(taggedMatrix, Highgui.IMREAD_COLOR);
+
+        Mat thumbnailImage = new Mat();
+        Imgproc.resize(taggedImage, thumbnailImage, resizeDimensions);
+
+        MatOfByte thumbnailMatrix = new MatOfByte();
+        Highgui.imencode(OPENCV_PNG, thumbnailImage, thumbnailMatrix);
+
+        thumbnailImage.release();
+        taggedImage.release();
+
+        return thumbnailMatrix;
+    }
+
+    private ImageTagsUtil() {
+    }
+
+    /**
+     * Sizes for thumbnails
+     */
+    public enum IconSize {
+        SMALL(50),
+        MEDIUM(100),
+        LARGE(200);
+
+        private final int SIZE;
+
+        IconSize(int size) {
+            this.SIZE = size;
+        }
+
+        public int getSize() {
+            return SIZE;
+        }
+    }
+}

Core/src/org/sleuthkit/autopsy/contentviewers/imagetagging/ImageTagsUtility.java

@@ -1,141 +0,0 @@
-/*
- * Autopsy Forensic Browser
- *
- * Copyright 2019 Basis Technology Corp.
- * Contact: carrier <at> sleuthkit <dot> org
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.sleuthkit.autopsy.contentviewers.imagetagging;
-
-import java.awt.image.BufferedImage;
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.util.Collection;
-import javax.imageio.ImageIO;
-import org.opencv.core.Core;
-import org.opencv.core.Mat;
-import org.opencv.core.MatOfByte;
-import org.opencv.core.MatOfInt;
-import org.opencv.core.Point;
-import org.opencv.core.Scalar;
-import org.opencv.core.Size;
-import org.opencv.highgui.Highgui;
-import org.opencv.imgproc.Imgproc;
-import org.sleuthkit.datamodel.AbstractFile;
-import org.sleuthkit.datamodel.TskCoreException;
-
-/**
- * Utility class for handling content viewer tags on images.
- */
-public final class ImageTagsUtility {
-
-    /**
-     * Sizes for thumbnails
-     */
-    public enum IconSize {
-        SMALL(50),
-        MEDIUM(100),
-        LARGE(200);
-
-        private final int SIZE;
-
-        IconSize(int size) {
-            this.SIZE = size;
-        }
-
-        public int getSize() {
-            return SIZE;
-        }
-    }
-
-    /**
-     * Embeds the tag regions into an image.
-     *
-     * @param file Base Image
-     * @param tagRegions Tag regions to be saved into the image
-     * @param outputEncoding Format of image (jpg, png, etc). See OpenCV for
-     * supported formats. Do not include a "."
-     * @return Output image as a BufferedImage
-     *
-     * @throws TskCoreException Cannot read from abstract file
-     * @throws IOException Could not create buffered image from OpenCV result
-     */
-    public static BufferedImage writeTags(AbstractFile file, Collection<ImageTagRegion> tagRegions,
-            String outputEncoding) throws TskCoreException, IOException {
-        byte[] imageInMemory = new byte[(int) file.getSize()];
-        file.read(imageInMemory, 0, file.getSize());
-        Mat originalImage = Highgui.imdecode(new MatOfByte(imageInMemory), Highgui.IMREAD_UNCHANGED);
-
-        tagRegions.forEach((region) -> {
-            Core.rectangle(
-                    originalImage, //Matrix obj of the image
-                    new Point(region.getX(), region.getY()), //p1
-                    new Point(region.getX() + region.getWidth(), region.getY() + region.getHeight()), //p2
-                    new Scalar(0, 0, 255), //Scalar object for color
-                    (int) Math.rint(region.getStrokeThickness())
-            );
-        });
-
-        MatOfByte matOfByte = new MatOfByte();
-        MatOfInt params = new MatOfInt(Highgui.IMWRITE_JPEG_QUALITY, 100);
-        Highgui.imencode("." + outputEncoding, originalImage, matOfByte, params);
-
-        try (ByteArrayInputStream imageStream = new ByteArrayInputStream(matOfByte.toArray())) {
-            BufferedImage result = ImageIO.read(imageStream);
-            originalImage.release();
-            matOfByte.release();
-            return result;
-        }
-    }
-
-    /**
-     * Creates a thumbnail version of the image with tags applied.
-     *
-     * @param file Input file to apply tags & produce thumbnail from
-     * @param tagRegions Tags to apply
-     * @param iconSize Size of the output thumbnail
-     * @param outputEncoding Format of thumbnail (jpg, png, etc). See OpenCV for
-     * supported formats. Do not include a "."
-     * @return BufferedImage representing the thumbnail
-     *
-     * @throws TskCoreException Could not read from file
-     * @throws IOException Could not create buffered image from OpenCV result
-     */
-    public static BufferedImage makeThumbnail(AbstractFile file, Collection<ImageTagRegion> tagRegions,
-            IconSize iconSize, String outputEncoding) throws TskCoreException, IOException {
-        try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
-            BufferedImage result = writeTags(file, tagRegions, outputEncoding);
-            ImageIO.write(result, outputEncoding, baos);
-            Mat markedUpImage = Highgui.imdecode(new MatOfByte(baos.toByteArray()), Highgui.IMREAD_UNCHANGED);
-            Mat thumbnail = new Mat();
-            Size resize = new Size(iconSize.getSize(), iconSize.getSize());
-
-            Imgproc.resize(markedUpImage, thumbnail, resize);
-            MatOfByte matOfByte = new MatOfByte();
-            Highgui.imencode("." + outputEncoding, thumbnail, matOfByte);
-
-            try (ByteArrayInputStream thumbnailStream = new ByteArrayInputStream(matOfByte.toArray())) {
-                BufferedImage thumbnailImage = ImageIO.read(thumbnailStream);
-                thumbnail.release();
-                matOfByte.release();
-                markedUpImage.release();
-                return thumbnailImage;
-            }
-        }
-    }
-
-    private ImageTagsUtility() {
-    }
-}

Core/src/org/sleuthkit/autopsy/report/ReportHTML.java

@@ -45,6 +45,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.TreeMap;
+import java.util.concurrent.ExecutionException;
 import java.util.logging.Level;
 import javax.imageio.ImageIO;
 import javax.swing.JPanel;
@@ -59,7 +60,7 @@ import org.sleuthkit.autopsy.casemodule.services.TagsManager;
 import org.sleuthkit.autopsy.casemodule.services.contentviewertags.ContentViewerTagManager;
 import org.sleuthkit.autopsy.casemodule.services.contentviewertags.ContentViewerTagManager.ContentViewerTag;
 import org.sleuthkit.autopsy.contentviewers.imagetagging.ImageTagRegion;
-import org.sleuthkit.autopsy.contentviewers.imagetagging.ImageTagsUtility;
+import org.sleuthkit.autopsy.contentviewers.imagetagging.ImageTagsUtil;
 import org.sleuthkit.autopsy.coreutils.EscapeUtil;
 import org.sleuthkit.autopsy.coreutils.ImageUtils;
 import org.sleuthkit.autopsy.coreutils.Logger;
@@ -805,10 +806,10 @@ class ReportHTML implements TableReportModule {
                 
                 if(!imageTags.isEmpty()) {
                     //Write the tags to the fullsize and thumbnail images
-                    BufferedImage fullImageWithTags = ImageTagsUtility.writeTags(file, imageTags, "png");
+                    BufferedImage fullImageWithTags = ImageTagsUtil.getImageWithTags(file, imageTags);
                     
-                    BufferedImage thumbnailImageWithTags = ImageTagsUtility.makeThumbnail(file, 
-                            imageTags, ImageTagsUtility.IconSize.MEDIUM, "png");
+                    BufferedImage thumbnailWithTags = ImageTagsUtil.getThumbnailWithTags(file, 
+                            imageTags, ImageTagsUtil.IconSize.MEDIUM);
                     
                     String fileName = org.sleuthkit.autopsy.coreutils.FileUtil.escapeFileName(file.getName());
                     
@@ -819,7 +820,7 @@ class ReportHTML implements TableReportModule {
                     File fullImageWithTagsFile = Paths.get(fullImageWithTagsPath).toFile();
                     
                     //Save images
-                    ImageIO.write(thumbnailImageWithTags, "png", thumbnailImageWithTagsFile);
+                    ImageIO.write(thumbnailWithTags, "png", thumbnailImageWithTagsFile);
                     ImageIO.write(fullImageWithTags, "png", fullImageWithTagsFile);
                     
                     thumbnailPath = THUMBS_REL_PATH + thumbnailImageWithTagsFile.getName();
@@ -828,7 +829,7 @@ class ReportHTML implements TableReportModule {
                 }
             } catch (TskCoreException ex) {
                 logger.log(Level.WARNING, "Could not get tags for file.", ex); //NON-NLS
-            } catch (IOException ex) {
+            } catch (IOException | InterruptedException | ExecutionException ex) {
                 logger.log(Level.WARNING, "Could make marked up thumbnail.", ex); //NON-NLS
             }