From 5c26ff25900d802b03cb535bb970e1eda91e2f15 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Thu, 15 Aug 2019 11:56:39 -0400 Subject: [PATCH] 5372 multiple thumbnails --- .../filequery/Bundle.properties-MERGED | 2 + .../filequery/FileDiscoveryDialog.java | 3 - .../autopsy/filequery/ResultFile.java | 181 +++++++++++++++++- .../autopsy/filequery/ResultsPanel.java | 6 +- .../autopsy/filequery/ThumbnailPanel.form | 8 +- .../autopsy/filequery/ThumbnailPanel.java | 30 ++- .../autopsy/filequery/ThumbnailsWrapper.java | 8 +- 7 files changed, 220 insertions(+), 18 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/filequery/Bundle.properties-MERGED b/Core/src/org/sleuthkit/autopsy/filequery/Bundle.properties-MERGED index 75000b45b7..ba763f1b6e 100644 --- a/Core/src/org/sleuthkit/autopsy/filequery/Bundle.properties-MERGED +++ b/Core/src/org/sleuthkit/autopsy/filequery/Bundle.properties-MERGED @@ -114,6 +114,8 @@ FileSorter.SortingMethod.filetype.displayName=By file type FileSorter.SortingMethod.frequency.displayName=By central repo frequency FileSorter.SortingMethod.keywordlist.displayName=By keyword list names FileSorter.SortingMethod.parent.displayName=By parent path +# {0} - file name +ResultFile.genVideoThumb.progress.text=extracting temporary file {0} ResultsDialog.dialogTitle.text=File search results ResultsDialog.exitButton.text=Exit ResultsDialog.searchButton.text=Run another search diff --git a/Core/src/org/sleuthkit/autopsy/filequery/FileDiscoveryDialog.java b/Core/src/org/sleuthkit/autopsy/filequery/FileDiscoveryDialog.java index 9372f84677..291f58b086 100644 --- a/Core/src/org/sleuthkit/autopsy/filequery/FileDiscoveryDialog.java +++ b/Core/src/org/sleuthkit/autopsy/filequery/FileDiscoveryDialog.java @@ -18,8 +18,6 @@ */ package org.sleuthkit.autopsy.filequery; -import java.util.ArrayList; -import java.util.List; import javax.swing.JFrame; import javax.swing.SwingUtilities; import javax.swing.event.ListSelectionEvent; @@ -31,7 +29,6 @@ import org.sleuthkit.autopsy.centralrepository.datamodel.EamDb; import org.sleuthkit.autopsy.corecomponents.DataContentPanel; import org.sleuthkit.autopsy.corecomponents.TableFilterNode; import org.sleuthkit.autopsy.datamodel.FileNode; -import org.sleuthkit.autopsy.directorytree.DataResultFilterNode; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.SleuthkitCase; diff --git a/Core/src/org/sleuthkit/autopsy/filequery/ResultFile.java b/Core/src/org/sleuthkit/autopsy/filequery/ResultFile.java index 64d5601657..fdab36d91e 100644 --- a/Core/src/org/sleuthkit/autopsy/filequery/ResultFile.java +++ b/Core/src/org/sleuthkit/autopsy/filequery/ResultFile.java @@ -18,14 +18,26 @@ */ package org.sleuthkit.autopsy.filequery; +import com.google.common.io.Files; import java.awt.Image; +import java.awt.image.BufferedImage; +import java.io.IOException; import org.sleuthkit.autopsy.filequery.FileSearchData.FileType; import org.sleuthkit.datamodel.AbstractFile; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.logging.Level; +import org.netbeans.api.progress.ProgressHandle; +import org.opencv.core.Mat; +import org.opencv.highgui.VideoCapture; +import org.openide.util.NbBundle; +import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; +import org.sleuthkit.autopsy.corelibs.ScalrWrapper; import org.sleuthkit.autopsy.coreutils.ImageUtils; +import static org.sleuthkit.autopsy.coreutils.VideoUtils.getVideoFileInTempDir; +import org.sleuthkit.autopsy.datamodel.ContentUtils; import org.sleuthkit.datamodel.HashUtility; /** @@ -210,10 +222,177 @@ class ResultFile { thumbnails.add(ImageUtils.getThumbnail(abstractFile, ImageUtils.ICON_SIZE_LARGE)); thumbnails.add(ImageUtils.getThumbnail(abstractFile, ImageUtils.ICON_SIZE_LARGE)); thumbnails.add(ImageUtils.getThumbnail(abstractFile, ImageUtils.ICON_SIZE_LARGE)); - } + } } + @NbBundle.Messages({"# {0} - file name", + "ResultFile.genVideoThumb.progress.text=extracting temporary file {0}"}) + static ThumbnailsWrapper createVideoThumbnails(AbstractFile file) { + java.io.File tempFile; + try { + tempFile = getVideoFileInTempDir(file); + } catch (NoCurrentCaseException ex) { +// LOGGER.log(Level.WARNING, "Exception while getting open case.", ex); //NON-NLS + int[] framePositions = new int[]{ + 0, + 0, + 0, + 0}; + List videoThumbnails = new ArrayList<>(); + videoThumbnails.add(ImageUtils.getDefaultThumbnail()); + videoThumbnails.add(ImageUtils.getDefaultThumbnail()); + videoThumbnails.add(ImageUtils.getDefaultThumbnail()); + videoThumbnails.add(ImageUtils.getDefaultThumbnail()); + return new ThumbnailsWrapper(videoThumbnails, framePositions, file); + } + if (tempFile.exists() == false || tempFile.length() < file.getSize()) { + ProgressHandle progress = ProgressHandle.createHandle(Bundle.ResultFile_genVideoThumb_progress_text(file.getName())); + progress.start(100); + try { + Files.createParentDirs(tempFile); + if (Thread.interrupted()) { + int[] framePositions = new int[]{ + 0, + 0, + 0, + 0}; + List videoThumbnails = new ArrayList<>(); + videoThumbnails.add(ImageUtils.getDefaultThumbnail()); + videoThumbnails.add(ImageUtils.getDefaultThumbnail()); + videoThumbnails.add(ImageUtils.getDefaultThumbnail()); + videoThumbnails.add(ImageUtils.getDefaultThumbnail()); + return new ThumbnailsWrapper(videoThumbnails, framePositions, file); + } + ContentUtils.writeToFile(file, tempFile, progress, null, true); + } catch (IOException ex) { +// LOGGER.log(Level.WARNING, "Error extracting temporary file for " + ImageUtils.getContentPathSafe(file), ex); //NON-NLS + } finally { + progress.finish(); + } + } + VideoCapture videoFile = new VideoCapture(); // will contain the video + BufferedImage bufferedImage = null; + + try { + if (!videoFile.open(tempFile.toString())) { +// LOGGER.log(Level.WARNING, "Error opening {0} for preview generation.", ImageUtils.getContentPathSafe(file)); //NON-NLS + int[] framePositions = new int[]{ + 0, + 0, + 0, + 0}; + List videoThumbnails = new ArrayList<>(); + videoThumbnails.add(ImageUtils.getDefaultThumbnail()); + videoThumbnails.add(ImageUtils.getDefaultThumbnail()); + videoThumbnails.add(ImageUtils.getDefaultThumbnail()); + videoThumbnails.add(ImageUtils.getDefaultThumbnail()); + return new ThumbnailsWrapper(videoThumbnails, framePositions, file); + } + double fps = videoFile.get(5); // gets frame per second + double totalFrames = videoFile.get(7); // gets total frames + if (fps <= 0 || totalFrames <= 0) { +// LOGGER.log(Level.WARNING, "Error getting fps or total frames for {0}", ImageUtils.getContentPathSafe(file)); //NON-NLS + int[] framePositions = new int[]{ + 0, + 0, + 0, + 0}; + List videoThumbnails = new ArrayList<>(); + videoThumbnails.add(ImageUtils.getDefaultThumbnail()); + videoThumbnails.add(ImageUtils.getDefaultThumbnail()); + videoThumbnails.add(ImageUtils.getDefaultThumbnail()); + videoThumbnails.add(ImageUtils.getDefaultThumbnail()); + return new ThumbnailsWrapper(videoThumbnails, framePositions, file); + } + if (Thread.interrupted()) { + int[] framePositions = new int[]{ + 0, + 0, + 0, + 0}; + List videoThumbnails = new ArrayList<>(); + videoThumbnails.add(ImageUtils.getDefaultThumbnail()); + videoThumbnails.add(ImageUtils.getDefaultThumbnail()); + videoThumbnails.add(ImageUtils.getDefaultThumbnail()); + videoThumbnails.add(ImageUtils.getDefaultThumbnail()); + return new ThumbnailsWrapper(videoThumbnails, framePositions, file); + } + + double duration = 1000 * (totalFrames / fps); //total milliseconds + + /* + * Four attempts are made to grab a frame from a video. The first + * attempt at 50% will give us a nice frame in the middle that gets + * to the heart of the content. If that fails, the next positions + * tried will be 25% and 75%. After three failed attempts, 1% will + * be tried in a last-ditch effort, the idea being the video may be + * corrupt and that our best chance at retrieving a frame is early + * on in the video. + * + * If no frame can be retrieved, no thumbnail will be created. + */ + int[] framePositions = new int[]{ + (int) (duration * .01), + (int) (duration * .25), + (int) (duration * .5), + (int) (duration * .75),}; + + Mat imageMatrix = new Mat(); + List videoThumbnails = new ArrayList<>(); + for (int i = 0; i < framePositions.length; i++) { + if (!videoFile.set(0, framePositions[i])) { +// LOGGER.log(Level.WARNING, "Error seeking to " + framePositions[i] + "ms in {0}", ImageUtils.getContentPathSafe(file)); //NON-NLS + // If we can't set the time, continue to the next frame position and try again. + + videoThumbnails.add(ImageUtils.getDefaultThumbnail()); + continue; + } + // Read the frame into the image/matrix. + if (!videoFile.read(imageMatrix)) { +// LOGGER.log(Level.WARNING, "Error reading frame at " + framePositions[i] + "ms from {0}", ImageUtils.getContentPathSafe(file)); //NON-NLS + // If the image is bad for some reason, continue to the next frame position and try again. + videoThumbnails.add(ImageUtils.getDefaultThumbnail()); + continue; + } + // If the image is empty, return since no buffered image can be created. + if (imageMatrix.empty()) { + videoThumbnails.add(ImageUtils.getDefaultThumbnail()); + continue; + } + + int matrixColumns = imageMatrix.cols(); + int matrixRows = imageMatrix.rows(); + + // Convert the matrix that contains the frame to a buffered image. + if (bufferedImage == null) { + bufferedImage = new BufferedImage(matrixColumns, matrixRows, BufferedImage.TYPE_3BYTE_BGR); + } + + byte[] data = new byte[matrixRows * matrixColumns * (int) (imageMatrix.elemSize())]; + imageMatrix.get(0, 0, data); //copy the image to data + + //todo: this looks like we are swapping the first and third channels. so we can use BufferedImage.TYPE_3BYTE_BGR + if (imageMatrix.channels() == 3) { + for (int k = 0; k < data.length; k += 3) { + byte temp = data[k]; + data[k] = data[k + 2]; + data[k + 2] = temp; + } + } + + bufferedImage.getRaster().setDataElements(0, 0, matrixColumns, matrixRows, data); + if (Thread.interrupted()) { + return new ThumbnailsWrapper(videoThumbnails, framePositions, file); + } + videoThumbnails.add(bufferedImage == null ? ImageUtils.getDefaultThumbnail() : ScalrWrapper.resizeFast(bufferedImage, ImageUtils.ICON_SIZE_MEDIUM)); + } + return new ThumbnailsWrapper(videoThumbnails, framePositions, file); + } finally { + videoFile.release(); // close the file} + } + } + List getThumbnails(FileSearchData.FileType resultType) { if (thumbnails.isEmpty()) { createThumbnails(resultType); diff --git a/Core/src/org/sleuthkit/autopsy/filequery/ResultsPanel.java b/Core/src/org/sleuthkit/autopsy/filequery/ResultsPanel.java index 9794821235..351355243a 100644 --- a/Core/src/org/sleuthkit/autopsy/filequery/ResultsPanel.java +++ b/Core/src/org/sleuthkit/autopsy/filequery/ResultsPanel.java @@ -421,7 +421,7 @@ public class ResultsPanel extends javax.swing.JPanel { private final ResultFile file; private final FileSearchData.FileType type; - private List thumbnails = new ArrayList<>(); + private ThumbnailsWrapper thumbnailWrapper; ThumbnailWorker(ResultFile file, FileSearchData.FileType resultType) { this.file = file; @@ -430,13 +430,13 @@ public class ResultsPanel extends javax.swing.JPanel { @Override protected Void doInBackground() throws Exception { - thumbnails.addAll(file.getThumbnails(type)); + thumbnailWrapper = ResultFile.createVideoThumbnails(file.getAbstractFile()); return null; } @Override protected void done() { - videoThumbnailViewer.addRow(new ThumbnailsWrapper(thumbnails, file.getAbstractFile())); + videoThumbnailViewer.addRow(thumbnailWrapper); } } diff --git a/Core/src/org/sleuthkit/autopsy/filequery/ThumbnailPanel.form b/Core/src/org/sleuthkit/autopsy/filequery/ThumbnailPanel.form index bea0b43ac8..912bf702eb 100644 --- a/Core/src/org/sleuthkit/autopsy/filequery/ThumbnailPanel.form +++ b/Core/src/org/sleuthkit/autopsy/filequery/ThumbnailPanel.form @@ -24,12 +24,12 @@ - + - + - + @@ -37,7 +37,7 @@ - + diff --git a/Core/src/org/sleuthkit/autopsy/filequery/ThumbnailPanel.java b/Core/src/org/sleuthkit/autopsy/filequery/ThumbnailPanel.java index 51f255d592..3883e2d9a2 100644 --- a/Core/src/org/sleuthkit/autopsy/filequery/ThumbnailPanel.java +++ b/Core/src/org/sleuthkit/autopsy/filequery/ThumbnailPanel.java @@ -10,6 +10,7 @@ import java.awt.Component; import java.awt.Image; import java.awt.GridBagConstraints; import java.util.List; +import java.util.concurrent.TimeUnit; import javax.swing.ImageIcon; import javax.swing.JLabel; import javax.swing.JList; @@ -32,19 +33,36 @@ public final class ThumbnailPanel extends javax.swing.JPanel implements ListCell this.setFocusable(true); } - private void addThumbnails(List thumbnails) { + private void addThumbnails(ThumbnailsWrapper thumbnailWrapper) { imagePanel.removeAll(); GridBagConstraints gridBagConstraints = new GridBagConstraints(); gridBagConstraints.gridx = 0; gridBagConstraints.gridy = 0; gridBagConstraints.anchor = GridBagConstraints.LINE_START; - imagePanel.add(new javax.swing.Box.Filler(new java.awt.Dimension(GAP_SIZE, 0), new java.awt.Dimension(GAP_SIZE, 0), new java.awt.Dimension(GAP_SIZE, 32767))); + imagePanel.add(new javax.swing.Box.Filler(new java.awt.Dimension(GAP_SIZE, 0), new java.awt.Dimension(GAP_SIZE, 0), new java.awt.Dimension(GAP_SIZE, 32767)), gridBagConstraints); + gridBagConstraints.gridy = 1; + imagePanel.add(new javax.swing.Box.Filler(new java.awt.Dimension(GAP_SIZE, 0), new java.awt.Dimension(GAP_SIZE, 0), new java.awt.Dimension(GAP_SIZE, 32767)), gridBagConstraints); gridBagConstraints.gridx++; - for (Image image : thumbnails) { + int timeIndex = 0; + int[] timeStamps = thumbnailWrapper.getTimeStamps(); + for (Image image : thumbnailWrapper.getThumbnails()) { + gridBagConstraints.gridy = 0; imagePanel.add(new JLabel(new ImageIcon(image)), gridBagConstraints); + gridBagConstraints.gridy = 1; + long millis = timeStamps[timeIndex]; + long hours = TimeUnit.MILLISECONDS.toHours(millis); + millis -= TimeUnit.HOURS.toMillis(hours); + long minutes = TimeUnit.MILLISECONDS.toMinutes(millis); + millis -= TimeUnit.MINUTES.toMillis(minutes); + long seconds = TimeUnit.MILLISECONDS.toSeconds(millis); + imagePanel.add(new JLabel(String.format("%01d:%02d:%02d", hours, minutes, seconds)), gridBagConstraints); gridBagConstraints.gridx++; - imagePanel.add(new javax.swing.Box.Filler(new java.awt.Dimension(GAP_SIZE, 0), new java.awt.Dimension(GAP_SIZE, 0), new java.awt.Dimension(GAP_SIZE, 32767))); + gridBagConstraints.gridy = 0; + imagePanel.add(new javax.swing.Box.Filler(new java.awt.Dimension(GAP_SIZE, 0), new java.awt.Dimension(GAP_SIZE, 0), new java.awt.Dimension(GAP_SIZE, 32767)), gridBagConstraints); + gridBagConstraints.gridy = 1; + imagePanel.add(new javax.swing.Box.Filler(new java.awt.Dimension(GAP_SIZE, 0), new java.awt.Dimension(GAP_SIZE, 0), new java.awt.Dimension(GAP_SIZE, 32767)), gridBagConstraints); gridBagConstraints.gridx++; + timeIndex++; } } @@ -81,7 +99,7 @@ public final class ThumbnailPanel extends javax.swing.JPanel implements ListCell layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addGroup(layout.createSequentialGroup() .addContainerGap() - .addComponent(imagePanel, javax.swing.GroupLayout.PREFERRED_SIZE, 148, javax.swing.GroupLayout.PREFERRED_SIZE) + .addComponent(imagePanel, javax.swing.GroupLayout.PREFERRED_SIZE, 170, javax.swing.GroupLayout.PREFERRED_SIZE) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) .addComponent(fileInfoLabel, javax.swing.GroupLayout.PREFERRED_SIZE, 19, javax.swing.GroupLayout.PREFERRED_SIZE) .addContainerGap(javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)) @@ -97,7 +115,7 @@ public final class ThumbnailPanel extends javax.swing.JPanel implements ListCell @Override public Component getListCellRendererComponent(JList list, ThumbnailsWrapper value, int index, boolean isSelected, boolean cellHasFocus) { fileInfoLabel.setText(value.getFileInfo()); - addThumbnails(value.getThumbnails()); + addThumbnails(value); setBackground(isSelected ? SELECTION_COLOR : list.getBackground()); return this; } diff --git a/Core/src/org/sleuthkit/autopsy/filequery/ThumbnailsWrapper.java b/Core/src/org/sleuthkit/autopsy/filequery/ThumbnailsWrapper.java index 8ca6313f49..0aaa255b71 100644 --- a/Core/src/org/sleuthkit/autopsy/filequery/ThumbnailsWrapper.java +++ b/Core/src/org/sleuthkit/autopsy/filequery/ThumbnailsWrapper.java @@ -18,9 +18,11 @@ public class ThumbnailsWrapper { private final List thumbnails; private final AbstractFile abstractFile; + private final int[] timeStamps; - public ThumbnailsWrapper(List thumbnails, AbstractFile file) { + public ThumbnailsWrapper(List thumbnails, int[] timeStamps, AbstractFile file) { this.thumbnails = thumbnails; + this.timeStamps = timeStamps; this.abstractFile = file; } @@ -28,6 +30,10 @@ public class ThumbnailsWrapper { return abstractFile; } + int[] getTimeStamps(){ + return timeStamps.clone(); + } + String getFileInfo() { return abstractFile.getParentPath(); }