mirror of
https://github.com/overcuriousity/autopsy-flatpak.git
synced 2025-07-12 16:06:15 +00:00
Merge remote-tracking branch 'upstream/master'
This commit is contained in:
commit
fdfd758911
@ -2,7 +2,7 @@ Manifest-Version: 1.0
|
||||
OpenIDE-Module: org.sleuthkit.autopsy.core/7
|
||||
OpenIDE-Module-Localizing-Bundle: org/sleuthkit/autopsy/core/Bundle.properties
|
||||
OpenIDE-Module-Layer: org/sleuthkit/autopsy/core/layer.xml
|
||||
OpenIDE-Module-Implementation-Version: 6
|
||||
OpenIDE-Module-Implementation-Version: 7
|
||||
OpenIDE-Module-Requires: org.openide.windows.WindowManager, org.netbeans.api.javahelp.Help
|
||||
AutoUpdate-Show-In-Client: true
|
||||
AutoUpdate-Essential-Module: true
|
||||
|
@ -6,5 +6,5 @@ license.file=../LICENSE-2.0.txt
|
||||
nbm.homepage=http://www.sleuthkit.org/
|
||||
nbm.module.author=Brian Carrier
|
||||
nbm.needs.restart=true
|
||||
spec.version.base=5.1
|
||||
spec.version.base=5.2
|
||||
|
||||
|
@ -30,6 +30,7 @@ import org.openide.explorer.ExplorerManager;
|
||||
import org.openide.explorer.ExplorerManager.Provider;
|
||||
import org.openide.nodes.Node;
|
||||
import org.openide.util.Lookup;
|
||||
import org.sleuthkit.autopsy.casemodule.Case;
|
||||
import org.sleuthkit.autopsy.corecomponentinterfaces.DataContent;
|
||||
import org.sleuthkit.autopsy.corecomponentinterfaces.DataResultViewer;
|
||||
import org.sleuthkit.autopsy.coreutils.Logger;
|
||||
@ -43,10 +44,9 @@ public abstract class AbstractDataResultViewer extends JPanel implements
|
||||
private static final Logger logger = Logger.getLogger(AbstractDataResultViewer.class.getName());
|
||||
protected transient ExplorerManager em = new ExplorerManager();
|
||||
private PropertyChangeListener nodeSelListener;
|
||||
|
||||
/**
|
||||
* Content viewer to respond to selection events
|
||||
* Either the main one, or custom one if set
|
||||
* Content viewer to respond to selection events Either the main one, or
|
||||
* custom one if set
|
||||
*/
|
||||
protected DataContent contentViewer;
|
||||
|
||||
@ -54,17 +54,22 @@ public abstract class AbstractDataResultViewer extends JPanel implements
|
||||
|
||||
//DataContent is designed to return only the default viewer from lookup
|
||||
//use the default one unless set otherwise
|
||||
contentViewer = Lookup.getDefault().lookup(DataContent.class);
|
||||
|
||||
contentViewer = Lookup.getDefault().lookup(DataContent.class);
|
||||
|
||||
//property listener to send nodes to content viewer
|
||||
nodeSelListener = new PropertyChangeListener() {
|
||||
|
||||
/**
|
||||
* Propagates changes in the current select node from the
|
||||
* DataResultViewer to the DataContentTopComponent
|
||||
*/
|
||||
@Override
|
||||
public void propertyChange(PropertyChangeEvent evt) {
|
||||
if (!Case.isCaseOpen()) {
|
||||
//handle in-between condition when case is being closed
|
||||
//and legacy selection events are pumped
|
||||
return;
|
||||
}
|
||||
|
||||
String changed = evt.getPropertyName();
|
||||
|
||||
// change that should affect view
|
||||
@ -77,10 +82,10 @@ public abstract class AbstractDataResultViewer extends JPanel implements
|
||||
setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR));
|
||||
try {
|
||||
Node selectedNode = getSelectedNode();
|
||||
|
||||
|
||||
nodeSelected(selectedNode);
|
||||
|
||||
|
||||
|
||||
|
||||
if (selectedNode != null) {
|
||||
// there's a new/changed node to display
|
||||
@ -142,10 +147,11 @@ public abstract class AbstractDataResultViewer extends JPanel implements
|
||||
@Override
|
||||
public void resetComponent() {
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Called when a new node has been selected in the result viewer
|
||||
* Can update the viewer, etc.
|
||||
* Called when a new node has been selected in the result viewer Can update
|
||||
* the viewer, etc.
|
||||
*
|
||||
* @param selectedNode the new node currently selected
|
||||
*/
|
||||
public abstract void nodeSelected(Node selectedNode);
|
||||
@ -168,7 +174,7 @@ public abstract class AbstractDataResultViewer extends JPanel implements
|
||||
logger.log(Level.WARNING, "Couldn't set selected nodes.", ex);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public void setContentViewer(DataContent contentViewer) {
|
||||
this.contentViewer = contentViewer;
|
||||
|
@ -93,6 +93,7 @@ public class DataContentViewerMedia extends javax.swing.JPanel implements DataCo
|
||||
private BufferedImage currentImage = null;
|
||||
private boolean gstInited = false;
|
||||
private AbstractFile lastFile;
|
||||
private boolean inImageMode; //keeps track if already in image mode to minimize UI setup
|
||||
|
||||
/**
|
||||
* Creates new form DataContentViewerVideo
|
||||
@ -103,6 +104,7 @@ public class DataContentViewerMedia extends javax.swing.JPanel implements DataCo
|
||||
}
|
||||
|
||||
private void customizeComponents() {
|
||||
inImageMode = false;
|
||||
|
||||
Platform.setImplicitExit(false);
|
||||
PlatformImpl.startup(new Runnable() {
|
||||
@ -112,17 +114,17 @@ public class DataContentViewerMedia extends javax.swing.JPanel implements DataCo
|
||||
}
|
||||
});
|
||||
logger.log(Level.INFO, "Supported image formats by javafx image viewer: ");
|
||||
|
||||
|
||||
//initialize supported image types
|
||||
//TODO use mime-types instead once we have support
|
||||
String[] fxSupportedImagesSuffixes = ImageIO.getReaderFileSuffixes();
|
||||
IMAGES = new String[fxSupportedImagesSuffixes.length];
|
||||
for (int i=0; i<fxSupportedImagesSuffixes.length; ++i) {
|
||||
for (int i = 0; i < fxSupportedImagesSuffixes.length; ++i) {
|
||||
String suffix = fxSupportedImagesSuffixes[i];
|
||||
logger.log(Level.INFO, "suffix: " + suffix);
|
||||
IMAGES[i] = "." + suffix;
|
||||
}
|
||||
|
||||
|
||||
try {
|
||||
logger.log(Level.INFO, "Initializing gstreamer for video/audio viewing");
|
||||
Gst.init();
|
||||
@ -261,7 +263,7 @@ public class DataContentViewerMedia extends javax.swing.JPanel implements DataCo
|
||||
if (selectedNode == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
AbstractFile file = selectedNode.getLookup().lookup(AbstractFile.class);
|
||||
if (file == null) {
|
||||
return;
|
||||
@ -275,7 +277,7 @@ public class DataContentViewerMedia extends javax.swing.JPanel implements DataCo
|
||||
|
||||
reset();
|
||||
setComponentsVisibility(false);
|
||||
|
||||
|
||||
// get rid of any existing videoProgressWorker thread
|
||||
if (videoProgressWorker != null) {
|
||||
videoProgressWorker.cancel(true);
|
||||
@ -288,6 +290,7 @@ public class DataContentViewerMedia extends javax.swing.JPanel implements DataCo
|
||||
showImageFx(file);
|
||||
} else if (gstInited
|
||||
&& (containsExt(file.getName(), VIDEOS) || containsExt(file.getName(), AUDIOS))) {
|
||||
inImageMode = false;
|
||||
setupVideo(file);
|
||||
}
|
||||
}
|
||||
@ -299,13 +302,19 @@ public class DataContentViewerMedia extends javax.swing.JPanel implements DataCo
|
||||
*/
|
||||
private void showImageFx(final AbstractFile file) {
|
||||
final String fileName = file.getName();
|
||||
|
||||
if (!Case.isCaseOpen()) {
|
||||
//handle in-between condition when case is being closed
|
||||
//and an image was previously selected
|
||||
return;
|
||||
}
|
||||
|
||||
// load the image
|
||||
PlatformImpl.runLater(new Runnable() {
|
||||
@Override
|
||||
public void run() {
|
||||
Dimension dims = DataContentViewerMedia.this.getSize();
|
||||
|
||||
|
||||
final InputStream inputStream = new ReadContentInputStream(file);
|
||||
|
||||
final Image fxImage;
|
||||
@ -313,19 +322,24 @@ public class DataContentViewerMedia extends javax.swing.JPanel implements DataCo
|
||||
//original input stream
|
||||
BufferedImage bi = ImageIO.read(inputStream);
|
||||
//scale image using Scalr
|
||||
BufferedImage biScaled = ScalrWrapper.resizeHighQuality(bi, (int)dims.getWidth(), (int)dims.getHeight());
|
||||
BufferedImage biScaled = ScalrWrapper.resizeHighQuality(bi, (int) dims.getWidth(), (int) dims.getHeight());
|
||||
//convert from awt imageto fx image
|
||||
fxImage = SwingFXUtils.toFXImage(biScaled, null);
|
||||
} catch (IOException ex) {
|
||||
logger.log(Level.WARNING, "Could not load image file into media view: " + fileName, ex);
|
||||
return;
|
||||
}
|
||||
catch (OutOfMemoryError ex) {
|
||||
} catch (OutOfMemoryError ex) {
|
||||
logger.log(Level.WARNING, "Could not load image file into media view (too large): " + fileName, ex);
|
||||
MessageNotifyUtil.Notify.warn("Could not load image file (too large): " + file.getName(), ex.getMessage());
|
||||
MessageNotifyUtil.Notify.warn("Could not load image file (too large): " + file.getName(), ex.getMessage());
|
||||
return;
|
||||
} finally {
|
||||
try {
|
||||
inputStream.close();
|
||||
} catch (IOException ex) {
|
||||
logger.log(Level.WARNING, "Could not close input stream after loading image in media view: " + fileName, ex);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if (fxImage == null) {
|
||||
logger.log(Level.WARNING, "Could not load image file into media view: " + fileName);
|
||||
return;
|
||||
@ -346,27 +360,36 @@ public class DataContentViewerMedia extends javax.swing.JPanel implements DataCo
|
||||
Group fxRoot = new Group();
|
||||
|
||||
//Scene fxScene = new Scene(fxRoot, dims.getWidth(), dims.getHeight(), javafx.scene.paint.Color.BLACK);
|
||||
Scene fxScene = new Scene(fxRoot,javafx.scene.paint.Color.BLACK);
|
||||
Scene fxScene = new Scene(fxRoot, javafx.scene.paint.Color.BLACK);
|
||||
fxRoot.getChildren().add(fxImageView);
|
||||
|
||||
final JFXPanel fxPanel = new JFXPanel();
|
||||
fxPanel.setScene(fxScene);
|
||||
if (inImageMode) {
|
||||
final JFXPanel fxPanel = (JFXPanel) videoPanel.getComponent(0);
|
||||
fxPanel.setScene(fxScene);
|
||||
videoPanel.setVisible(true);
|
||||
} else {
|
||||
inImageMode = true;
|
||||
final JFXPanel fxPanel = new JFXPanel();
|
||||
fxPanel.setScene(fxScene);
|
||||
|
||||
//when done, join with the swing panel
|
||||
EventQueue.invokeLater(new Runnable() {
|
||||
@Override
|
||||
public void run() {
|
||||
//remove video panels and recreate image view panel
|
||||
//TODO use swing layered pane to switch between different modes
|
||||
videoPanel.removeAll();
|
||||
videoPanel.setLayout(new BoxLayout(videoPanel, BoxLayout.Y_AXIS));
|
||||
videoPanel.add(fxPanel);
|
||||
videoPanel.setVisible(true);
|
||||
|
||||
//when done, join with the swing panel
|
||||
EventQueue.invokeLater(new Runnable() {
|
||||
@Override
|
||||
public void run() {
|
||||
videoPanel.removeAll();
|
||||
videoPanel.setLayout(new BoxLayout(videoPanel, BoxLayout.Y_AXIS));
|
||||
videoPanel.add(fxPanel);
|
||||
videoPanel.setVisible(true);
|
||||
|
||||
if (fxImage.isError() ) {
|
||||
logger.log(Level.WARNING, "Could not load image file into media view: " + fileName);
|
||||
//MessageNotifyUtil.Message.warn("Could not load image file: " + file.getName());
|
||||
if (fxImage.isError()) {
|
||||
logger.log(Level.WARNING, "Could not load image file into media view: " + fileName);
|
||||
//MessageNotifyUtil.Message.warn("Could not load image file: " + file.getName());
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
|
@ -22,6 +22,8 @@ import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.logging.Level;
|
||||
import javax.imageio.ImageIO;
|
||||
import org.openide.nodes.AbstractNode;
|
||||
import org.openide.nodes.Children;
|
||||
import org.openide.nodes.Node;
|
||||
@ -154,7 +156,17 @@ class ThumbnailViewChildren extends Children.Keys<Integer> {
|
||||
|
||||
private static class IsSupportedContentVisitor extends ContentVisitor.Default<Boolean> {
|
||||
|
||||
private static final List<String> SUPP_EXTENSIONS = Arrays.asList(".jpeg", ".jpg", ".gif", ".png");
|
||||
private final List<String> SUPP_EXTENSIONS;
|
||||
|
||||
IsSupportedContentVisitor() {
|
||||
String[] supportedImagesSuffixes = ImageIO.getReaderFileSuffixes();
|
||||
|
||||
SUPP_EXTENSIONS = new ArrayList<String>(supportedImagesSuffixes.length);
|
||||
for (int i = 0; i < supportedImagesSuffixes.length; ++i) {
|
||||
String suffix = supportedImagesSuffixes[i];
|
||||
SUPP_EXTENSIONS.add("." + suffix);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Boolean visit(DerivedFile f) {
|
||||
|
@ -26,6 +26,7 @@ import java.awt.Toolkit;
|
||||
import java.awt.image.BufferedImage;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.lang.ref.SoftReference;
|
||||
import java.util.logging.Level;
|
||||
import javax.imageio.ImageIO;
|
||||
@ -34,33 +35,41 @@ import javax.swing.JFrame;
|
||||
import org.openide.nodes.Children;
|
||||
import org.openide.nodes.FilterNode;
|
||||
import org.openide.nodes.Node;
|
||||
import org.openide.util.Exceptions;
|
||||
import org.sleuthkit.autopsy.casemodule.Case;
|
||||
import org.sleuthkit.autopsy.corelibs.ScalrWrapper;
|
||||
import org.sleuthkit.autopsy.coreutils.Logger;
|
||||
import org.sleuthkit.datamodel.Content;
|
||||
import org.sleuthkit.datamodel.ReadContentInputStream;
|
||||
import org.sleuthkit.datamodel.TskException;
|
||||
|
||||
/**
|
||||
* Node that wraps around original node and adds the bitmap icon representing the picture
|
||||
* Node that wraps around original node and adds the bitmap icon representing
|
||||
* the picture
|
||||
*/
|
||||
class ThumbnailViewNode extends FilterNode {
|
||||
|
||||
private SoftReference<Image> iconCache;
|
||||
|
||||
private static final Image defaultIcon = new ImageIcon("/org/sleuthkit/autopsy/images/file-icon.png").getImage();
|
||||
private static final Logger logger = Logger.getLogger(ThumbnailViewNode.class.getName());
|
||||
//private final BufferedImage defaultIconBI;
|
||||
|
||||
/** the constructor */
|
||||
/**
|
||||
* the constructor
|
||||
*/
|
||||
ThumbnailViewNode(Node arg) {
|
||||
super(arg, Children.LEAF);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getDisplayName(){
|
||||
if(super.getDisplayName().length() > 15)
|
||||
public String getDisplayName() {
|
||||
if (super.getDisplayName().length() > 15) {
|
||||
return super.getDisplayName().substring(0, 15).concat("...");
|
||||
else
|
||||
} else {
|
||||
return super.getDisplayName();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public Image getIcon(int type) {
|
||||
Image icon = null;
|
||||
@ -68,12 +77,11 @@ class ThumbnailViewNode extends FilterNode {
|
||||
if (iconCache != null) {
|
||||
icon = iconCache.get();
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
if (icon == null) {
|
||||
Content content = this.getLookup().lookup(Content.class);
|
||||
|
||||
|
||||
if (content != null) {
|
||||
if (getFile(content.getId()).exists()) {
|
||||
try {
|
||||
@ -84,85 +92,56 @@ class ThumbnailViewNode extends FilterNode {
|
||||
} else {
|
||||
try {
|
||||
icon = generateIcon(content);
|
||||
ImageIO.write(toBufferedImage(icon), "jpg", getFile(content.getId()));
|
||||
} catch (TskException ex) {
|
||||
icon = ThumbnailViewNode.defaultIcon;
|
||||
if (icon == null) {
|
||||
icon = ThumbnailViewNode.defaultIcon;
|
||||
} else {
|
||||
ImageIO.write((BufferedImage) icon, "jpg", getFile(content.getId()));
|
||||
}
|
||||
} catch (IOException ex) {
|
||||
logger.log(Level.WARNING, "Could not write cache thumbnail: " + content, ex);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
icon = ThumbnailViewNode.defaultIcon;
|
||||
}
|
||||
|
||||
|
||||
iconCache = new SoftReference<Image>(icon);
|
||||
}
|
||||
|
||||
return icon;
|
||||
}
|
||||
|
||||
static private Image generateIcon(Content content) throws TskException {
|
||||
byte[] data = new byte[(int)content.getSize()];
|
||||
int bytesRead = content.read(data, 0, content.getSize());
|
||||
|
||||
if (bytesRead < 1)
|
||||
/*
|
||||
* Generate a scaled image
|
||||
*/
|
||||
static private BufferedImage generateIcon(Content content) {
|
||||
|
||||
InputStream inputStream = null;
|
||||
try {
|
||||
inputStream = new ReadContentInputStream(content);
|
||||
BufferedImage bi = ImageIO.read(inputStream);
|
||||
BufferedImage biScaled = ScalrWrapper.resizeFast(bi, 100, 100);
|
||||
return biScaled;
|
||||
}catch (OutOfMemoryError e) {
|
||||
logger.log(Level.WARNING, "Could not scale image (too large): " + content.getName(), e);
|
||||
return null;
|
||||
|
||||
Image result = Toolkit.getDefaultToolkit().createImage(data);
|
||||
}
|
||||
catch (Exception e) {
|
||||
logger.log(Level.WARNING, "Could not scale image: " + content.getName(), e);
|
||||
return null;
|
||||
} finally {
|
||||
if (inputStream != null) {
|
||||
try {
|
||||
inputStream.close();
|
||||
} catch (IOException ex) {
|
||||
logger.log(Level.WARNING, "Could not close input stream after resizing thumbnail: " + content.getName(), ex);
|
||||
}
|
||||
}
|
||||
|
||||
// scale the image
|
||||
MediaTracker mTracker = new MediaTracker(new JFrame());
|
||||
mTracker.addImage(result, 1);
|
||||
try {
|
||||
mTracker.waitForID(1);
|
||||
} catch (InterruptedException ex) {
|
||||
// TODO: maybe make bubble instead
|
||||
Logger.getLogger(ThumbnailViewNode.class.getName()).log(Level.WARNING, "Error while trying to scale the icon.", ex);
|
||||
}
|
||||
int width = result.getWidth(null);
|
||||
int height = result.getHeight(null);
|
||||
|
||||
int max = Math.max(width, height);
|
||||
double scale = (75 * 100) / max;
|
||||
|
||||
// getScaledInstance can't take have width or height be 0, so round
|
||||
// up by adding 1 after truncating to int.
|
||||
width = (int) ((width * scale) / 100) + 1;
|
||||
height = (int) ((height * scale) / 100) + 1;
|
||||
|
||||
result = result.getScaledInstance(width, height, Image.SCALE_SMOOTH);
|
||||
|
||||
// load the image completely
|
||||
mTracker.addImage(result, 1);
|
||||
try {
|
||||
mTracker.waitForID(1);
|
||||
} catch (InterruptedException ex) {
|
||||
// TODO: maybe make bubble instead
|
||||
Logger.getLogger(ThumbnailViewNode.class.getName()).log(Level.WARNING, "Error while trying to load the icon.", ex);
|
||||
}
|
||||
|
||||
// create 75x75 image for the icon with the icon on the center
|
||||
BufferedImage combined = new BufferedImage(75, 75, BufferedImage.TYPE_INT_ARGB);
|
||||
Graphics2D g = (Graphics2D) combined.getGraphics();
|
||||
g.setColor(Color.WHITE);
|
||||
g.setBackground(Color.WHITE);
|
||||
g.drawImage(result, (75 - width) / 2, (75 - height) / 2, null);
|
||||
|
||||
return Toolkit.getDefaultToolkit().createImage(combined.getSource());
|
||||
}
|
||||
|
||||
private static BufferedImage toBufferedImage(Image src) {
|
||||
int w = src.getWidth(null);
|
||||
int h = src.getHeight(null);
|
||||
int type = BufferedImage.TYPE_INT_RGB; // other options
|
||||
BufferedImage dest = new BufferedImage(w, h, type);
|
||||
Graphics2D g2 = dest.createGraphics();
|
||||
g2.drawImage(src, 0, 0, null);
|
||||
g2.dispose();
|
||||
return dest;
|
||||
}
|
||||
|
||||
private static File getFile(long id) {
|
||||
return new File(Case.getCurrentCase().getCacheDirectory() + File.separator + id + ".jpg");
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -595,14 +595,18 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat
|
||||
|
||||
/**
|
||||
* Event handler to run when selection changed
|
||||
*
|
||||
*
|
||||
* TODO this needs to be revised
|
||||
*
|
||||
*
|
||||
* @param oldNodes
|
||||
* @param newNodes
|
||||
* @param newNodes
|
||||
*/
|
||||
private void respondSelection(final Node[] oldNodes, final Node[] newNodes) {
|
||||
|
||||
if (!Case.isCaseOpen()) {
|
||||
//handle in-between condition when case is being closed
|
||||
//and legacy selection events are pumped
|
||||
return;
|
||||
}
|
||||
//this looks redundant?
|
||||
// if (getSelectedNode() == null && oldNodes != null) {
|
||||
// try {
|
||||
|
@ -899,7 +899,14 @@ public class IngestManager {
|
||||
logger.log(Level.SEVERE, "Error: unexpected exception from module: " + module.getName(), e);
|
||||
stats.addError(module);
|
||||
}
|
||||
}
|
||||
catch (OutOfMemoryError e) {
|
||||
logger.log(Level.SEVERE, "Error: out of memory from module: " + module.getName(), e);
|
||||
stats.addError(module);
|
||||
}
|
||||
} //end for every module
|
||||
|
||||
//free the internal file resource after done with every module
|
||||
fileToProcess.close();
|
||||
|
||||
int newTotalEnqueuedFiles = fileScheduler.getFilesEnqueuedEst();
|
||||
if (newTotalEnqueuedFiles > totalEnqueuedFiles) {
|
||||
@ -915,7 +922,8 @@ public class IngestManager {
|
||||
}
|
||||
//--totalEnqueuedFiles;
|
||||
|
||||
} //end of this AbstractFile
|
||||
|
||||
} //end of for every AbstractFile
|
||||
logger.log(Level.INFO, "IngestManager: Finished processing files");
|
||||
return null;
|
||||
}
|
||||
@ -933,8 +941,7 @@ public class IngestManager {
|
||||
}
|
||||
|
||||
logger.log(Level.INFO, PlatformUtil.getAllMemUsageInfo());
|
||||
logger.log(Level.INFO, "Freeing resources post file pipeline run");
|
||||
System.gc();
|
||||
logger.log(Level.INFO, "Freeing jvm heap resources post file pipeline run");
|
||||
System.gc();
|
||||
logger.log(Level.INFO, PlatformUtil.getAllMemUsageInfo());
|
||||
|
||||
|
@ -30,11 +30,11 @@ import org.imgscalr.Scalr.Method;
|
||||
public class ScalrWrapper {
|
||||
|
||||
public static synchronized BufferedImage resize(BufferedImage input, int width, int height) {
|
||||
return Scalr.resize(input, width, height);
|
||||
return Scalr.resize(input, width, height, Scalr.OP_ANTIALIAS);
|
||||
}
|
||||
|
||||
public static synchronized BufferedImage resize(BufferedImage input, int size) {
|
||||
return Scalr.resize(input, size);
|
||||
return Scalr.resize(input, size, Scalr.OP_ANTIALIAS);
|
||||
}
|
||||
|
||||
public static synchronized BufferedImage resizeHighQuality(BufferedImage input, int width, int height) {
|
||||
@ -42,6 +42,10 @@ public class ScalrWrapper {
|
||||
}
|
||||
|
||||
public static synchronized BufferedImage resizeFast(BufferedImage input, int size) {
|
||||
return Scalr.resize(input, Method.SPEED, size, Scalr.OP_ANTIALIAS);
|
||||
return Scalr.resize(input, Method.SPEED, Scalr.Mode.AUTOMATIC, size, Scalr.OP_ANTIALIAS);
|
||||
}
|
||||
|
||||
public static synchronized BufferedImage resizeFast(BufferedImage input, int width, int height) {
|
||||
return Scalr.resize(input, Method.SPEED, Scalr.Mode.AUTOMATIC, width, height, Scalr.OP_ANTIALIAS);
|
||||
}
|
||||
}
|
||||
|
@ -1,6 +1,6 @@
|
||||
Manifest-Version: 1.0
|
||||
AutoUpdate-Show-In-Client: true
|
||||
OpenIDE-Module: org.sleuthkit.autopsy.exifparser/3
|
||||
OpenIDE-Module-Implementation-Version: 6
|
||||
OpenIDE-Module-Implementation-Version: 7
|
||||
OpenIDE-Module-Layer: org/sleuthkit/autopsy/exifparser/layer.xml
|
||||
OpenIDE-Module-Localizing-Bundle: org/sleuthkit/autopsy/exifparser/Bundle.properties
|
||||
|
@ -3,4 +3,4 @@ javac.compilerargs=-Xlint -Xlint:-serial
|
||||
license.file=../LICENSE-2.0.txt
|
||||
nbm.homepage=http://www.sleuthkit.org/autopsy/
|
||||
nbm.needs.restart=true
|
||||
spec.version.base=1.0
|
||||
spec.version.base=1.1
|
||||
|
@ -12,7 +12,7 @@
|
||||
<compile-dependency/>
|
||||
<run-dependency>
|
||||
<release-version>7</release-version>
|
||||
<specification-version>5.0</specification-version>
|
||||
<specification-version>5.2</specification-version>
|
||||
</run-dependency>
|
||||
</dependency>
|
||||
</module-dependencies>
|
||||
|
@ -1,7 +1,7 @@
|
||||
Manifest-Version: 1.0
|
||||
AutoUpdate-Show-In-Client: true
|
||||
OpenIDE-Module: org.sleuthkit.autopsy.hashdatabase/3
|
||||
OpenIDE-Module-Implementation-Version: 6
|
||||
OpenIDE-Module-Implementation-Version: 7
|
||||
OpenIDE-Module-Layer: org/sleuthkit/autopsy/hashdatabase/layer.xml
|
||||
OpenIDE-Module-Localizing-Bundle: org/sleuthkit/autopsy/hashdatabase/Bundle.properties
|
||||
|
||||
|
@ -3,4 +3,4 @@ javac.compilerargs=-Xlint -Xlint:-serial
|
||||
license.file=../LICENSE-2.0.txt
|
||||
nbm.homepage=http://www.sleuthkit.org/autopsy/
|
||||
nbm.needs.restart=true
|
||||
spec.version.base=1.1
|
||||
spec.version.base=1.2
|
||||
|
@ -78,7 +78,7 @@
|
||||
<compile-dependency/>
|
||||
<run-dependency>
|
||||
<release-version>7</release-version>
|
||||
<specification-version>5.0</specification-version>
|
||||
<specification-version>5.2</specification-version>
|
||||
</run-dependency>
|
||||
</dependency>
|
||||
</module-dependencies>
|
||||
|
@ -1,7 +1,7 @@
|
||||
Manifest-Version: 1.0
|
||||
AutoUpdate-Show-In-Client: true
|
||||
OpenIDE-Module: org.sleuthkit.autopsy.keywordsearch/5
|
||||
OpenIDE-Module-Implementation-Version: 6
|
||||
OpenIDE-Module-Implementation-Version: 7
|
||||
OpenIDE-Module-Install: org/sleuthkit/autopsy/keywordsearch/Installer.class
|
||||
OpenIDE-Module-Layer: org/sleuthkit/autopsy/keywordsearch/layer.xml
|
||||
OpenIDE-Module-Localizing-Bundle: org/sleuthkit/autopsy/keywordsearch/Bundle.properties
|
||||
|
@ -3,4 +3,4 @@ javac.compilerargs=-Xlint -Xlint:-serial
|
||||
license.file=../LICENSE-2.0.txt
|
||||
nbm.homepage=http://www.sleuthkit.org/autopsy/
|
||||
nbm.needs.restart=true
|
||||
spec.version.base=3.0
|
||||
spec.version.base=3.1
|
||||
|
@ -96,7 +96,7 @@
|
||||
<compile-dependency/>
|
||||
<run-dependency>
|
||||
<release-version>7</release-version>
|
||||
<specification-version>5.0</specification-version>
|
||||
<specification-version>5.2</specification-version>
|
||||
</run-dependency>
|
||||
</dependency>
|
||||
</module-dependencies>
|
||||
|
@ -16,9 +16,9 @@
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.sleuthkit.autopsy.keywordsearch;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import org.sleuthkit.autopsy.coreutils.StringExtract.StringExtractUnicodeTable.SCRIPT;
|
||||
@ -29,72 +29,124 @@ import org.sleuthkit.datamodel.AbstractFile;
|
||||
* chunks
|
||||
*/
|
||||
interface AbstractFileExtract {
|
||||
|
||||
|
||||
/**
|
||||
* Common options that can be used by some extractors
|
||||
*/
|
||||
enum ExtractOptions {
|
||||
|
||||
EXTRACT_UTF16, ///< extract UTF16 text, possible values Boolean.TRUE.toString(), Boolean.FALSE.toString()
|
||||
EXTRACT_UTF8, ///< extract UTF8 text, possible values Boolean.TRUE.toString(), Boolean.FALSE.toString()
|
||||
};
|
||||
|
||||
//generally text extractors should ignore archives
|
||||
//and let unpacking modules take case of them
|
||||
static final List<String> ARCHIVE_MIME_TYPES =
|
||||
Arrays.asList(
|
||||
//ignore unstructured binary and compressed data, for which string extraction or unzipper works better
|
||||
"application/x-7z-compressed",
|
||||
"application/x-ace-compressed",
|
||||
"application/x-alz-compressed",
|
||||
"application/x-arj",
|
||||
"application/vnd.ms-cab-compressed",
|
||||
"application/x-cfs-compressed",
|
||||
"application/x-dgc-compressed",
|
||||
"application/x-apple-diskimage",
|
||||
"application/x-gca-compressed",
|
||||
"application/x-dar",
|
||||
"application/x-lzx",
|
||||
"application/x-lzh",
|
||||
"application/x-rar-compressed",
|
||||
"application/x-stuffit",
|
||||
"application/x-stuffitx",
|
||||
"application/x-gtar",
|
||||
"application/x-archive",
|
||||
"application/x-executable",
|
||||
"application/x-gzip",
|
||||
"application/zip",
|
||||
"application/x-zoo",
|
||||
"application/x-cpio",
|
||||
"application/x-shar",
|
||||
"application/x-tar",
|
||||
"application/x-bzip",
|
||||
"application/x-bzip2",
|
||||
"application/x-lzip",
|
||||
"application/x-lzma",
|
||||
"application/x-lzop",
|
||||
"application/x-z",
|
||||
"application/x-compress");
|
||||
|
||||
/**
|
||||
* Get number of chunks resulted from extracting this AbstractFile
|
||||
*
|
||||
* @return the number of chunks produced
|
||||
*/
|
||||
int getNumChunks();
|
||||
|
||||
/**
|
||||
* Get the source file associated with this extraction
|
||||
*
|
||||
* @return the source AbstractFile
|
||||
*/
|
||||
AbstractFile getSourceFile();
|
||||
|
||||
/**
|
||||
* Index the Abstract File
|
||||
*
|
||||
* @param sourceFile file to index
|
||||
* @return true if indexed successfully, false otherwise
|
||||
* @throws org.sleuthkit.autopsy.keywordsearch.Ingester.IngesterException
|
||||
* @throws org.sleuthkit.autopsy.keywordsearch.Ingester.IngesterException
|
||||
*/
|
||||
boolean index(AbstractFile sourceFile) throws Ingester.IngesterException;
|
||||
|
||||
|
||||
/**
|
||||
* Sets the scripts to use for the extraction
|
||||
*
|
||||
* @param extractScripts scripts to use
|
||||
* @return true if extractor supports script - specific extraction, false otherwise
|
||||
* @return true if extractor supports script - specific extraction, false
|
||||
* otherwise
|
||||
*/
|
||||
boolean setScripts(List<SCRIPT> extractScript);
|
||||
|
||||
|
||||
/**
|
||||
* Get the currently used scripts for extraction
|
||||
*
|
||||
* @return scripts currently used or null if not supported
|
||||
*/
|
||||
List<SCRIPT> getScripts();
|
||||
|
||||
|
||||
/**
|
||||
* Get current options
|
||||
* @return currently used, extractor specific options, or null of not supported
|
||||
*
|
||||
* @return currently used, extractor specific options, or null of not
|
||||
* supported
|
||||
*/
|
||||
Map<String,String> getOptions();
|
||||
|
||||
Map<String, String> getOptions();
|
||||
|
||||
/**
|
||||
* Set extractor specific options
|
||||
*
|
||||
* @param options options to use
|
||||
*/
|
||||
void setOptions(Map<String,String> options);
|
||||
|
||||
void setOptions(Map<String, String> options);
|
||||
|
||||
/**
|
||||
* Determines if the extractor works only for specified types
|
||||
* is supportedTypes() or whether is a generic content extractor (such as string extractor)
|
||||
* @return
|
||||
* Determines if the extractor works only for specified types is
|
||||
* supportedTypes() or whether is a generic content extractor (such as
|
||||
* string extractor)
|
||||
*
|
||||
* @return
|
||||
*/
|
||||
boolean isContentTypeSpecific();
|
||||
|
||||
|
||||
/**
|
||||
* Determines if the file content is supported by the extractor,
|
||||
* if isContentTypeSpecific() returns true.
|
||||
* Determines if the file content is supported by the extractor if
|
||||
* isContentTypeSpecific() returns true.
|
||||
*
|
||||
* @param file to test if its content should be supported
|
||||
* @param detectedFormat mime-type with detected format (such as text/plain)
|
||||
* or null if not detected
|
||||
* @return true if the file content is supported, false otherwise
|
||||
*/
|
||||
boolean isSupported(AbstractFile file);
|
||||
boolean isSupported(AbstractFile file, String detectedFormat);
|
||||
}
|
||||
|
@ -22,6 +22,7 @@ import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.Reader;
|
||||
import java.nio.charset.Charset;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.logging.Level;
|
||||
@ -49,9 +50,15 @@ public class AbstractFileHtmlExtract implements AbstractFileExtract {
|
||||
private AbstractFile sourceFile;
|
||||
private int numChunks = 0;
|
||||
//private static final String UTF16BOM = "\uFEFF"; disabled prepending of BOM
|
||||
private static final String[] SUPPORTED_EXTENSIONS = {
|
||||
"htm", "html", "xhtml", "shtml", "xhtm", "shtm", "css", "js", "php", "jsp"
|
||||
};
|
||||
static final List<String> WEB_MIME_TYPES = Arrays.asList(
|
||||
"application/javascript",
|
||||
"application/xhtml+xml",
|
||||
"application/json",
|
||||
"text/css",
|
||||
"text/html",
|
||||
"text/javascript" //"application/xml",
|
||||
//"application/xml-dtd",
|
||||
);
|
||||
|
||||
AbstractFileHtmlExtract() {
|
||||
this.module = KeywordSearchIngestModule.getDefault();
|
||||
@ -67,7 +74,7 @@ public class AbstractFileHtmlExtract implements AbstractFileExtract {
|
||||
public List<SCRIPT> getScripts() {
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public Map<String, String> getOptions() {
|
||||
return null;
|
||||
@ -75,7 +82,6 @@ public class AbstractFileHtmlExtract implements AbstractFileExtract {
|
||||
|
||||
@Override
|
||||
public void setOptions(Map<String, String> options) {
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
@ -207,13 +213,16 @@ public class AbstractFileHtmlExtract implements AbstractFileExtract {
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isSupported(AbstractFile file) {
|
||||
String fileNameLower = file.getName().toLowerCase();
|
||||
for (int i = 0; i < SUPPORTED_EXTENSIONS.length; ++i) {
|
||||
if (fileNameLower.endsWith(SUPPORTED_EXTENSIONS[i])) {
|
||||
return true;
|
||||
}
|
||||
public boolean isSupported(AbstractFile file, String detectedFormat) {
|
||||
if (detectedFormat == null) {
|
||||
return false;
|
||||
}
|
||||
return false;
|
||||
else if (WEB_MIME_TYPES.contains(detectedFormat) ) {
|
||||
return true;
|
||||
}
|
||||
else {
|
||||
return false;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
@ -51,13 +51,7 @@ class AbstractFileStringExtract implements AbstractFileExtract {
|
||||
private static final SCRIPT DEFAULT_SCRIPT = SCRIPT.LATIN_2;
|
||||
private final List<SCRIPT> extractScripts = new ArrayList<SCRIPT>();
|
||||
private Map<String, String> extractOptions = new HashMap<String, String>();
|
||||
//string extractor extracts from all other than archives
|
||||
//TODO use content type detection mechanism
|
||||
static final String[] UNSUPPORTED_EXTENSIONS = {
|
||||
//Archives
|
||||
//Note: archive unpacker module will process these instead
|
||||
"tar", "jar", "zip", "7z", "gzip", "bzip", "bzip2", "gz", "tgz", "cab", "rar", "arj", "dmg", "iso"
|
||||
};
|
||||
|
||||
|
||||
//disabled prepending of BOM
|
||||
//static {
|
||||
@ -185,18 +179,26 @@ class AbstractFileStringExtract implements AbstractFileExtract {
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isSupported(AbstractFile file) {
|
||||
String fileNameLower = file.getName().toLowerCase();
|
||||
int dotI = fileNameLower.lastIndexOf(".");
|
||||
if (dotI == -1 || dotI == fileNameLower.length() - 1) {
|
||||
return true; //no extension
|
||||
public boolean isSupported(AbstractFile file, String detectedFormat) {
|
||||
if (detectedFormat == null) {
|
||||
return true;
|
||||
}
|
||||
final String extension = fileNameLower.substring(dotI + 1);
|
||||
for (int i = 0; i < UNSUPPORTED_EXTENSIONS.length; ++i) {
|
||||
if (extension.equals(UNSUPPORTED_EXTENSIONS[i])) {
|
||||
return false;
|
||||
}
|
||||
else if (detectedFormat.equals("application/octet-stream")) {
|
||||
//any binary unstructured blobs (string extraction will be used)
|
||||
return true;
|
||||
}
|
||||
else if (AbstractFileExtract.ARCHIVE_MIME_TYPES.contains(detectedFormat)) {
|
||||
return false; //let unzipper take care of it
|
||||
}
|
||||
//skip images/video/audio
|
||||
else if (detectedFormat.contains("image/")
|
||||
|| detectedFormat.contains("audio/")
|
||||
|| detectedFormat.contains("video/")
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
else {
|
||||
return true;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
@ -66,30 +66,6 @@ public class AbstractFileTikaTextExtract implements AbstractFileExtract {
|
||||
private int numChunks = 0;
|
||||
//private static final String UTF16BOM = "\uFEFF"; disabled prepending of BOM
|
||||
private final ExecutorService tikaParseExecutor = Executors.newSingleThreadExecutor();
|
||||
// TODO: use type detection mechanism instead, and maintain supported MimeTypes, not extensions
|
||||
// supported extensions list from http://www.lucidimagination.com/devzone/technical-articles/content-extraction-tika
|
||||
static final String[] SUPPORTED_EXTENSIONS = {
|
||||
//Archive (to be removed when we have archive module
|
||||
/// handled by 7zip module now "tar", "jar", "zip", "gzip", "bzip2", "gz", "tgz", "ar", "cpio",
|
||||
//MS Office
|
||||
"doc", "dot", "docx", "docm", "dotx", "dotm",
|
||||
"xls", "xlw", "xlt", "xlsx", "xlsm", "xltx", "xltm",
|
||||
"ppt", "pps", "pot", "pptx", "pptm", "potx", "potm",
|
||||
//Open Office
|
||||
"odf", "odt", "ott", "ods", "ots", "odp", "otp",
|
||||
"sxw", "stw", "sxc", "stc", "sxi", "sxi",
|
||||
"sdw", "sdc", "vor", "sgl",
|
||||
//rich text, pdf
|
||||
"rtf", "pdf",
|
||||
//html (other extractors take priority)
|
||||
"html", "htm", "xhtml",
|
||||
//text
|
||||
"txt", "log", "manifest",
|
||||
//code
|
||||
"class",
|
||||
//images, media, other
|
||||
"bmp", "gif", "png", "jpeg", "jpg", "tiff", "mp3", "flv", "aiff", "au", "midi", "wav",
|
||||
"pst", "xml", "class", "dwg", "eml", "emlx", "mbox", "mht"};
|
||||
|
||||
AbstractFileTikaTextExtract() {
|
||||
this.module = KeywordSearchIngestModule.getDefault();
|
||||
@ -282,19 +258,27 @@ public class AbstractFileTikaTextExtract implements AbstractFileExtract {
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isSupported(AbstractFile file) {
|
||||
String fileNameLower = file.getName().toLowerCase();
|
||||
int dotI = fileNameLower.lastIndexOf(".");
|
||||
if (dotI == -1 || dotI == fileNameLower.length() - 1) {
|
||||
return false; //no extension
|
||||
public boolean isSupported(AbstractFile file, String detectedFormat) {
|
||||
if (detectedFormat == null) {
|
||||
return false;
|
||||
} else if (detectedFormat.equals("application/octet-stream")) {
|
||||
//any binary unstructured blobs (string extraction will be used)
|
||||
return false;
|
||||
} else if (AbstractFileExtract.ARCHIVE_MIME_TYPES.contains(detectedFormat)) {
|
||||
return false;
|
||||
} //skip video other than flv (tika supports flv only)
|
||||
else if (detectedFormat.contains("video/")
|
||||
&& !detectedFormat.equals("video/x-flv")) {
|
||||
return false;
|
||||
}
|
||||
final String extension = fileNameLower.substring(dotI + 1);
|
||||
for (int i = 0; i < SUPPORTED_EXTENSIONS.length; ++i) {
|
||||
if (extension.equals(SUPPORTED_EXTENSIONS[i])) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
|
||||
//TODO might need to add more mime-types to ignore
|
||||
|
||||
//default to true, which includes
|
||||
//text, docs, pdf and others
|
||||
|
||||
return true;
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -20,6 +20,8 @@ package org.sleuthkit.autopsy.keywordsearch;
|
||||
|
||||
import java.awt.event.ActionEvent;
|
||||
import java.awt.event.ActionListener;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.lang.Long;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
@ -37,10 +39,12 @@ import org.sleuthkit.autopsy.coreutils.Logger;
|
||||
import javax.swing.SwingUtilities;
|
||||
import javax.swing.SwingWorker;
|
||||
import javax.swing.Timer;
|
||||
import org.apache.tika.Tika;
|
||||
import org.netbeans.api.progress.aggregate.AggregateProgressFactory;
|
||||
import org.netbeans.api.progress.aggregate.AggregateProgressHandle;
|
||||
import org.netbeans.api.progress.aggregate.ProgressContributor;
|
||||
import org.openide.util.Cancellable;
|
||||
import org.openide.util.Exceptions;
|
||||
import org.sleuthkit.autopsy.casemodule.Case;
|
||||
import org.sleuthkit.autopsy.coreutils.EscapeUtil;
|
||||
import org.sleuthkit.autopsy.coreutils.StopWatch;
|
||||
@ -57,6 +61,7 @@ import org.sleuthkit.datamodel.BlackboardArtifact;
|
||||
import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE;
|
||||
import org.sleuthkit.datamodel.BlackboardAttribute;
|
||||
import org.sleuthkit.datamodel.AbstractFile;
|
||||
import org.sleuthkit.datamodel.ReadContentInputStream;
|
||||
import org.sleuthkit.datamodel.SleuthkitCase;
|
||||
import org.sleuthkit.datamodel.TskCoreException;
|
||||
import org.sleuthkit.datamodel.TskData;
|
||||
@ -123,6 +128,7 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
|
||||
private static AbstractFileStringExtract stringExtractor;
|
||||
private boolean initialized = false;
|
||||
private KeywordSearchConfigurationPanel panel;
|
||||
private Tika tikaFormatDetector;
|
||||
|
||||
private enum IngestStatus {
|
||||
|
||||
@ -147,7 +153,7 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
|
||||
}
|
||||
|
||||
@Override
|
||||
public ProcessResult process(PipelineContext<IngestModuleAbstractFile>pipelineContext, AbstractFile abstractFile) {
|
||||
public ProcessResult process(PipelineContext<IngestModuleAbstractFile> pipelineContext, AbstractFile abstractFile) {
|
||||
|
||||
if (initialized == false) //error initializing indexing/Solr
|
||||
{
|
||||
@ -189,7 +195,6 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
|
||||
return ProcessResult.OK;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* After all files are ingested, execute final index commit and final search
|
||||
* Cleanup resources, threads, timers
|
||||
@ -297,6 +302,8 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
|
||||
keywordLists.clear();
|
||||
keywordToList.clear();
|
||||
|
||||
tikaFormatDetector = null;
|
||||
|
||||
initialized = false;
|
||||
}
|
||||
|
||||
@ -338,6 +345,8 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
|
||||
|
||||
caseHandle = Case.getCurrentCase().getSleuthkitCase();
|
||||
|
||||
tikaFormatDetector = new Tika();
|
||||
|
||||
ingester = Server.getIngester();
|
||||
|
||||
final Server server = KeywordSearch.getServer();
|
||||
@ -659,18 +668,20 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
|
||||
* index
|
||||
* @param stringsOnly true if use string extraction, false if to use a
|
||||
* content-type specific text extractor
|
||||
* @param detectedFormat mime-type detected, or null if none detected
|
||||
* @return true if the file was indexed, false otherwise
|
||||
* @throws IngesterException exception thrown if indexing failed
|
||||
*/
|
||||
private boolean extractIndex(AbstractFile aFile, boolean stringsOnly) throws IngesterException {
|
||||
private boolean extractIndex(AbstractFile aFile, boolean stringsOnly, String detectedFormat) throws IngesterException {
|
||||
AbstractFileExtract fileExtract = null;
|
||||
|
||||
if (stringsOnly && stringExtractor.isSupported(aFile)) {
|
||||
if (stringsOnly && stringExtractor.isSupported(aFile, detectedFormat)) {
|
||||
fileExtract = stringExtractor;
|
||||
} else {
|
||||
//go over available text extractors and pick the first one (most specific one)
|
||||
//not only strings
|
||||
//go over available text extractors in order, and pick the first one (most specific one)
|
||||
for (AbstractFileExtract fe : textExtractors) {
|
||||
if (fe.isSupported(aFile)) {
|
||||
if (fe.isSupported(aFile, detectedFormat)) {
|
||||
fileExtract = fe;
|
||||
break;
|
||||
}
|
||||
@ -678,7 +689,8 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
|
||||
}
|
||||
|
||||
if (fileExtract == null) {
|
||||
logger.log(Level.INFO, "No supported file extractor found for file: " + aFile.getId() + " " + aFile.getName());
|
||||
logger.log(Level.INFO, "No text extractor found for file id:"
|
||||
+ aFile.getId() + ", name: " + aFile.getName() + ", detected format: " + detectedFormat);
|
||||
return false;
|
||||
}
|
||||
|
||||
@ -688,10 +700,16 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
|
||||
return fileExtract.index(aFile);
|
||||
}
|
||||
|
||||
private boolean isTextExtractSupported(AbstractFile aFile) {
|
||||
/**
|
||||
* Check with every extractor if it supports the file with the detected format
|
||||
* @param aFile file to check for
|
||||
* @param detectedFormat mime-type with detected format (such as text/plain) or null if not detected
|
||||
* @return true if text extraction is supported
|
||||
*/
|
||||
private boolean isTextExtractSupported(AbstractFile aFile, String detectedFormat) {
|
||||
for (AbstractFileExtract extractor : textExtractors) {
|
||||
if (extractor.isContentTypeSpecific() == true
|
||||
&& extractor.isSupported(aFile)) {
|
||||
&& extractor.isSupported(aFile, detectedFormat)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@ -706,8 +724,8 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
|
||||
if (aType.equals(TskData.TSK_DB_FILES_TYPE_ENUM.VIRTUAL_DIR)) {
|
||||
//skip indexing of virtual dirs (no content, no real name) - will index children files
|
||||
return;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
boolean isUnallocFile = aType.equals(TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS);
|
||||
|
||||
final long size = aFile.getSize();
|
||||
@ -725,18 +743,39 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
|
||||
return;
|
||||
}
|
||||
|
||||
boolean extractTextSupported = isTextExtractSupported(aFile);
|
||||
//use Tika to detect the format
|
||||
String detectedFormat = null;
|
||||
InputStream is = null;
|
||||
try {
|
||||
is = new ReadContentInputStream(aFile);
|
||||
detectedFormat = tikaFormatDetector.detect(is, aFile.getName());
|
||||
|
||||
} catch (Exception e) {
|
||||
logger.log(Level.WARNING, "Could not detect format using tika for file: " + aFile, e);
|
||||
} finally {
|
||||
if (is != null) {
|
||||
try {
|
||||
is.close();
|
||||
} catch (IOException ex) {
|
||||
logger.log(Level.WARNING, "Could not close stream after detecting format using tika for file: "
|
||||
+ aFile, ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
logger.log(Level.INFO, "Detected format: " + aFile.getName() + " " + detectedFormat);
|
||||
|
||||
boolean extractTextSupported = isTextExtractSupported(aFile, detectedFormat);
|
||||
if (isUnallocFile == false && extractTextSupported) {
|
||||
//we know it's an allocated FS file
|
||||
//extract text with one of the extractors, divide into chunks and index with Solr
|
||||
try {
|
||||
//logger.log(Level.INFO, "indexing: " + aFile.getName());
|
||||
if (!extractIndex(aFile, false)) {
|
||||
if (!extractIndex(aFile, false, detectedFormat)) {
|
||||
logger.log(Level.WARNING, "Failed to extract text and ingest, file '" + aFile.getName() + "' (id: " + aFile.getId() + ").");
|
||||
ingestStatus.put(aFile.getId(), IngestStatus.SKIPPED);
|
||||
//try to extract strings, if a file
|
||||
if (aFile.isFile() == true) {
|
||||
processNonIngestible(aFile);
|
||||
processNonIngestible(aFile, detectedFormat);
|
||||
}
|
||||
|
||||
} else {
|
||||
@ -749,7 +788,7 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
|
||||
ingestStatus.put(aFile.getId(), IngestStatus.SKIPPED);
|
||||
//try to extract strings, if a file
|
||||
if (aFile.isFile() == true) {
|
||||
processNonIngestible(aFile);
|
||||
processNonIngestible(aFile, detectedFormat);
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
@ -758,18 +797,18 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
|
||||
ingestStatus.put(aFile.getId(), IngestStatus.SKIPPED);
|
||||
//try to extract strings if a file
|
||||
if (aFile.isFile() == true) {
|
||||
processNonIngestible(aFile);
|
||||
processNonIngestible(aFile, detectedFormat);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
//unallocated file or unsupported content type by Solr
|
||||
processNonIngestible(aFile);
|
||||
processNonIngestible(aFile, detectedFormat);
|
||||
}
|
||||
}
|
||||
|
||||
private boolean processNonIngestible(AbstractFile aFile) {
|
||||
private boolean processNonIngestible(AbstractFile aFile, String detectedFormat) {
|
||||
try {
|
||||
if (!extractIndex(aFile, true)) {
|
||||
if (!extractIndex(aFile, true, detectedFormat)) {
|
||||
logger.log(Level.WARNING, "Failed to extract strings and ingest, file '" + aFile.getName() + "' (id: " + aFile.getId() + ").");
|
||||
ingestStatus.put(aFile.getId(), IngestStatus.SKIPPED);
|
||||
return false;
|
||||
@ -1025,7 +1064,7 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
|
||||
detailsSb.append("<tr>");
|
||||
detailsSb.append("<th>File</th>");
|
||||
detailsSb.append("<td>").append(hitFile.getParentPath()).append(hitFile.getName()).append("</td>");
|
||||
|
||||
|
||||
detailsSb.append("</tr>");
|
||||
|
||||
|
||||
|
5
NEWS.txt
5
NEWS.txt
@ -1,4 +1,4 @@
|
||||
---------------- VERSION Current (development) --------------
|
||||
---------------- VERSION 3.0.5 --------------
|
||||
|
||||
New features:
|
||||
- Archive extractor ingest module (uses 7zip)
|
||||
@ -6,6 +6,8 @@ New features:
|
||||
|
||||
Improvements:
|
||||
- Sleuthkit-4.0.2 and libewf-20130128
|
||||
- improved image loading in Media View and Thumbnail View (faster loading, handles large files better)
|
||||
- improve Keyword Search file indexing (use detected mime-type instead of file extension)
|
||||
- show children counts in directory tree
|
||||
|
||||
Bugfixes:
|
||||
@ -16,6 +18,7 @@ Bugfixes:
|
||||
- exif module better jpeg detection using signature and not only file extension.
|
||||
- The "media view" tab is inactive for deleted files (#165)
|
||||
|
||||
|
||||
---------------- VERSION 3.0.4 --------------
|
||||
|
||||
New features:
|
||||
|
@ -1,6 +1,6 @@
|
||||
Manifest-Version: 1.0
|
||||
OpenIDE-Module: org.sleuthkit.autopsy.recentactivity/4
|
||||
OpenIDE-Module-Implementation-Version: 6
|
||||
OpenIDE-Module-Implementation-Version: 7
|
||||
OpenIDE-Module-Layer: org/sleuthkit/autopsy/recentactivity/layer.xml
|
||||
OpenIDE-Module-Localizing-Bundle: org/sleuthkit/autopsy/recentactivity/Bundle.properties
|
||||
OpenIDE-Module-Requires:
|
||||
|
@ -4,4 +4,4 @@ javac.compilerargs=-Xlint -Xlint:-serial
|
||||
license.file=../LICENSE-2.0.txt
|
||||
nbm.homepage=http://www.sleuthkit.org/autopsy/
|
||||
nbm.needs.restart=true
|
||||
spec.version.base=2.0
|
||||
spec.version.base=2.1
|
||||
|
@ -36,7 +36,7 @@
|
||||
<compile-dependency/>
|
||||
<run-dependency>
|
||||
<release-version>7</release-version>
|
||||
<specification-version>5.0</specification-version>
|
||||
<specification-version>5.2</specification-version>
|
||||
</run-dependency>
|
||||
</dependency>
|
||||
</module-dependencies>
|
||||
|
@ -21,7 +21,7 @@
|
||||
<compile-dependency/>
|
||||
<run-dependency>
|
||||
<release-version>7</release-version>
|
||||
<specification-version>5.0</specification-version>
|
||||
<specification-version>5.2</specification-version>
|
||||
</run-dependency>
|
||||
</dependency>
|
||||
<dependency>
|
||||
|
@ -1,6 +1,6 @@
|
||||
Manifest-Version: 1.0
|
||||
AutoUpdate-Show-In-Client: false
|
||||
OpenIDE-Module: org.sleuthkit.autopsy.testing/3
|
||||
OpenIDE-Module-Implementation-Version: 5
|
||||
OpenIDE-Module-Implementation-Version: 6
|
||||
OpenIDE-Module-Localizing-Bundle: org/sleuthkit/autopsy/testing/Bundle.properties
|
||||
|
||||
|
@ -3,4 +3,4 @@ javac.compilerargs=-Xlint -Xlint:-serial
|
||||
license.file=../LICENSE-2.0.txt
|
||||
nbm.homepage=http://www.sleuthkit.org/autopsy/
|
||||
nbm.needs.restart=true
|
||||
spec.version.base=1.1
|
||||
spec.version.base=1.2
|
||||
|
@ -12,7 +12,7 @@
|
||||
<compile-dependency/>
|
||||
<run-dependency>
|
||||
<release-version>7</release-version>
|
||||
<specification-version>5.0</specification-version>
|
||||
<specification-version>5.2</specification-version>
|
||||
</run-dependency>
|
||||
</dependency>
|
||||
<dependency>
|
||||
|
@ -4,4 +4,5 @@ OpenIDE-Module-Layer: org/sleuthkit/autopsy/timeline/layer.xml
|
||||
OpenIDE-Module-Localizing-Bundle: org/sleuthkit/autopsy/timeline/Bundle.properties
|
||||
OpenIDE-Module-Requires: org.openide.windows.WindowManager
|
||||
OpenIDE-Module-Specification-Version: 1.0
|
||||
OpenIDE-Module-Implementation-Version: 1
|
||||
|
||||
|
@ -94,7 +94,7 @@
|
||||
<compile-dependency/>
|
||||
<run-dependency>
|
||||
<release-version>7</release-version>
|
||||
<specification-version>3.0</specification-version>
|
||||
<specification-version>5.2</specification-version>
|
||||
</run-dependency>
|
||||
</dependency>
|
||||
<dependency>
|
||||
|
@ -1,4 +1,9 @@
|
||||
OpenIDE-Module-Display-Category=External Viewers
|
||||
OpenIDE-Module-Long-Description=\
|
||||
Displays user activity as an interactive timeline chart with year, month and day granularity. \n\
|
||||
Events for a selected day are viewable in the built-in result and content viewers.
|
||||
OpenIDE-Module-Name=Timeline
|
||||
CTL_MakeTimeline="Make Timeline (Beta)"
|
||||
OpenIDE-Module-Short-Description=Displays user activity timeline
|
||||
TimelineProgressDialog.jLabel1.text=Creating timeline . . .
|
||||
TimelineFrame.title=Timeline
|
||||
|
@ -1,7 +1,7 @@
|
||||
Manifest-Version: 1.0
|
||||
AutoUpdate-Show-In-Client: true
|
||||
OpenIDE-Module: org.sleuthkit.autopsy.thunderbirdparser/3
|
||||
OpenIDE-Module-Implementation-Version: 6
|
||||
OpenIDE-Module-Implementation-Version: 7
|
||||
OpenIDE-Module-Layer: org/sleuthkit/autopsy/thunderbirdparser/layer.xml
|
||||
OpenIDE-Module-Localizing-Bundle: org/sleuthkit/autopsy/thunderbirdparser/Bundle.properties
|
||||
|
||||
|
@ -3,4 +3,4 @@ javac.compilerargs=-Xlint -Xlint:-serial
|
||||
license.file=../LICENSE-2.0.txt
|
||||
nbm.homepage=http://www.sleuthkit.org/autopsy/
|
||||
nbm.needs.restart=true
|
||||
spec.version.base=1.0
|
||||
spec.version.base=1.1
|
||||
|
@ -12,7 +12,7 @@
|
||||
<compile-dependency/>
|
||||
<run-dependency>
|
||||
<release-version>7</release-version>
|
||||
<specification-version>5.0</specification-version>
|
||||
<specification-version>5.2</specification-version>
|
||||
</run-dependency>
|
||||
</dependency>
|
||||
<dependency>
|
||||
@ -21,7 +21,7 @@
|
||||
<compile-dependency/>
|
||||
<run-dependency>
|
||||
<release-version>5</release-version>
|
||||
<specification-version>3.0</specification-version>
|
||||
<specification-version>3.1</specification-version>
|
||||
</run-dependency>
|
||||
</dependency>
|
||||
</module-dependencies>
|
||||
|
Loading…
x
Reference in New Issue
Block a user