Merge remote-tracking branch 'upstream/master'

This commit is contained in:
Sean-M 2013-03-20 17:06:33 -04:00
commit fdfd758911
37 changed files with 383 additions and 253 deletions

View File

@ -2,7 +2,7 @@ Manifest-Version: 1.0
OpenIDE-Module: org.sleuthkit.autopsy.core/7
OpenIDE-Module-Localizing-Bundle: org/sleuthkit/autopsy/core/Bundle.properties
OpenIDE-Module-Layer: org/sleuthkit/autopsy/core/layer.xml
OpenIDE-Module-Implementation-Version: 6
OpenIDE-Module-Implementation-Version: 7
OpenIDE-Module-Requires: org.openide.windows.WindowManager, org.netbeans.api.javahelp.Help
AutoUpdate-Show-In-Client: true
AutoUpdate-Essential-Module: true

View File

@ -6,5 +6,5 @@ license.file=../LICENSE-2.0.txt
nbm.homepage=http://www.sleuthkit.org/
nbm.module.author=Brian Carrier
nbm.needs.restart=true
spec.version.base=5.1
spec.version.base=5.2

View File

@ -30,6 +30,7 @@ import org.openide.explorer.ExplorerManager;
import org.openide.explorer.ExplorerManager.Provider;
import org.openide.nodes.Node;
import org.openide.util.Lookup;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.corecomponentinterfaces.DataContent;
import org.sleuthkit.autopsy.corecomponentinterfaces.DataResultViewer;
import org.sleuthkit.autopsy.coreutils.Logger;
@ -43,10 +44,9 @@ public abstract class AbstractDataResultViewer extends JPanel implements
private static final Logger logger = Logger.getLogger(AbstractDataResultViewer.class.getName());
protected transient ExplorerManager em = new ExplorerManager();
private PropertyChangeListener nodeSelListener;
/**
* Content viewer to respond to selection events
* Either the main one, or custom one if set
* Content viewer to respond to selection events Either the main one, or
* custom one if set
*/
protected DataContent contentViewer;
@ -54,17 +54,22 @@ public abstract class AbstractDataResultViewer extends JPanel implements
//DataContent is designed to return only the default viewer from lookup
//use the default one unless set otherwise
contentViewer = Lookup.getDefault().lookup(DataContent.class);
contentViewer = Lookup.getDefault().lookup(DataContent.class);
//property listener to send nodes to content viewer
nodeSelListener = new PropertyChangeListener() {
/**
* Propagates changes in the current select node from the
* DataResultViewer to the DataContentTopComponent
*/
@Override
public void propertyChange(PropertyChangeEvent evt) {
if (!Case.isCaseOpen()) {
//handle in-between condition when case is being closed
//and legacy selection events are pumped
return;
}
String changed = evt.getPropertyName();
// change that should affect view
@ -77,10 +82,10 @@ public abstract class AbstractDataResultViewer extends JPanel implements
setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR));
try {
Node selectedNode = getSelectedNode();
nodeSelected(selectedNode);
if (selectedNode != null) {
// there's a new/changed node to display
@ -142,10 +147,11 @@ public abstract class AbstractDataResultViewer extends JPanel implements
@Override
public void resetComponent() {
}
/**
* Called when a new node has been selected in the result viewer
* Can update the viewer, etc.
* Called when a new node has been selected in the result viewer Can update
* the viewer, etc.
*
* @param selectedNode the new node currently selected
*/
public abstract void nodeSelected(Node selectedNode);
@ -168,7 +174,7 @@ public abstract class AbstractDataResultViewer extends JPanel implements
logger.log(Level.WARNING, "Couldn't set selected nodes.", ex);
}
}
@Override
public void setContentViewer(DataContent contentViewer) {
this.contentViewer = contentViewer;

View File

@ -93,6 +93,7 @@ public class DataContentViewerMedia extends javax.swing.JPanel implements DataCo
private BufferedImage currentImage = null;
private boolean gstInited = false;
private AbstractFile lastFile;
private boolean inImageMode; //keeps track if already in image mode to minimize UI setup
/**
* Creates new form DataContentViewerVideo
@ -103,6 +104,7 @@ public class DataContentViewerMedia extends javax.swing.JPanel implements DataCo
}
private void customizeComponents() {
inImageMode = false;
Platform.setImplicitExit(false);
PlatformImpl.startup(new Runnable() {
@ -112,17 +114,17 @@ public class DataContentViewerMedia extends javax.swing.JPanel implements DataCo
}
});
logger.log(Level.INFO, "Supported image formats by javafx image viewer: ");
//initialize supported image types
//TODO use mime-types instead once we have support
String[] fxSupportedImagesSuffixes = ImageIO.getReaderFileSuffixes();
IMAGES = new String[fxSupportedImagesSuffixes.length];
for (int i=0; i<fxSupportedImagesSuffixes.length; ++i) {
for (int i = 0; i < fxSupportedImagesSuffixes.length; ++i) {
String suffix = fxSupportedImagesSuffixes[i];
logger.log(Level.INFO, "suffix: " + suffix);
IMAGES[i] = "." + suffix;
}
try {
logger.log(Level.INFO, "Initializing gstreamer for video/audio viewing");
Gst.init();
@ -261,7 +263,7 @@ public class DataContentViewerMedia extends javax.swing.JPanel implements DataCo
if (selectedNode == null) {
return;
}
AbstractFile file = selectedNode.getLookup().lookup(AbstractFile.class);
if (file == null) {
return;
@ -275,7 +277,7 @@ public class DataContentViewerMedia extends javax.swing.JPanel implements DataCo
reset();
setComponentsVisibility(false);
// get rid of any existing videoProgressWorker thread
if (videoProgressWorker != null) {
videoProgressWorker.cancel(true);
@ -288,6 +290,7 @@ public class DataContentViewerMedia extends javax.swing.JPanel implements DataCo
showImageFx(file);
} else if (gstInited
&& (containsExt(file.getName(), VIDEOS) || containsExt(file.getName(), AUDIOS))) {
inImageMode = false;
setupVideo(file);
}
}
@ -299,13 +302,19 @@ public class DataContentViewerMedia extends javax.swing.JPanel implements DataCo
*/
private void showImageFx(final AbstractFile file) {
final String fileName = file.getName();
if (!Case.isCaseOpen()) {
//handle in-between condition when case is being closed
//and an image was previously selected
return;
}
// load the image
PlatformImpl.runLater(new Runnable() {
@Override
public void run() {
Dimension dims = DataContentViewerMedia.this.getSize();
final InputStream inputStream = new ReadContentInputStream(file);
final Image fxImage;
@ -313,19 +322,24 @@ public class DataContentViewerMedia extends javax.swing.JPanel implements DataCo
//original input stream
BufferedImage bi = ImageIO.read(inputStream);
//scale image using Scalr
BufferedImage biScaled = ScalrWrapper.resizeHighQuality(bi, (int)dims.getWidth(), (int)dims.getHeight());
BufferedImage biScaled = ScalrWrapper.resizeHighQuality(bi, (int) dims.getWidth(), (int) dims.getHeight());
//convert from awt imageto fx image
fxImage = SwingFXUtils.toFXImage(biScaled, null);
} catch (IOException ex) {
logger.log(Level.WARNING, "Could not load image file into media view: " + fileName, ex);
return;
}
catch (OutOfMemoryError ex) {
} catch (OutOfMemoryError ex) {
logger.log(Level.WARNING, "Could not load image file into media view (too large): " + fileName, ex);
MessageNotifyUtil.Notify.warn("Could not load image file (too large): " + file.getName(), ex.getMessage());
MessageNotifyUtil.Notify.warn("Could not load image file (too large): " + file.getName(), ex.getMessage());
return;
} finally {
try {
inputStream.close();
} catch (IOException ex) {
logger.log(Level.WARNING, "Could not close input stream after loading image in media view: " + fileName, ex);
}
}
if (fxImage == null) {
logger.log(Level.WARNING, "Could not load image file into media view: " + fileName);
return;
@ -346,27 +360,36 @@ public class DataContentViewerMedia extends javax.swing.JPanel implements DataCo
Group fxRoot = new Group();
//Scene fxScene = new Scene(fxRoot, dims.getWidth(), dims.getHeight(), javafx.scene.paint.Color.BLACK);
Scene fxScene = new Scene(fxRoot,javafx.scene.paint.Color.BLACK);
Scene fxScene = new Scene(fxRoot, javafx.scene.paint.Color.BLACK);
fxRoot.getChildren().add(fxImageView);
final JFXPanel fxPanel = new JFXPanel();
fxPanel.setScene(fxScene);
if (inImageMode) {
final JFXPanel fxPanel = (JFXPanel) videoPanel.getComponent(0);
fxPanel.setScene(fxScene);
videoPanel.setVisible(true);
} else {
inImageMode = true;
final JFXPanel fxPanel = new JFXPanel();
fxPanel.setScene(fxScene);
//when done, join with the swing panel
EventQueue.invokeLater(new Runnable() {
@Override
public void run() {
//remove video panels and recreate image view panel
//TODO use swing layered pane to switch between different modes
videoPanel.removeAll();
videoPanel.setLayout(new BoxLayout(videoPanel, BoxLayout.Y_AXIS));
videoPanel.add(fxPanel);
videoPanel.setVisible(true);
//when done, join with the swing panel
EventQueue.invokeLater(new Runnable() {
@Override
public void run() {
videoPanel.removeAll();
videoPanel.setLayout(new BoxLayout(videoPanel, BoxLayout.Y_AXIS));
videoPanel.add(fxPanel);
videoPanel.setVisible(true);
if (fxImage.isError() ) {
logger.log(Level.WARNING, "Could not load image file into media view: " + fileName);
//MessageNotifyUtil.Message.warn("Could not load image file: " + file.getName());
if (fxImage.isError()) {
logger.log(Level.WARNING, "Could not load image file into media view: " + fileName);
//MessageNotifyUtil.Message.warn("Could not load image file: " + file.getName());
}
}
}
});
});
}
}
});

View File

@ -22,6 +22,8 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.logging.Level;
import javax.imageio.ImageIO;
import org.openide.nodes.AbstractNode;
import org.openide.nodes.Children;
import org.openide.nodes.Node;
@ -154,7 +156,17 @@ class ThumbnailViewChildren extends Children.Keys<Integer> {
private static class IsSupportedContentVisitor extends ContentVisitor.Default<Boolean> {
private static final List<String> SUPP_EXTENSIONS = Arrays.asList(".jpeg", ".jpg", ".gif", ".png");
private final List<String> SUPP_EXTENSIONS;
IsSupportedContentVisitor() {
String[] supportedImagesSuffixes = ImageIO.getReaderFileSuffixes();
SUPP_EXTENSIONS = new ArrayList<String>(supportedImagesSuffixes.length);
for (int i = 0; i < supportedImagesSuffixes.length; ++i) {
String suffix = supportedImagesSuffixes[i];
SUPP_EXTENSIONS.add("." + suffix);
}
}
@Override
public Boolean visit(DerivedFile f) {

View File

@ -26,6 +26,7 @@ import java.awt.Toolkit;
import java.awt.image.BufferedImage;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.lang.ref.SoftReference;
import java.util.logging.Level;
import javax.imageio.ImageIO;
@ -34,33 +35,41 @@ import javax.swing.JFrame;
import org.openide.nodes.Children;
import org.openide.nodes.FilterNode;
import org.openide.nodes.Node;
import org.openide.util.Exceptions;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.corelibs.ScalrWrapper;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.datamodel.Content;
import org.sleuthkit.datamodel.ReadContentInputStream;
import org.sleuthkit.datamodel.TskException;
/**
* Node that wraps around original node and adds the bitmap icon representing the picture
* Node that wraps around original node and adds the bitmap icon representing
* the picture
*/
class ThumbnailViewNode extends FilterNode {
private SoftReference<Image> iconCache;
private static final Image defaultIcon = new ImageIcon("/org/sleuthkit/autopsy/images/file-icon.png").getImage();
private static final Logger logger = Logger.getLogger(ThumbnailViewNode.class.getName());
//private final BufferedImage defaultIconBI;
/** the constructor */
/**
* the constructor
*/
ThumbnailViewNode(Node arg) {
super(arg, Children.LEAF);
}
@Override
public String getDisplayName(){
if(super.getDisplayName().length() > 15)
public String getDisplayName() {
if (super.getDisplayName().length() > 15) {
return super.getDisplayName().substring(0, 15).concat("...");
else
} else {
return super.getDisplayName();
}
}
@Override
public Image getIcon(int type) {
Image icon = null;
@ -68,12 +77,11 @@ class ThumbnailViewNode extends FilterNode {
if (iconCache != null) {
icon = iconCache.get();
}
if (icon == null) {
Content content = this.getLookup().lookup(Content.class);
if (content != null) {
if (getFile(content.getId()).exists()) {
try {
@ -84,85 +92,56 @@ class ThumbnailViewNode extends FilterNode {
} else {
try {
icon = generateIcon(content);
ImageIO.write(toBufferedImage(icon), "jpg", getFile(content.getId()));
} catch (TskException ex) {
icon = ThumbnailViewNode.defaultIcon;
if (icon == null) {
icon = ThumbnailViewNode.defaultIcon;
} else {
ImageIO.write((BufferedImage) icon, "jpg", getFile(content.getId()));
}
} catch (IOException ex) {
logger.log(Level.WARNING, "Could not write cache thumbnail: " + content, ex);
}
}
} else {
icon = ThumbnailViewNode.defaultIcon;
}
iconCache = new SoftReference<Image>(icon);
}
return icon;
}
static private Image generateIcon(Content content) throws TskException {
byte[] data = new byte[(int)content.getSize()];
int bytesRead = content.read(data, 0, content.getSize());
if (bytesRead < 1)
/*
* Generate a scaled image
*/
static private BufferedImage generateIcon(Content content) {
InputStream inputStream = null;
try {
inputStream = new ReadContentInputStream(content);
BufferedImage bi = ImageIO.read(inputStream);
BufferedImage biScaled = ScalrWrapper.resizeFast(bi, 100, 100);
return biScaled;
}catch (OutOfMemoryError e) {
logger.log(Level.WARNING, "Could not scale image (too large): " + content.getName(), e);
return null;
Image result = Toolkit.getDefaultToolkit().createImage(data);
}
catch (Exception e) {
logger.log(Level.WARNING, "Could not scale image: " + content.getName(), e);
return null;
} finally {
if (inputStream != null) {
try {
inputStream.close();
} catch (IOException ex) {
logger.log(Level.WARNING, "Could not close input stream after resizing thumbnail: " + content.getName(), ex);
}
}
// scale the image
MediaTracker mTracker = new MediaTracker(new JFrame());
mTracker.addImage(result, 1);
try {
mTracker.waitForID(1);
} catch (InterruptedException ex) {
// TODO: maybe make bubble instead
Logger.getLogger(ThumbnailViewNode.class.getName()).log(Level.WARNING, "Error while trying to scale the icon.", ex);
}
int width = result.getWidth(null);
int height = result.getHeight(null);
int max = Math.max(width, height);
double scale = (75 * 100) / max;
// getScaledInstance can't take have width or height be 0, so round
// up by adding 1 after truncating to int.
width = (int) ((width * scale) / 100) + 1;
height = (int) ((height * scale) / 100) + 1;
result = result.getScaledInstance(width, height, Image.SCALE_SMOOTH);
// load the image completely
mTracker.addImage(result, 1);
try {
mTracker.waitForID(1);
} catch (InterruptedException ex) {
// TODO: maybe make bubble instead
Logger.getLogger(ThumbnailViewNode.class.getName()).log(Level.WARNING, "Error while trying to load the icon.", ex);
}
// create 75x75 image for the icon with the icon on the center
BufferedImage combined = new BufferedImage(75, 75, BufferedImage.TYPE_INT_ARGB);
Graphics2D g = (Graphics2D) combined.getGraphics();
g.setColor(Color.WHITE);
g.setBackground(Color.WHITE);
g.drawImage(result, (75 - width) / 2, (75 - height) / 2, null);
return Toolkit.getDefaultToolkit().createImage(combined.getSource());
}
private static BufferedImage toBufferedImage(Image src) {
int w = src.getWidth(null);
int h = src.getHeight(null);
int type = BufferedImage.TYPE_INT_RGB; // other options
BufferedImage dest = new BufferedImage(w, h, type);
Graphics2D g2 = dest.createGraphics();
g2.drawImage(src, 0, 0, null);
g2.dispose();
return dest;
}
private static File getFile(long id) {
return new File(Case.getCurrentCase().getCacheDirectory() + File.separator + id + ".jpg");
}
}

View File

@ -595,14 +595,18 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat
/**
* Event handler to run when selection changed
*
*
* TODO this needs to be revised
*
*
* @param oldNodes
* @param newNodes
* @param newNodes
*/
private void respondSelection(final Node[] oldNodes, final Node[] newNodes) {
if (!Case.isCaseOpen()) {
//handle in-between condition when case is being closed
//and legacy selection events are pumped
return;
}
//this looks redundant?
// if (getSelectedNode() == null && oldNodes != null) {
// try {

View File

@ -899,7 +899,14 @@ public class IngestManager {
logger.log(Level.SEVERE, "Error: unexpected exception from module: " + module.getName(), e);
stats.addError(module);
}
}
catch (OutOfMemoryError e) {
logger.log(Level.SEVERE, "Error: out of memory from module: " + module.getName(), e);
stats.addError(module);
}
} //end for every module
//free the internal file resource after done with every module
fileToProcess.close();
int newTotalEnqueuedFiles = fileScheduler.getFilesEnqueuedEst();
if (newTotalEnqueuedFiles > totalEnqueuedFiles) {
@ -915,7 +922,8 @@ public class IngestManager {
}
//--totalEnqueuedFiles;
} //end of this AbstractFile
} //end of for every AbstractFile
logger.log(Level.INFO, "IngestManager: Finished processing files");
return null;
}
@ -933,8 +941,7 @@ public class IngestManager {
}
logger.log(Level.INFO, PlatformUtil.getAllMemUsageInfo());
logger.log(Level.INFO, "Freeing resources post file pipeline run");
System.gc();
logger.log(Level.INFO, "Freeing jvm heap resources post file pipeline run");
System.gc();
logger.log(Level.INFO, PlatformUtil.getAllMemUsageInfo());

View File

@ -30,11 +30,11 @@ import org.imgscalr.Scalr.Method;
public class ScalrWrapper {
public static synchronized BufferedImage resize(BufferedImage input, int width, int height) {
return Scalr.resize(input, width, height);
return Scalr.resize(input, width, height, Scalr.OP_ANTIALIAS);
}
public static synchronized BufferedImage resize(BufferedImage input, int size) {
return Scalr.resize(input, size);
return Scalr.resize(input, size, Scalr.OP_ANTIALIAS);
}
public static synchronized BufferedImage resizeHighQuality(BufferedImage input, int width, int height) {
@ -42,6 +42,10 @@ public class ScalrWrapper {
}
public static synchronized BufferedImage resizeFast(BufferedImage input, int size) {
return Scalr.resize(input, Method.SPEED, size, Scalr.OP_ANTIALIAS);
return Scalr.resize(input, Method.SPEED, Scalr.Mode.AUTOMATIC, size, Scalr.OP_ANTIALIAS);
}
public static synchronized BufferedImage resizeFast(BufferedImage input, int width, int height) {
return Scalr.resize(input, Method.SPEED, Scalr.Mode.AUTOMATIC, width, height, Scalr.OP_ANTIALIAS);
}
}

View File

@ -1,6 +1,6 @@
Manifest-Version: 1.0
AutoUpdate-Show-In-Client: true
OpenIDE-Module: org.sleuthkit.autopsy.exifparser/3
OpenIDE-Module-Implementation-Version: 6
OpenIDE-Module-Implementation-Version: 7
OpenIDE-Module-Layer: org/sleuthkit/autopsy/exifparser/layer.xml
OpenIDE-Module-Localizing-Bundle: org/sleuthkit/autopsy/exifparser/Bundle.properties

View File

@ -3,4 +3,4 @@ javac.compilerargs=-Xlint -Xlint:-serial
license.file=../LICENSE-2.0.txt
nbm.homepage=http://www.sleuthkit.org/autopsy/
nbm.needs.restart=true
spec.version.base=1.0
spec.version.base=1.1

View File

@ -12,7 +12,7 @@
<compile-dependency/>
<run-dependency>
<release-version>7</release-version>
<specification-version>5.0</specification-version>
<specification-version>5.2</specification-version>
</run-dependency>
</dependency>
</module-dependencies>

View File

@ -1,7 +1,7 @@
Manifest-Version: 1.0
AutoUpdate-Show-In-Client: true
OpenIDE-Module: org.sleuthkit.autopsy.hashdatabase/3
OpenIDE-Module-Implementation-Version: 6
OpenIDE-Module-Implementation-Version: 7
OpenIDE-Module-Layer: org/sleuthkit/autopsy/hashdatabase/layer.xml
OpenIDE-Module-Localizing-Bundle: org/sleuthkit/autopsy/hashdatabase/Bundle.properties

View File

@ -3,4 +3,4 @@ javac.compilerargs=-Xlint -Xlint:-serial
license.file=../LICENSE-2.0.txt
nbm.homepage=http://www.sleuthkit.org/autopsy/
nbm.needs.restart=true
spec.version.base=1.1
spec.version.base=1.2

View File

@ -78,7 +78,7 @@
<compile-dependency/>
<run-dependency>
<release-version>7</release-version>
<specification-version>5.0</specification-version>
<specification-version>5.2</specification-version>
</run-dependency>
</dependency>
</module-dependencies>

View File

@ -1,7 +1,7 @@
Manifest-Version: 1.0
AutoUpdate-Show-In-Client: true
OpenIDE-Module: org.sleuthkit.autopsy.keywordsearch/5
OpenIDE-Module-Implementation-Version: 6
OpenIDE-Module-Implementation-Version: 7
OpenIDE-Module-Install: org/sleuthkit/autopsy/keywordsearch/Installer.class
OpenIDE-Module-Layer: org/sleuthkit/autopsy/keywordsearch/layer.xml
OpenIDE-Module-Localizing-Bundle: org/sleuthkit/autopsy/keywordsearch/Bundle.properties

View File

@ -3,4 +3,4 @@ javac.compilerargs=-Xlint -Xlint:-serial
license.file=../LICENSE-2.0.txt
nbm.homepage=http://www.sleuthkit.org/autopsy/
nbm.needs.restart=true
spec.version.base=3.0
spec.version.base=3.1

View File

@ -96,7 +96,7 @@
<compile-dependency/>
<run-dependency>
<release-version>7</release-version>
<specification-version>5.0</specification-version>
<specification-version>5.2</specification-version>
</run-dependency>
</dependency>
</module-dependencies>

View File

@ -16,9 +16,9 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.keywordsearch;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import org.sleuthkit.autopsy.coreutils.StringExtract.StringExtractUnicodeTable.SCRIPT;
@ -29,72 +29,124 @@ import org.sleuthkit.datamodel.AbstractFile;
* chunks
*/
interface AbstractFileExtract {
/**
* Common options that can be used by some extractors
*/
enum ExtractOptions {
EXTRACT_UTF16, ///< extract UTF16 text, possible values Boolean.TRUE.toString(), Boolean.FALSE.toString()
EXTRACT_UTF8, ///< extract UTF8 text, possible values Boolean.TRUE.toString(), Boolean.FALSE.toString()
};
//generally text extractors should ignore archives
//and let unpacking modules take case of them
static final List<String> ARCHIVE_MIME_TYPES =
Arrays.asList(
//ignore unstructured binary and compressed data, for which string extraction or unzipper works better
"application/x-7z-compressed",
"application/x-ace-compressed",
"application/x-alz-compressed",
"application/x-arj",
"application/vnd.ms-cab-compressed",
"application/x-cfs-compressed",
"application/x-dgc-compressed",
"application/x-apple-diskimage",
"application/x-gca-compressed",
"application/x-dar",
"application/x-lzx",
"application/x-lzh",
"application/x-rar-compressed",
"application/x-stuffit",
"application/x-stuffitx",
"application/x-gtar",
"application/x-archive",
"application/x-executable",
"application/x-gzip",
"application/zip",
"application/x-zoo",
"application/x-cpio",
"application/x-shar",
"application/x-tar",
"application/x-bzip",
"application/x-bzip2",
"application/x-lzip",
"application/x-lzma",
"application/x-lzop",
"application/x-z",
"application/x-compress");
/**
* Get number of chunks resulted from extracting this AbstractFile
*
* @return the number of chunks produced
*/
int getNumChunks();
/**
* Get the source file associated with this extraction
*
* @return the source AbstractFile
*/
AbstractFile getSourceFile();
/**
* Index the Abstract File
*
* @param sourceFile file to index
* @return true if indexed successfully, false otherwise
* @throws org.sleuthkit.autopsy.keywordsearch.Ingester.IngesterException
* @throws org.sleuthkit.autopsy.keywordsearch.Ingester.IngesterException
*/
boolean index(AbstractFile sourceFile) throws Ingester.IngesterException;
/**
* Sets the scripts to use for the extraction
*
* @param extractScripts scripts to use
* @return true if extractor supports script - specific extraction, false otherwise
* @return true if extractor supports script - specific extraction, false
* otherwise
*/
boolean setScripts(List<SCRIPT> extractScript);
/**
* Get the currently used scripts for extraction
*
* @return scripts currently used or null if not supported
*/
List<SCRIPT> getScripts();
/**
* Get current options
* @return currently used, extractor specific options, or null of not supported
*
* @return currently used, extractor specific options, or null of not
* supported
*/
Map<String,String> getOptions();
Map<String, String> getOptions();
/**
* Set extractor specific options
*
* @param options options to use
*/
void setOptions(Map<String,String> options);
void setOptions(Map<String, String> options);
/**
* Determines if the extractor works only for specified types
* is supportedTypes() or whether is a generic content extractor (such as string extractor)
* @return
* Determines if the extractor works only for specified types is
* supportedTypes() or whether is a generic content extractor (such as
* string extractor)
*
* @return
*/
boolean isContentTypeSpecific();
/**
* Determines if the file content is supported by the extractor,
* if isContentTypeSpecific() returns true.
* Determines if the file content is supported by the extractor if
* isContentTypeSpecific() returns true.
*
* @param file to test if its content should be supported
* @param detectedFormat mime-type with detected format (such as text/plain)
* or null if not detected
* @return true if the file content is supported, false otherwise
*/
boolean isSupported(AbstractFile file);
boolean isSupported(AbstractFile file, String detectedFormat);
}

View File

@ -22,6 +22,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.Reader;
import java.nio.charset.Charset;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.logging.Level;
@ -49,9 +50,15 @@ public class AbstractFileHtmlExtract implements AbstractFileExtract {
private AbstractFile sourceFile;
private int numChunks = 0;
//private static final String UTF16BOM = "\uFEFF"; disabled prepending of BOM
private static final String[] SUPPORTED_EXTENSIONS = {
"htm", "html", "xhtml", "shtml", "xhtm", "shtm", "css", "js", "php", "jsp"
};
static final List<String> WEB_MIME_TYPES = Arrays.asList(
"application/javascript",
"application/xhtml+xml",
"application/json",
"text/css",
"text/html",
"text/javascript" //"application/xml",
//"application/xml-dtd",
);
AbstractFileHtmlExtract() {
this.module = KeywordSearchIngestModule.getDefault();
@ -67,7 +74,7 @@ public class AbstractFileHtmlExtract implements AbstractFileExtract {
public List<SCRIPT> getScripts() {
return null;
}
@Override
public Map<String, String> getOptions() {
return null;
@ -75,7 +82,6 @@ public class AbstractFileHtmlExtract implements AbstractFileExtract {
@Override
public void setOptions(Map<String, String> options) {
}
@Override
@ -207,13 +213,16 @@ public class AbstractFileHtmlExtract implements AbstractFileExtract {
}
@Override
public boolean isSupported(AbstractFile file) {
String fileNameLower = file.getName().toLowerCase();
for (int i = 0; i < SUPPORTED_EXTENSIONS.length; ++i) {
if (fileNameLower.endsWith(SUPPORTED_EXTENSIONS[i])) {
return true;
}
public boolean isSupported(AbstractFile file, String detectedFormat) {
if (detectedFormat == null) {
return false;
}
return false;
else if (WEB_MIME_TYPES.contains(detectedFormat) ) {
return true;
}
else {
return false;
}
}
}

View File

@ -51,13 +51,7 @@ class AbstractFileStringExtract implements AbstractFileExtract {
private static final SCRIPT DEFAULT_SCRIPT = SCRIPT.LATIN_2;
private final List<SCRIPT> extractScripts = new ArrayList<SCRIPT>();
private Map<String, String> extractOptions = new HashMap<String, String>();
//string extractor extracts from all other than archives
//TODO use content type detection mechanism
static final String[] UNSUPPORTED_EXTENSIONS = {
//Archives
//Note: archive unpacker module will process these instead
"tar", "jar", "zip", "7z", "gzip", "bzip", "bzip2", "gz", "tgz", "cab", "rar", "arj", "dmg", "iso"
};
//disabled prepending of BOM
//static {
@ -185,18 +179,26 @@ class AbstractFileStringExtract implements AbstractFileExtract {
}
@Override
public boolean isSupported(AbstractFile file) {
String fileNameLower = file.getName().toLowerCase();
int dotI = fileNameLower.lastIndexOf(".");
if (dotI == -1 || dotI == fileNameLower.length() - 1) {
return true; //no extension
public boolean isSupported(AbstractFile file, String detectedFormat) {
if (detectedFormat == null) {
return true;
}
final String extension = fileNameLower.substring(dotI + 1);
for (int i = 0; i < UNSUPPORTED_EXTENSIONS.length; ++i) {
if (extension.equals(UNSUPPORTED_EXTENSIONS[i])) {
return false;
}
else if (detectedFormat.equals("application/octet-stream")) {
//any binary unstructured blobs (string extraction will be used)
return true;
}
else if (AbstractFileExtract.ARCHIVE_MIME_TYPES.contains(detectedFormat)) {
return false; //let unzipper take care of it
}
//skip images/video/audio
else if (detectedFormat.contains("image/")
|| detectedFormat.contains("audio/")
|| detectedFormat.contains("video/")
) {
return false;
}
else {
return true;
}
return true;
}
}

View File

@ -66,30 +66,6 @@ public class AbstractFileTikaTextExtract implements AbstractFileExtract {
private int numChunks = 0;
//private static final String UTF16BOM = "\uFEFF"; disabled prepending of BOM
private final ExecutorService tikaParseExecutor = Executors.newSingleThreadExecutor();
// TODO: use type detection mechanism instead, and maintain supported MimeTypes, not extensions
// supported extensions list from http://www.lucidimagination.com/devzone/technical-articles/content-extraction-tika
static final String[] SUPPORTED_EXTENSIONS = {
//Archive (to be removed when we have archive module
/// handled by 7zip module now "tar", "jar", "zip", "gzip", "bzip2", "gz", "tgz", "ar", "cpio",
//MS Office
"doc", "dot", "docx", "docm", "dotx", "dotm",
"xls", "xlw", "xlt", "xlsx", "xlsm", "xltx", "xltm",
"ppt", "pps", "pot", "pptx", "pptm", "potx", "potm",
//Open Office
"odf", "odt", "ott", "ods", "ots", "odp", "otp",
"sxw", "stw", "sxc", "stc", "sxi", "sxi",
"sdw", "sdc", "vor", "sgl",
//rich text, pdf
"rtf", "pdf",
//html (other extractors take priority)
"html", "htm", "xhtml",
//text
"txt", "log", "manifest",
//code
"class",
//images, media, other
"bmp", "gif", "png", "jpeg", "jpg", "tiff", "mp3", "flv", "aiff", "au", "midi", "wav",
"pst", "xml", "class", "dwg", "eml", "emlx", "mbox", "mht"};
AbstractFileTikaTextExtract() {
this.module = KeywordSearchIngestModule.getDefault();
@ -282,19 +258,27 @@ public class AbstractFileTikaTextExtract implements AbstractFileExtract {
}
@Override
public boolean isSupported(AbstractFile file) {
String fileNameLower = file.getName().toLowerCase();
int dotI = fileNameLower.lastIndexOf(".");
if (dotI == -1 || dotI == fileNameLower.length() - 1) {
return false; //no extension
public boolean isSupported(AbstractFile file, String detectedFormat) {
if (detectedFormat == null) {
return false;
} else if (detectedFormat.equals("application/octet-stream")) {
//any binary unstructured blobs (string extraction will be used)
return false;
} else if (AbstractFileExtract.ARCHIVE_MIME_TYPES.contains(detectedFormat)) {
return false;
} //skip video other than flv (tika supports flv only)
else if (detectedFormat.contains("video/")
&& !detectedFormat.equals("video/x-flv")) {
return false;
}
final String extension = fileNameLower.substring(dotI + 1);
for (int i = 0; i < SUPPORTED_EXTENSIONS.length; ++i) {
if (extension.equals(SUPPORTED_EXTENSIONS[i])) {
return true;
}
}
return false;
//TODO might need to add more mime-types to ignore
//default to true, which includes
//text, docs, pdf and others
return true;
}
/**

View File

@ -20,6 +20,8 @@ package org.sleuthkit.autopsy.keywordsearch;
import java.awt.event.ActionEvent;
import java.awt.event.ActionListener;
import java.io.IOException;
import java.io.InputStream;
import java.lang.Long;
import java.util.ArrayList;
import java.util.Collection;
@ -37,10 +39,12 @@ import org.sleuthkit.autopsy.coreutils.Logger;
import javax.swing.SwingUtilities;
import javax.swing.SwingWorker;
import javax.swing.Timer;
import org.apache.tika.Tika;
import org.netbeans.api.progress.aggregate.AggregateProgressFactory;
import org.netbeans.api.progress.aggregate.AggregateProgressHandle;
import org.netbeans.api.progress.aggregate.ProgressContributor;
import org.openide.util.Cancellable;
import org.openide.util.Exceptions;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.coreutils.EscapeUtil;
import org.sleuthkit.autopsy.coreutils.StopWatch;
@ -57,6 +61,7 @@ import org.sleuthkit.datamodel.BlackboardArtifact;
import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE;
import org.sleuthkit.datamodel.BlackboardAttribute;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.ReadContentInputStream;
import org.sleuthkit.datamodel.SleuthkitCase;
import org.sleuthkit.datamodel.TskCoreException;
import org.sleuthkit.datamodel.TskData;
@ -123,6 +128,7 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
private static AbstractFileStringExtract stringExtractor;
private boolean initialized = false;
private KeywordSearchConfigurationPanel panel;
private Tika tikaFormatDetector;
private enum IngestStatus {
@ -147,7 +153,7 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
}
@Override
public ProcessResult process(PipelineContext<IngestModuleAbstractFile>pipelineContext, AbstractFile abstractFile) {
public ProcessResult process(PipelineContext<IngestModuleAbstractFile> pipelineContext, AbstractFile abstractFile) {
if (initialized == false) //error initializing indexing/Solr
{
@ -189,7 +195,6 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
return ProcessResult.OK;
}
/**
* After all files are ingested, execute final index commit and final search
* Cleanup resources, threads, timers
@ -297,6 +302,8 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
keywordLists.clear();
keywordToList.clear();
tikaFormatDetector = null;
initialized = false;
}
@ -338,6 +345,8 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
caseHandle = Case.getCurrentCase().getSleuthkitCase();
tikaFormatDetector = new Tika();
ingester = Server.getIngester();
final Server server = KeywordSearch.getServer();
@ -659,18 +668,20 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
* index
* @param stringsOnly true if use string extraction, false if to use a
* content-type specific text extractor
* @param detectedFormat mime-type detected, or null if none detected
* @return true if the file was indexed, false otherwise
* @throws IngesterException exception thrown if indexing failed
*/
private boolean extractIndex(AbstractFile aFile, boolean stringsOnly) throws IngesterException {
private boolean extractIndex(AbstractFile aFile, boolean stringsOnly, String detectedFormat) throws IngesterException {
AbstractFileExtract fileExtract = null;
if (stringsOnly && stringExtractor.isSupported(aFile)) {
if (stringsOnly && stringExtractor.isSupported(aFile, detectedFormat)) {
fileExtract = stringExtractor;
} else {
//go over available text extractors and pick the first one (most specific one)
//not only strings
//go over available text extractors in order, and pick the first one (most specific one)
for (AbstractFileExtract fe : textExtractors) {
if (fe.isSupported(aFile)) {
if (fe.isSupported(aFile, detectedFormat)) {
fileExtract = fe;
break;
}
@ -678,7 +689,8 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
}
if (fileExtract == null) {
logger.log(Level.INFO, "No supported file extractor found for file: " + aFile.getId() + " " + aFile.getName());
logger.log(Level.INFO, "No text extractor found for file id:"
+ aFile.getId() + ", name: " + aFile.getName() + ", detected format: " + detectedFormat);
return false;
}
@ -688,10 +700,16 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
return fileExtract.index(aFile);
}
private boolean isTextExtractSupported(AbstractFile aFile) {
/**
* Check with every extractor if it supports the file with the detected format
* @param aFile file to check for
* @param detectedFormat mime-type with detected format (such as text/plain) or null if not detected
* @return true if text extraction is supported
*/
private boolean isTextExtractSupported(AbstractFile aFile, String detectedFormat) {
for (AbstractFileExtract extractor : textExtractors) {
if (extractor.isContentTypeSpecific() == true
&& extractor.isSupported(aFile)) {
&& extractor.isSupported(aFile, detectedFormat)) {
return true;
}
}
@ -706,8 +724,8 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
if (aType.equals(TskData.TSK_DB_FILES_TYPE_ENUM.VIRTUAL_DIR)) {
//skip indexing of virtual dirs (no content, no real name) - will index children files
return;
}
}
boolean isUnallocFile = aType.equals(TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS);
final long size = aFile.getSize();
@ -725,18 +743,39 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
return;
}
boolean extractTextSupported = isTextExtractSupported(aFile);
//use Tika to detect the format
String detectedFormat = null;
InputStream is = null;
try {
is = new ReadContentInputStream(aFile);
detectedFormat = tikaFormatDetector.detect(is, aFile.getName());
} catch (Exception e) {
logger.log(Level.WARNING, "Could not detect format using tika for file: " + aFile, e);
} finally {
if (is != null) {
try {
is.close();
} catch (IOException ex) {
logger.log(Level.WARNING, "Could not close stream after detecting format using tika for file: "
+ aFile, ex);
}
}
}
logger.log(Level.INFO, "Detected format: " + aFile.getName() + " " + detectedFormat);
boolean extractTextSupported = isTextExtractSupported(aFile, detectedFormat);
if (isUnallocFile == false && extractTextSupported) {
//we know it's an allocated FS file
//extract text with one of the extractors, divide into chunks and index with Solr
try {
//logger.log(Level.INFO, "indexing: " + aFile.getName());
if (!extractIndex(aFile, false)) {
if (!extractIndex(aFile, false, detectedFormat)) {
logger.log(Level.WARNING, "Failed to extract text and ingest, file '" + aFile.getName() + "' (id: " + aFile.getId() + ").");
ingestStatus.put(aFile.getId(), IngestStatus.SKIPPED);
//try to extract strings, if a file
if (aFile.isFile() == true) {
processNonIngestible(aFile);
processNonIngestible(aFile, detectedFormat);
}
} else {
@ -749,7 +788,7 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
ingestStatus.put(aFile.getId(), IngestStatus.SKIPPED);
//try to extract strings, if a file
if (aFile.isFile() == true) {
processNonIngestible(aFile);
processNonIngestible(aFile, detectedFormat);
}
} catch (Exception e) {
@ -758,18 +797,18 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
ingestStatus.put(aFile.getId(), IngestStatus.SKIPPED);
//try to extract strings if a file
if (aFile.isFile() == true) {
processNonIngestible(aFile);
processNonIngestible(aFile, detectedFormat);
}
}
} else {
//unallocated file or unsupported content type by Solr
processNonIngestible(aFile);
processNonIngestible(aFile, detectedFormat);
}
}
private boolean processNonIngestible(AbstractFile aFile) {
private boolean processNonIngestible(AbstractFile aFile, String detectedFormat) {
try {
if (!extractIndex(aFile, true)) {
if (!extractIndex(aFile, true, detectedFormat)) {
logger.log(Level.WARNING, "Failed to extract strings and ingest, file '" + aFile.getName() + "' (id: " + aFile.getId() + ").");
ingestStatus.put(aFile.getId(), IngestStatus.SKIPPED);
return false;
@ -1025,7 +1064,7 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
detailsSb.append("<tr>");
detailsSb.append("<th>File</th>");
detailsSb.append("<td>").append(hitFile.getParentPath()).append(hitFile.getName()).append("</td>");
detailsSb.append("</tr>");

View File

@ -1,4 +1,4 @@
---------------- VERSION Current (development) --------------
---------------- VERSION 3.0.5 --------------
New features:
- Archive extractor ingest module (uses 7zip)
@ -6,6 +6,8 @@ New features:
Improvements:
- Sleuthkit-4.0.2 and libewf-20130128
- improved image loading in Media View and Thumbnail View (faster loading, handles large files better)
- improve Keyword Search file indexing (use detected mime-type instead of file extension)
- show children counts in directory tree
Bugfixes:
@ -16,6 +18,7 @@ Bugfixes:
- exif module better jpeg detection using signature and not only file extension.
- The "media view" tab is inactive for deleted files (#165)
---------------- VERSION 3.0.4 --------------
New features:

View File

@ -1,6 +1,6 @@
Manifest-Version: 1.0
OpenIDE-Module: org.sleuthkit.autopsy.recentactivity/4
OpenIDE-Module-Implementation-Version: 6
OpenIDE-Module-Implementation-Version: 7
OpenIDE-Module-Layer: org/sleuthkit/autopsy/recentactivity/layer.xml
OpenIDE-Module-Localizing-Bundle: org/sleuthkit/autopsy/recentactivity/Bundle.properties
OpenIDE-Module-Requires:

View File

@ -4,4 +4,4 @@ javac.compilerargs=-Xlint -Xlint:-serial
license.file=../LICENSE-2.0.txt
nbm.homepage=http://www.sleuthkit.org/autopsy/
nbm.needs.restart=true
spec.version.base=2.0
spec.version.base=2.1

View File

@ -36,7 +36,7 @@
<compile-dependency/>
<run-dependency>
<release-version>7</release-version>
<specification-version>5.0</specification-version>
<specification-version>5.2</specification-version>
</run-dependency>
</dependency>
</module-dependencies>

View File

@ -21,7 +21,7 @@
<compile-dependency/>
<run-dependency>
<release-version>7</release-version>
<specification-version>5.0</specification-version>
<specification-version>5.2</specification-version>
</run-dependency>
</dependency>
<dependency>

View File

@ -1,6 +1,6 @@
Manifest-Version: 1.0
AutoUpdate-Show-In-Client: false
OpenIDE-Module: org.sleuthkit.autopsy.testing/3
OpenIDE-Module-Implementation-Version: 5
OpenIDE-Module-Implementation-Version: 6
OpenIDE-Module-Localizing-Bundle: org/sleuthkit/autopsy/testing/Bundle.properties

View File

@ -3,4 +3,4 @@ javac.compilerargs=-Xlint -Xlint:-serial
license.file=../LICENSE-2.0.txt
nbm.homepage=http://www.sleuthkit.org/autopsy/
nbm.needs.restart=true
spec.version.base=1.1
spec.version.base=1.2

View File

@ -12,7 +12,7 @@
<compile-dependency/>
<run-dependency>
<release-version>7</release-version>
<specification-version>5.0</specification-version>
<specification-version>5.2</specification-version>
</run-dependency>
</dependency>
<dependency>

View File

@ -4,4 +4,5 @@ OpenIDE-Module-Layer: org/sleuthkit/autopsy/timeline/layer.xml
OpenIDE-Module-Localizing-Bundle: org/sleuthkit/autopsy/timeline/Bundle.properties
OpenIDE-Module-Requires: org.openide.windows.WindowManager
OpenIDE-Module-Specification-Version: 1.0
OpenIDE-Module-Implementation-Version: 1

View File

@ -94,7 +94,7 @@
<compile-dependency/>
<run-dependency>
<release-version>7</release-version>
<specification-version>3.0</specification-version>
<specification-version>5.2</specification-version>
</run-dependency>
</dependency>
<dependency>

View File

@ -1,4 +1,9 @@
OpenIDE-Module-Display-Category=External Viewers
OpenIDE-Module-Long-Description=\
Displays user activity as an interactive timeline chart with year, month and day granularity. \n\
Events for a selected day are viewable in the built-in result and content viewers.
OpenIDE-Module-Name=Timeline
CTL_MakeTimeline="Make Timeline (Beta)"
OpenIDE-Module-Short-Description=Displays user activity timeline
TimelineProgressDialog.jLabel1.text=Creating timeline . . .
TimelineFrame.title=Timeline

View File

@ -1,7 +1,7 @@
Manifest-Version: 1.0
AutoUpdate-Show-In-Client: true
OpenIDE-Module: org.sleuthkit.autopsy.thunderbirdparser/3
OpenIDE-Module-Implementation-Version: 6
OpenIDE-Module-Implementation-Version: 7
OpenIDE-Module-Layer: org/sleuthkit/autopsy/thunderbirdparser/layer.xml
OpenIDE-Module-Localizing-Bundle: org/sleuthkit/autopsy/thunderbirdparser/Bundle.properties

View File

@ -3,4 +3,4 @@ javac.compilerargs=-Xlint -Xlint:-serial
license.file=../LICENSE-2.0.txt
nbm.homepage=http://www.sleuthkit.org/autopsy/
nbm.needs.restart=true
spec.version.base=1.0
spec.version.base=1.1

View File

@ -12,7 +12,7 @@
<compile-dependency/>
<run-dependency>
<release-version>7</release-version>
<specification-version>5.0</specification-version>
<specification-version>5.2</specification-version>
</run-dependency>
</dependency>
<dependency>
@ -21,7 +21,7 @@
<compile-dependency/>
<run-dependency>
<release-version>5</release-version>
<specification-version>3.0</specification-version>
<specification-version>3.1</specification-version>
</run-dependency>
</dependency>
</module-dependencies>