mstoeck3/autopsy-flatpak
1
0
Fork 0
mirror of https://github.com/overcuriousity/autopsy-flatpak.git synced 2025-07-06 21:00:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #606 from SynapticNulship/onemsg2

Browse Source 
ingest message updates
This commit is contained in:
Richard Cordovano 2014-04-10 13:06:47 -04:00
commit 57a767f39f
10 changed files with 306 additions and 235 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

126
Core/src/org/sleuthkit/autopsy/ingest/IngestMessagePanel.java
View File

@ -395,18 +395,18 @@ class IngestMessagePanel extends JPanel implements TableModelListener {
} }
@Override @Override
synchronized public int getRowCount() { public synchronized int getRowCount() {
return getNumberGroups(); return getNumberGroups();
} }
public void markAllSeen() { public synchronized void markAllSeen() {
for (TableEntry entry : messageData) { for (TableEntry entry : messageData) {
entry.hasBeenSeen(true); entry.hasBeenSeen(true);
} }
fireTableChanged(new TableModelEvent(this)); fireTableChanged(new TableModelEvent(this));
} }
public int getNumberNewMessages() { public synchronized int getNumberNewMessages() {
int newMessages = 0; int newMessages = 0;
for (TableEntry entry : messageData) { for (TableEntry entry : messageData) {
if (!entry.hasBeenSeen()) { if (!entry.hasBeenSeen()) {
@ -416,11 +416,11 @@ class IngestMessagePanel extends JPanel implements TableModelListener {
return newMessages; return newMessages;
} }
synchronized int getNumberGroups() { public synchronized int getNumberGroups() {
return messageData.size(); return messageData.size();
} }
synchronized int getNumberMessages() { public synchronized int getNumberMessages() {
int total = 0; int total = 0;
for (TableEntry e : messageData) { for (TableEntry e : messageData) {
total += e.messageGroup.getCount(); total += e.messageGroup.getCount();
@ -428,7 +428,7 @@ class IngestMessagePanel extends JPanel implements TableModelListener {
return total; return total;
} }
synchronized int getNumberUnreadMessages() { public synchronized int getNumberUnreadMessages() {
int total = 0; int total = 0;
for (TableEntry e : messageData) { for (TableEntry e : messageData) {
if (!e.hasBeenVisited) { if (!e.hasBeenVisited) {
@ -438,7 +438,7 @@ class IngestMessagePanel extends JPanel implements TableModelListener {
return total; return total;
} }
synchronized int getNumberUnreadGroups() { public synchronized int getNumberUnreadGroups() {
int total = 0; int total = 0;
for (TableEntry e : messageData) { for (TableEntry e : messageData) {
if (!e.hasBeenVisited) { if (!e.hasBeenVisited) {
@ -513,7 +513,7 @@ class IngestMessagePanel extends JPanel implements TableModelListener {
return ret; return ret;
} }
synchronized public void addMessage(IngestMessage m) { public synchronized void addMessage(IngestMessage m) {
//check how many messages per module with the same uniqness //check how many messages per module with the same uniqness
//and add to existing group or create a new group //and add to existing group or create a new group
String moduleName = m.getSource(); String moduleName = m.getSource();
@ -628,15 +628,27 @@ class IngestMessagePanel extends JPanel implements TableModelListener {
} }
public synchronized boolean isVisited(int rowNumber) { public synchronized boolean isVisited(int rowNumber) {
return messageData.get(rowNumber).hasBeenVisited(); if (rowNumber < messageData.size()) {
return messageData.get(rowNumber).hasBeenVisited();
} else {
return false;
}
} }
public synchronized MessageType getMessageType(int rowNumber) { public synchronized MessageType getMessageType(int rowNumber) {
return messageData.get(rowNumber).messageGroup.getMessageType(); if (rowNumber < messageData.size()) {
return messageData.get(rowNumber).messageGroup.getMessageType();
} else {
return null;
}
} }
public synchronized IngestMessageGroup getMessageGroup(int rowNumber) { public synchronized IngestMessageGroup getMessageGroup(int rowNumber) {
return messageData.get(rowNumber).messageGroup; if (rowNumber < messageData.size()) {
return messageData.get(rowNumber).messageGroup;
} else {
return null;
}
} }
public synchronized void reSort(boolean chronoLogical) { public synchronized void reSort(boolean chronoLogical) {
@ -701,26 +713,22 @@ class IngestMessagePanel extends JPanel implements TableModelListener {
messages.add(message); messages.add(message);
} }
List<IngestMessage> getMessages() { synchronized void add(IngestMessage message) {
return messages;
}
void add(IngestMessage message) {
messages.add(message); messages.add(message);
} }
//add all messages from another group //add all messages from another group
void addAll(IngestMessageGroup group) { synchronized void addAll(IngestMessageGroup group) {
for (IngestMessage m : group.getMessages()) { for (IngestMessage m : messages) {
messages.add(m); messages.add(m);
} }
} }
int getCount() { synchronized int getCount() {
return messages.size(); return messages.size();
} }
String getDetails() { synchronized String getDetails() {
StringBuilder b = new StringBuilder(""); StringBuilder b = new StringBuilder("");
for (IngestMessage m : messages) { for (IngestMessage m : messages) {
String details = m.getDetails(); String details = m.getDetails();
@ -739,7 +747,7 @@ class IngestMessagePanel extends JPanel implements TableModelListener {
* return color corresp to priority * return color corresp to priority
* @return * @return
*/ */
Color getColor() { synchronized Color getColor() {
int count = messages.size(); int count = messages.size();
if (count == 1) { if (count == 1) {
return VERY_HIGH_PRI_COLOR; return VERY_HIGH_PRI_COLOR;
@ -757,7 +765,7 @@ class IngestMessagePanel extends JPanel implements TableModelListener {
* used for chrono sort * used for chrono sort
* @return * @return
*/ */
Date getDatePosted() { synchronized Date getDatePosted() {
return messages.get(messages.size() - 1).getDatePosted(); return messages.get(messages.size() - 1).getDatePosted();
} }
@ -765,35 +773,35 @@ class IngestMessagePanel extends JPanel implements TableModelListener {
* get subject of the first message * get subject of the first message
* @return * @return
*/ */
String getSubject() { synchronized String getSubject() {
return messages.get(0).getSubject(); return messages.get(0).getSubject();
} }
/* /*
* return unique key, should be the same for all msgs * return unique key, should be the same for all msgs
*/ */
String getUniqueKey() { synchronized String getUniqueKey() {
return messages.get(0).getUniqueKey(); return messages.get(0).getUniqueKey();
} }
/* /*
* return source module, should be the same for all msgs * return source module, should be the same for all msgs
*/ */
String getSource() { synchronized String getSource() {
return messages.get(0).getSource(); return messages.get(0).getSource();
} }
/* /*
* return data of the first message * return data of the first message
*/ */
BlackboardArtifact getData() { synchronized BlackboardArtifact getData() {
return messages.get(0).getData(); return messages.get(0).getData();
} }
/* /*
* return message type, should be the same for all msgs * return message type, should be the same for all msgs
*/ */
IngestMessage.MessageType getMessageType() { synchronized IngestMessage.MessageType getMessageType() {
return messages.get(0).getMessageType(); return messages.get(0).getMessageType();
} }
} }
@ -858,16 +866,17 @@ class IngestMessagePanel extends JPanel implements TableModelListener {
cell.setFont(new Font("", Font.PLAIN, 16)); cell.setFont(new Font("", Font.PLAIN, 16));
final IngestMessageGroup messageGroup = tableModel.getMessageGroup(row); final IngestMessageGroup messageGroup = tableModel.getMessageGroup(row);
MessageType mt = messageGroup.getMessageType(); if (messageGroup != null) {
if (mt == MessageType.ERROR) { MessageType mt = messageGroup.getMessageType();
cell.setBackground(ERROR_COLOR); if (mt == MessageType.ERROR) {
} else if (mt == MessageType.WARNING) { cell.setBackground(ERROR_COLOR);
cell.setBackground(Color.orange); } else if (mt == MessageType.WARNING) {
} else { cell.setBackground(Color.orange);
//cell.setBackground(table.getBackground()); } else {
cell.setBackground(messageGroup.getColor()); //cell.setBackground(table.getBackground());
cell.setBackground(messageGroup.getColor());
}
} }
return cell; return cell;
} }
} }
@ -898,16 +907,17 @@ class IngestMessagePanel extends JPanel implements TableModelListener {
} }
final IngestMessageGroup messageGroup = tableModel.getMessageGroup(row); final IngestMessageGroup messageGroup = tableModel.getMessageGroup(row);
MessageType mt = messageGroup.getMessageType(); if (messageGroup != null) {
if (mt == MessageType.ERROR) { MessageType mt = messageGroup.getMessageType();
cell.setBackground(ERROR_COLOR); if (mt == MessageType.ERROR) {
} else if (mt == MessageType.WARNING) { cell.setBackground(ERROR_COLOR);
cell.setBackground(Color.orange); } else if (mt == MessageType.WARNING) {
} else { cell.setBackground(Color.orange);
//cell.setBackground(table.getBackground()); } else {
cell.setBackground(messageGroup.getColor()); //cell.setBackground(table.getBackground());
cell.setBackground(messageGroup.getColor());
}
} }
return cell; return cell;
} }
} }
@ -933,14 +943,16 @@ class IngestMessagePanel extends JPanel implements TableModelListener {
Component cell = super.getTableCellRendererComponent(table, aValue, isSelected, hasFocus, row, column); Component cell = super.getTableCellRendererComponent(table, aValue, isSelected, hasFocus, row, column);
final IngestMessageGroup messageGroup = tableModel.getMessageGroup(row); final IngestMessageGroup messageGroup = tableModel.getMessageGroup(row);
MessageType mt = messageGroup.getMessageType(); if (messageGroup != null) {
if (mt == MessageType.ERROR) { MessageType mt = messageGroup.getMessageType();
cell.setBackground(ERROR_COLOR); if (mt == MessageType.ERROR) {
} else if (mt == MessageType.WARNING) { cell.setBackground(ERROR_COLOR);
cell.setBackground(Color.orange); } else if (mt == MessageType.WARNING) {
} else { cell.setBackground(Color.orange);
//cell.setBackground(table.getBackground()); } else {
cell.setBackground(messageGroup.getColor()); //cell.setBackground(table.getBackground());
cell.setBackground(messageGroup.getColor());
}
} }
return cell; return cell;
@ -974,9 +986,11 @@ class IngestMessagePanel extends JPanel implements TableModelListener {
messageTable.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR)); messageTable.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR));
//check if has details //check if has details
IngestMessageGroup m = getMessageGroup(selected); IngestMessageGroup m = getMessageGroup(selected);
String details = m.getDetails(); if (m != null) {
if (details != null && !details.equals("")) { String details = m.getDetails();
mainPanel.showDetails(selected); if (details != null && !details.equals("")) {
mainPanel.showDetails(selected);
}
} }
messageTable.setCursor(null); messageTable.setCursor(null);
} }

21
Core/src/org/sleuthkit/autopsy/ingest/IngestModuleAdapter.java
View File

@ -18,11 +18,32 @@
*/ */
package org.sleuthkit.autopsy.ingest; package org.sleuthkit.autopsy.ingest;
import java.util.HashMap;
/** /**
* An adapter that provides a default implementation of the IngestModule * An adapter that provides a default implementation of the IngestModule
* interface. * interface.
*/ */
public abstract class IngestModuleAdapter implements IngestModule { public abstract class IngestModuleAdapter implements IngestModule {
// Maps a JobId to the count of instances
static HashMap<Long, Long> moduleRefCount = new HashMap<>();
public static synchronized long moduleRefCountIncrement(long jobId) {
long count = moduleRefCount.containsKey(jobId) ? moduleRefCount.get(jobId) : 0;
long nextCount = count + 1;
moduleRefCount.put(jobId, nextCount);
return nextCount;
}
public static synchronized long moduleRefCountDecrementAndGet(long jobId) {
if (moduleRefCount.containsKey(jobId)) {
long count = moduleRefCount.get(jobId);
moduleRefCount.put(jobId, --count);
return count;
} else {
return 0;
}
}
@Override @Override
public void startUp(IngestJobContext context) throws IngestModuleException { public void startUp(IngestJobContext context) throws IngestModuleException {

27
Core/src/org/sleuthkit/autopsy/modules/exif/ExifParserFileIngestModule.java
View File

@ -32,11 +32,13 @@ import java.io.InputStream;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collection; import java.util.Collection;
import java.util.Date; import java.util.Date;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.logging.Level; import java.util.logging.Level;
import org.sleuthkit.autopsy.coreutils.ImageUtils; import org.sleuthkit.autopsy.coreutils.ImageUtils;
import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.ingest.IngestModuleAdapter; import org.sleuthkit.autopsy.ingest.IngestModuleAdapter;
import org.sleuthkit.autopsy.ingest.FileIngestModule; import org.sleuthkit.autopsy.ingest.FileIngestModule;
import org.sleuthkit.autopsy.ingest.IngestJobContext;
import org.sleuthkit.autopsy.ingest.IngestServices; import org.sleuthkit.autopsy.ingest.IngestServices;
import org.sleuthkit.autopsy.ingest.ModuleDataEvent; import org.sleuthkit.autopsy.ingest.ModuleDataEvent;
import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.AbstractFile;
@ -57,12 +59,20 @@ public final class ExifParserFileIngestModule extends IngestModuleAdapter implem
private static final Logger logger = Logger.getLogger(ExifParserFileIngestModule.class.getName()); private static final Logger logger = Logger.getLogger(ExifParserFileIngestModule.class.getName());
private final IngestServices services = IngestServices.getInstance(); private final IngestServices services = IngestServices.getInstance();
private int filesProcessed = 0; private AtomicInteger filesProcessed = new AtomicInteger(0);
private boolean filesToFire = false; private volatile boolean filesToFire = false;
private long jobId;
ExifParserFileIngestModule() { ExifParserFileIngestModule() {
} }
@Override
public void startUp(IngestJobContext context) throws IngestModuleException {
jobId = context.getJobId();
IngestModuleAdapter.moduleRefCountIncrement(jobId);
}
@Override @Override
public ProcessResult process(AbstractFile content) { public ProcessResult process(AbstractFile content) {
//skip unalloc //skip unalloc
@ -76,8 +86,8 @@ public final class ExifParserFileIngestModule extends IngestModuleAdapter implem
} }
// update the tree every 1000 files if we have EXIF data that is not being being displayed // update the tree every 1000 files if we have EXIF data that is not being being displayed
filesProcessed++; final int filesProcessedValue = filesProcessed.incrementAndGet();
if ((filesToFire) && (filesProcessed % 1000 == 0)) { if ((filesToFire) && (filesProcessedValue % 1000 == 0)) {
services.fireModuleDataEvent(new ModuleDataEvent(ExifParserModuleFactory.getModuleName(), BlackboardArtifact.ARTIFACT_TYPE.TSK_METADATA_EXIF)); services.fireModuleDataEvent(new ModuleDataEvent(ExifParserModuleFactory.getModuleName(), BlackboardArtifact.ARTIFACT_TYPE.TSK_METADATA_EXIF));
filesToFire = false; filesToFire = false;
} }
@ -187,9 +197,12 @@ public final class ExifParserFileIngestModule extends IngestModuleAdapter implem
@Override @Override
public void shutDown(boolean ingestJobCancelled) { public void shutDown(boolean ingestJobCancelled) {
if (filesToFire) { // We only need to check for this final event on the last module per job
//send the final new data event if (IngestModuleAdapter.moduleRefCountDecrementAndGet(jobId) == 0) {
services.fireModuleDataEvent(new ModuleDataEvent(ExifParserModuleFactory.getModuleName(), BlackboardArtifact.ARTIFACT_TYPE.TSK_METADATA_EXIF)); if (filesToFire) {
//send the final new data event
services.fireModuleDataEvent(new ModuleDataEvent(ExifParserModuleFactory.getModuleName(), BlackboardArtifact.ARTIFACT_TYPE.TSK_METADATA_EXIF));
}
} }
} }
} }

44
Core/src/org/sleuthkit/autopsy/modules/fileextmismatch/FileExtMismatchIngestModule.java
View File

@ -23,6 +23,7 @@ import java.util.Arrays;
import java.util.Collections; import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.concurrent.atomic.AtomicLong;
import java.util.logging.Level; import java.util.logging.Level;
import org.openide.util.NbBundle; import org.openide.util.NbBundle;
import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.Logger;
@ -38,7 +39,6 @@ import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE;
import org.sleuthkit.datamodel.BlackboardAttribute; import org.sleuthkit.datamodel.BlackboardAttribute;
import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskCoreException;
import org.sleuthkit.datamodel.TskData; import org.sleuthkit.datamodel.TskData;
import org.sleuthkit.datamodel.TskData.FileKnown;
import org.sleuthkit.datamodel.TskException; import org.sleuthkit.datamodel.TskException;
/** /**
@ -50,8 +50,9 @@ public class FileExtMismatchIngestModule extends IngestModuleAdapter implements
private final IngestServices services = IngestServices.getInstance(); private final IngestServices services = IngestServices.getInstance();
private final FileExtMismatchDetectorModuleSettings settings; private final FileExtMismatchDetectorModuleSettings settings;
private HashMap<String, String[]> SigTypeToExtMap = new HashMap<>(); private HashMap<String, String[]> SigTypeToExtMap = new HashMap<>();
private long processTime = 0; private long jobId;
private long numFiles = 0; private static AtomicLong processTime = new AtomicLong(0);
private static AtomicLong numFiles = new AtomicLong(0);
FileExtMismatchIngestModule(FileExtMismatchDetectorModuleSettings settings) { FileExtMismatchIngestModule(FileExtMismatchDetectorModuleSettings settings) {
this.settings = settings; this.settings = settings;
@ -59,6 +60,8 @@ public class FileExtMismatchIngestModule extends IngestModuleAdapter implements
@Override @Override
public void startUp(IngestJobContext context) throws IngestModuleException { public void startUp(IngestJobContext context) throws IngestModuleException {
jobId = context.getJobId();
IngestModuleAdapter.moduleRefCountIncrement(jobId);
FileExtMismatchXML xmlLoader = FileExtMismatchXML.getDefault(); FileExtMismatchXML xmlLoader = FileExtMismatchXML.getDefault();
SigTypeToExtMap = xmlLoader.load(); SigTypeToExtMap = xmlLoader.load();
} }
@ -82,8 +85,8 @@ public class FileExtMismatchIngestModule extends IngestModuleAdapter implements
boolean mismatchDetected = compareSigTypeToExt(abstractFile); boolean mismatchDetected = compareSigTypeToExt(abstractFile);
processTime += (System.currentTimeMillis() - startTime); processTime.getAndAdd(System.currentTimeMillis() - startTime);
numFiles++; numFiles.getAndIncrement();
if (mismatchDetected) { if (mismatchDetected) {
// add artifact // add artifact
@ -149,19 +152,22 @@ public class FileExtMismatchIngestModule extends IngestModuleAdapter implements
@Override @Override
public void shutDown(boolean ingestJobCancelled) { public void shutDown(boolean ingestJobCancelled) {
StringBuilder detailsSb = new StringBuilder(); // We only need to post the summary msg from the last module per job
detailsSb.append("<table border='0' cellpadding='4' width='280'>"); if (IngestModuleAdapter.moduleRefCountDecrementAndGet(jobId) == 0) {
detailsSb.append("<tr><td>").append(FileExtMismatchDetectorModuleFactory.getModuleName()).append("</td></tr>"); StringBuilder detailsSb = new StringBuilder();
detailsSb.append("<tr><td>").append( detailsSb.append("<table border='0' cellpadding='4' width='280'>");
NbBundle.getMessage(this.getClass(), "FileExtMismatchIngestModule.complete.totalProcTime")) detailsSb.append("<tr><td>").append(FileExtMismatchDetectorModuleFactory.getModuleName()).append("</td></tr>");
.append("</td><td>").append(processTime).append("</td></tr>\n"); detailsSb.append("<tr><td>").append(
detailsSb.append("<tr><td>").append( NbBundle.getMessage(this.getClass(), "FileExtMismatchIngestModule.complete.totalProcTime"))
NbBundle.getMessage(this.getClass(), "FileExtMismatchIngestModule.complete.totalFiles")) .append("</td><td>").append(processTime.get()).append("</td></tr>\n");
.append("</td><td>").append(numFiles).append("</td></tr>\n"); detailsSb.append("<tr><td>").append(
detailsSb.append("</table>"); NbBundle.getMessage(this.getClass(), "FileExtMismatchIngestModule.complete.totalFiles"))
services.postMessage(IngestMessage.createMessage(IngestMessage.MessageType.INFO, FileExtMismatchDetectorModuleFactory.getModuleName(), .append("</td><td>").append(numFiles.get()).append("</td></tr>\n");
NbBundle.getMessage(this.getClass(), detailsSb.append("</table>");
"FileExtMismatchIngestModule.complete.svcMsg.text"), services.postMessage(IngestMessage.createMessage(IngestMessage.MessageType.INFO, FileExtMismatchDetectorModuleFactory.getModuleName(),
detailsSb.toString())); NbBundle.getMessage(this.getClass(),
"FileExtMismatchIngestModule.complete.svcMsg.text"),
detailsSb.toString()));
}
} }
} }

52
Core/src/org/sleuthkit/autopsy/modules/filetypeid/FileTypeIdIngestModule.java
View File

@ -18,11 +18,12 @@
*/ */
package org.sleuthkit.autopsy.modules.filetypeid; package org.sleuthkit.autopsy.modules.filetypeid;
import java.util.concurrent.atomic.AtomicLong;
import java.util.logging.Level; import java.util.logging.Level;
import org.openide.util.NbBundle; import org.openide.util.NbBundle;
import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.ingest.FileIngestModule; import org.sleuthkit.autopsy.ingest.FileIngestModule;
import org.sleuthkit.autopsy.ingest.IngestJobContext;
import org.sleuthkit.autopsy.ingest.IngestMessage; import org.sleuthkit.autopsy.ingest.IngestMessage;
import org.sleuthkit.autopsy.ingest.IngestServices; import org.sleuthkit.autopsy.ingest.IngestServices;
import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.AbstractFile;
@ -44,8 +45,10 @@ public class FileTypeIdIngestModule extends IngestModuleAdapter implements FileI
private static final Logger logger = Logger.getLogger(FileTypeIdIngestModule.class.getName()); private static final Logger logger = Logger.getLogger(FileTypeIdIngestModule.class.getName());
private static final long MIN_FILE_SIZE = 512; private static final long MIN_FILE_SIZE = 512;
private final FileTypeIdModuleSettings settings; private final FileTypeIdModuleSettings settings;
private long matchTime = 0; private long jobId;
private long numFiles = 0; private static AtomicLong matchTime = new AtomicLong(0);
private static AtomicLong numFiles = new AtomicLong(0);
// The detector. Swap out with a different implementation of FileTypeDetectionInterface as needed. // The detector. Swap out with a different implementation of FileTypeDetectionInterface as needed.
// If desired in the future to be more knowledgable about weird files or rare formats, we could // If desired in the future to be more knowledgable about weird files or rare formats, we could
// actually have a list of detectors which are called in order until a match is found. // actually have a list of detectors which are called in order until a match is found.
@ -55,6 +58,12 @@ public class FileTypeIdIngestModule extends IngestModuleAdapter implements FileI
this.settings = settings; this.settings = settings;
} }
@Override
public void startUp(IngestJobContext context) throws IngestModuleException {
jobId = context.getJobId();
IngestModuleAdapter.moduleRefCountIncrement(jobId);
}
@Override @Override
public ProcessResult process(AbstractFile abstractFile) { public ProcessResult process(AbstractFile abstractFile) {
// skip non-files // skip non-files
@ -75,8 +84,8 @@ public class FileTypeIdIngestModule extends IngestModuleAdapter implements FileI
try { try {
long startTime = System.currentTimeMillis(); long startTime = System.currentTimeMillis();
FileTypeDetectionInterface.FileIdInfo fileId = detector.attemptMatch(abstractFile); FileTypeDetectionInterface.FileIdInfo fileId = detector.attemptMatch(abstractFile);
matchTime += (System.currentTimeMillis() - startTime); matchTime.getAndAdd(System.currentTimeMillis() - startTime);
numFiles++; numFiles.getAndIncrement();
if (!fileId.type.isEmpty()) { if (!fileId.type.isEmpty()) {
// add artifact // add artifact
@ -98,22 +107,25 @@ public class FileTypeIdIngestModule extends IngestModuleAdapter implements FileI
@Override @Override
public void shutDown(boolean ingestJobCancelled) { public void shutDown(boolean ingestJobCancelled) {
StringBuilder detailsSb = new StringBuilder(); // We only need to post the summary msg from the last module per job
detailsSb.append("<table border='0' cellpadding='4' width='280'>"); if (IngestModuleAdapter.moduleRefCountDecrementAndGet(jobId) == 0) {
detailsSb.append("<tr><td>").append(FileTypeIdModuleFactory.getModuleName()).append("</td></tr>"); StringBuilder detailsSb = new StringBuilder();
detailsSb.append("<tr><td>") detailsSb.append("<table border='0' cellpadding='4' width='280'>");
.append(NbBundle.getMessage(this.getClass(), "FileTypeIdIngestModule.complete.totalProcTime")) detailsSb.append("<tr><td>").append(FileTypeIdModuleFactory.getModuleName()).append("</td></tr>");
.append("</td><td>").append(matchTime).append("</td></tr>\n"); detailsSb.append("<tr><td>")
detailsSb.append("<tr><td>") .append(NbBundle.getMessage(this.getClass(), "FileTypeIdIngestModule.complete.totalProcTime"))
.append(NbBundle.getMessage(this.getClass(), "FileTypeIdIngestModule.complete.totalFiles")) .append("</td><td>").append(matchTime.get()).append("</td></tr>\n");
.append("</td><td>").append(numFiles).append("</td></tr>\n"); detailsSb.append("<tr><td>")
detailsSb.append("</table>"); .append(NbBundle.getMessage(this.getClass(), "FileTypeIdIngestModule.complete.totalFiles"))
IngestServices.getInstance().postMessage(IngestMessage.createMessage(IngestMessage.MessageType.INFO, FileTypeIdModuleFactory.getModuleName(), .append("</td><td>").append(numFiles.get()).append("</td></tr>\n");
NbBundle.getMessage(this.getClass(), detailsSb.append("</table>");
"FileTypeIdIngestModule.complete.srvMsg.text"), IngestServices.getInstance().postMessage(IngestMessage.createMessage(IngestMessage.MessageType.INFO, FileTypeIdModuleFactory.getModuleName(),
detailsSb.toString())); NbBundle.getMessage(this.getClass(),
"FileTypeIdIngestModule.complete.srvMsg.text"),
detailsSb.toString()));
}
} }
/** /**
* Validate if a given mime type is in the detector's registry. * Validate if a given mime type is in the detector's registry.
* *

31
Core/src/org/sleuthkit/autopsy/modules/sevenzip/SevenZipIngestModule.java
View File

@ -90,6 +90,7 @@ public final class SevenZipIngestModule extends IngestModuleAdapter implements F
private final byte[] fileHeaderBuffer = new byte[readHeaderSize]; private final byte[] fileHeaderBuffer = new byte[readHeaderSize];
private static final int ZIP_SIGNATURE_BE = 0x504B0304; private static final int ZIP_SIGNATURE_BE = 0x504B0304;
private IngestJobContext context; private IngestJobContext context;
private long jobId;
SevenZipIngestModule() { SevenZipIngestModule() {
} }
@ -97,6 +98,7 @@ public final class SevenZipIngestModule extends IngestModuleAdapter implements F
@Override @Override
public void startUp(IngestJobContext context) throws IngestModuleException { public void startUp(IngestJobContext context) throws IngestModuleException {
this.context = context; this.context = context;
jobId = context.getJobId();
final Case currentCase = Case.getCurrentCase(); final Case currentCase = Case.getCurrentCase();
@ -121,20 +123,23 @@ public final class SevenZipIngestModule extends IngestModuleAdapter implements F
} }
} }
try { // if first instance of this module for this job then check 7zip init
SevenZip.initSevenZipFromPlatformJAR(); if (IngestModuleAdapter.moduleRefCountIncrement(jobId) == 1) {
String platform = SevenZip.getUsedPlatform(); try {
logger.log(Level.INFO, "7-Zip-JBinding library was initialized on supported platform: {0}", platform); SevenZip.initSevenZipFromPlatformJAR();
} catch (SevenZipNativeInitializationException e) { String platform = SevenZip.getUsedPlatform();
logger.log(Level.SEVERE, "Error initializing 7-Zip-JBinding library", e); logger.log(Level.INFO, "7-Zip-JBinding library was initialized on supported platform: {0}", platform);
String msg = NbBundle.getMessage(this.getClass(), "SevenZipIngestModule.init.errInitModule.msg", } catch (SevenZipNativeInitializationException e) {
ArchiveFileExtractorModuleFactory.getModuleName()); logger.log(Level.SEVERE, "Error initializing 7-Zip-JBinding library", e);
String details = NbBundle.getMessage(this.getClass(), "SevenZipIngestModule.init.errCantInitLib", String msg = NbBundle.getMessage(this.getClass(), "SevenZipIngestModule.init.errInitModule.msg",
e.getMessage()); ArchiveFileExtractorModuleFactory.getModuleName());
services.postMessage(IngestMessage.createErrorMessage(ArchiveFileExtractorModuleFactory.getModuleName(), msg, details)); String details = NbBundle.getMessage(this.getClass(), "SevenZipIngestModule.init.errCantInitLib",
throw new RuntimeException(e); e.getMessage());
services.postMessage(IngestMessage.createErrorMessage(ArchiveFileExtractorModuleFactory.getModuleName(), msg, details));
throw new RuntimeException(e);
}
} }
archiveDepthCountTree = new ArchiveDepthCountTree(); archiveDepthCountTree = new ArchiveDepthCountTree();
} }

64
HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbIngestModule.java
View File

@ -21,7 +21,6 @@ package org.sleuthkit.autopsy.hashdatabase;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;
import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicLong;
import java.util.logging.Level; import java.util.logging.Level;
@ -56,54 +55,41 @@ public class HashDbIngestModule extends IngestModuleAdapter implements FileInges
private final HashLookupModuleSettings settings; private final HashLookupModuleSettings settings;
private List<HashDb> knownBadHashSets = new ArrayList<>(); private List<HashDb> knownBadHashSets = new ArrayList<>();
private List<HashDb> knownHashSets = new ArrayList<>(); private List<HashDb> knownHashSets = new ArrayList<>();
private long jobID; private long jobId;
// Maps a JobId to the count of instances
static HashMap<Long, Long> moduleRefCount = new HashMap<>();
static AtomicLong totalKnownBadCount = new AtomicLong(0); static AtomicLong totalKnownBadCount = new AtomicLong(0);
static AtomicLong totalCalctime = new AtomicLong(0); static AtomicLong totalCalctime = new AtomicLong(0);
static AtomicLong totalLookuptime = new AtomicLong(0); static AtomicLong totalLookuptime = new AtomicLong(0);
private static synchronized void moduleRefCountIncrement(long jobID) {
long count = moduleRefCount.containsKey(jobID) ? moduleRefCount.get(jobID) : 0;
moduleRefCount.put(jobID, count + 1);
}
private static synchronized long moduleRefCountDecrementAndGet(long jobID) {
if (moduleRefCount.containsKey(jobID)) {
long count = moduleRefCount.get(jobID);
moduleRefCount.put(jobID, --count);
return count;
} else {
return 0;
}
}
HashDbIngestModule(HashLookupModuleSettings settings) { HashDbIngestModule(HashLookupModuleSettings settings) {
this.settings = settings; this.settings = settings;
} }
@Override @Override
public void startUp(org.sleuthkit.autopsy.ingest.IngestJobContext context) throws IngestModuleException { public void startUp(org.sleuthkit.autopsy.ingest.IngestJobContext context) throws IngestModuleException {
jobID = context.getJobId(); jobId = context.getJobId();
moduleRefCountIncrement(jobID);
getEnabledHashSets(hashDbManager.getKnownBadFileHashSets(), knownBadHashSets); getEnabledHashSets(hashDbManager.getKnownBadFileHashSets(), knownBadHashSets);
if (knownBadHashSets.isEmpty()) {
services.postMessage(IngestMessage.createWarningMessage(
HashLookupModuleFactory.getModuleName(),
NbBundle.getMessage(this.getClass(),
"HashDbIngestModule.noKnownBadHashDbSetMsg"),
NbBundle.getMessage(this.getClass(),
"HashDbIngestModule.knownBadFileSearchWillNotExecuteWarn")));
}
getEnabledHashSets(hashDbManager.getKnownFileHashSets(), knownHashSets); getEnabledHashSets(hashDbManager.getKnownFileHashSets(), knownHashSets);
if (knownHashSets.isEmpty()) {
services.postMessage(IngestMessage.createWarningMessage( if (IngestModuleAdapter.moduleRefCountIncrement(jobId) == 1) {
HashLookupModuleFactory.getModuleName(), // if first module for this job then post error msgs if needed
NbBundle.getMessage(this.getClass(),
"HashDbIngestModule.noKnownHashDbSetMsg"), if (knownBadHashSets.isEmpty()) {
NbBundle.getMessage(this.getClass(), services.postMessage(IngestMessage.createWarningMessage(
"HashDbIngestModule.knownFileSearchWillNotExecuteWarn"))); HashLookupModuleFactory.getModuleName(),
NbBundle.getMessage(this.getClass(),
"HashDbIngestModule.noKnownBadHashDbSetMsg"),
NbBundle.getMessage(this.getClass(),
"HashDbIngestModule.knownBadFileSearchWillNotExecuteWarn")));
}
if (knownHashSets.isEmpty()) {
services.postMessage(IngestMessage.createWarningMessage(
HashLookupModuleFactory.getModuleName(),
NbBundle.getMessage(this.getClass(),
"HashDbIngestModule.noKnownHashDbSetMsg"),
NbBundle.getMessage(this.getClass(),
"HashDbIngestModule.knownFileSearchWillNotExecuteWarn")));
}
} }
} }
@ -317,7 +303,7 @@ public class HashDbIngestModule extends IngestModuleAdapter implements FileInges
@Override @Override
public void shutDown(boolean ingestJobCancelled) { public void shutDown(boolean ingestJobCancelled) {
if (moduleRefCountDecrementAndGet(jobID) == 0) { if (IngestModuleAdapter.moduleRefCountDecrementAndGet(jobId) == 0) {
if ((!knownBadHashSets.isEmpty()) || (!knownHashSets.isEmpty())) { if ((!knownBadHashSets.isEmpty()) || (!knownHashSets.isEmpty())) {
StringBuilder detailsSb = new StringBuilder(); StringBuilder detailsSb = new StringBuilder();
//details //details

173
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java
View File

@ -91,7 +91,7 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme
private long dataSourceId; private long dataSourceId;
private static AtomicInteger instanceCount = new AtomicInteger(0); //just used for logging private static AtomicInteger instanceCount = new AtomicInteger(0); //just used for logging
private int instanceNum = 0; private int instanceNum = 0;
private enum IngestStatus { private enum IngestStatus {
TEXT_INGESTED, /// Text was extracted by knowing file type and text_ingested TEXT_INGESTED, /// Text was extracted by knowing file type and text_ingested
@ -101,8 +101,14 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme
SKIPPED_ERROR_TEXTEXTRACT, ///< File was skipped because of text extraction issues SKIPPED_ERROR_TEXTEXTRACT, ///< File was skipped because of text extraction issues
SKIPPED_ERROR_IO ///< File was skipped because of IO issues reading it SKIPPED_ERROR_IO ///< File was skipped because of IO issues reading it
}; };
private Map<Long, IngestStatus> ingestStatus; private static final Map<Long, IngestStatus> ingestStatus = new HashMap<>(); //guarded by itself
static void putIngestStatus(long id, IngestStatus status) {
synchronized(ingestStatus) {
ingestStatus.put(id, status);
}
}
KeywordSearchIngestModule(KeywordSearchJobSettings settings) { KeywordSearchIngestModule(KeywordSearchJobSettings settings) {
this.settings = settings; this.settings = settings;
instanceNum = instanceCount.getAndIncrement(); instanceNum = instanceCount.getAndIncrement();
@ -116,40 +122,57 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme
@Override @Override
public void startUp(IngestJobContext context) throws IngestModuleException { public void startUp(IngestJobContext context) throws IngestModuleException {
logger.log(Level.INFO, "Initializing instance {0}", instanceNum); logger.log(Level.INFO, "Initializing instance {0}", instanceNum);
initialized = false; initialized = false;
jobId = context.getJobId(); jobId = context.getJobId();
caseHandle = Case.getCurrentCase().getSleuthkitCase(); caseHandle = Case.getCurrentCase().getSleuthkitCase();
tikaFormatDetector = new Tika(); tikaFormatDetector = new Tika();
ingester = Server.getIngester(); ingester = Server.getIngester();
final Server server = KeywordSearch.getServer(); // increment the module reference count
try { // if first instance of this module for this job then check the server and existence of keywords
if (!server.isRunning()) { if (IngestModuleAdapter.moduleRefCountIncrement(jobId) == 1) {
final Server server = KeywordSearch.getServer();
try {
if (!server.isRunning()) {
String msg = NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.init.badInitMsg");
logger.log(Level.SEVERE, msg);
String details = NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.init.tryStopSolrMsg", msg);
services.postMessage(IngestMessage.createErrorMessage(KeywordSearchModuleFactory.getModuleName(), msg, details));
throw new IngestModuleException(msg);
}
} catch (KeywordSearchModuleException ex) {
logger.log(Level.WARNING, "Error checking if Solr server is running while initializing ingest", ex);
//this means Solr is not properly initialized
String msg = NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.init.badInitMsg"); String msg = NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.init.badInitMsg");
logger.log(Level.SEVERE, msg);
String details = NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.init.tryStopSolrMsg", msg); String details = NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.init.tryStopSolrMsg", msg);
services.postMessage(IngestMessage.createErrorMessage(KeywordSearchModuleFactory.getModuleName(), msg, details)); services.postMessage(IngestMessage.createErrorMessage(KeywordSearchModuleFactory.getModuleName(), msg, details));
throw new IngestModuleException(msg); throw new IngestModuleException(msg);
} }
} catch (KeywordSearchModuleException ex) { try {
logger.log(Level.WARNING, "Error checking if Solr server is running while initializing ingest", ex); // make an actual query to verify that server is responding
//this means Solr is not properly initialized // we had cases where getStatus was OK, but the connection resulted in a 404
String msg = NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.init.badInitMsg"); server.queryNumIndexedDocuments();
String details = NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.init.tryStopSolrMsg", msg); } catch (KeywordSearchModuleException | NoOpenCoreException ex) {
services.postMessage(IngestMessage.createErrorMessage(KeywordSearchModuleFactory.getModuleName(), msg, details)); throw new IngestModuleException(
throw new IngestModuleException(msg); NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.init.exception.errConnToSolr.msg",
} ex.getMessage()));
try { }
// make an actual query to verify that server is responding
// we had cases where getStatus was OK, but the connection resulted in a 404
server.queryNumIndexedDocuments();
} catch (KeywordSearchModuleException | NoOpenCoreException ex) {
throw new IngestModuleException(
NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.init.exception.errConnToSolr.msg",
ex.getMessage()));
}
// check if this job has any searchable keywords
List<KeywordList> keywordLists = KeywordSearchListsXML.getCurrent().getListsL();
boolean hasKeywordsForSearch = false;
for (KeywordList keywordList : keywordLists) {
if (settings.isKeywordListEnabled(keywordList.getName()) && !keywordList.getKeywords().isEmpty()) {
hasKeywordsForSearch = true;
break;
}
}
if (!hasKeywordsForSearch) {
services.postMessage(IngestMessage.createWarningMessage(KeywordSearchModuleFactory.getModuleName(), NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.init.noKwInLstMsg"),
NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.init.onlyIdxKwSkipMsg")));
}
}
//initialize extractors //initialize extractors
stringExtractor = new AbstractFileStringExtract(this); stringExtractor = new AbstractFileStringExtract(this);
stringExtractor.setScripts(KeywordSearchSettings.getStringExtractScripts()); stringExtractor.setScripts(KeywordSearchSettings.getStringExtractScripts());
@ -166,22 +189,7 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme
//order matters, more specific extractors first //order matters, more specific extractors first
textExtractors.add(new AbstractFileHtmlExtract(this)); textExtractors.add(new AbstractFileHtmlExtract(this));
textExtractors.add(new AbstractFileTikaTextExtract(this)); textExtractors.add(new AbstractFileTikaTextExtract(this));
ingestStatus = new HashMap<>();
List<KeywordList> keywordLists = KeywordSearchListsXML.getCurrent().getListsL();
boolean hasKeywordsForSearch = false;
for (KeywordList keywordList : keywordLists) {
if (settings.isKeywordListEnabled(keywordList.getName()) && !keywordList.getKeywords().isEmpty()) {
hasKeywordsForSearch = true;
break;
}
}
if (!hasKeywordsForSearch) {
services.postMessage(IngestMessage.createWarningMessage(KeywordSearchModuleFactory.getModuleName(), NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.init.noKwInLstMsg"),
NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.init.onlyIdxKwSkipMsg")));
}
indexer = new Indexer(); indexer = new Indexer();
initialized = true; initialized = true;
} }
@ -191,7 +199,7 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme
if (initialized == false) //error initializing indexing/Solr if (initialized == false) //error initializing indexing/Solr
{ {
logger.log(Level.WARNING, "Skipping processing, module not initialized, file: {0}", abstractFile.getName()); logger.log(Level.WARNING, "Skipping processing, module not initialized, file: {0}", abstractFile.getName());
ingestStatus.put(abstractFile.getId(), IngestStatus.SKIPPED_ERROR_INDEXING); putIngestStatus(abstractFile.getId(), IngestStatus.SKIPPED_ERROR_INDEXING);
return ProcessResult.OK; return ProcessResult.OK;
} }
try { try {
@ -247,7 +255,10 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme
// Remove from the search list and trigger final commit and final search // Remove from the search list and trigger final commit and final search
SearchRunner.getInstance().endJob(jobId); SearchRunner.getInstance().endJob(jobId);
postIndexSummary(); // We only need to post the summary msg from the last module per job
if (IngestModuleAdapter.moduleRefCountDecrementAndGet(jobId) == 0) {
postIndexSummary();
}
//log number of files / chunks in index //log number of files / chunks in index
//signal a potential change in number of text_ingested files //signal a potential change in number of text_ingested files
@ -276,7 +287,9 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme
* Common cleanup code when module stops or final searcher completes * Common cleanup code when module stops or final searcher completes
*/ */
private void cleanup() { private void cleanup() {
ingestStatus.clear(); synchronized(ingestStatus) {
ingestStatus.clear();
}
textExtractors.clear(); textExtractors.clear();
textExtractors = null; textExtractors = null;
@ -297,31 +310,34 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme
int error_text = 0; int error_text = 0;
int error_index = 0; int error_index = 0;
int error_io = 0; int error_io = 0;
for (IngestStatus s : ingestStatus.values()) {
switch (s) { synchronized(ingestStatus) {
case TEXT_INGESTED: for (IngestStatus s : ingestStatus.values()) {
++text_ingested; switch (s) {
break; case TEXT_INGESTED:
case METADATA_INGESTED: ++text_ingested;
++metadata_ingested; break;
break; case METADATA_INGESTED:
case STRINGS_INGESTED: ++metadata_ingested;
++strings_ingested; break;
break; case STRINGS_INGESTED:
case SKIPPED_ERROR_TEXTEXTRACT: ++strings_ingested;
error_text++; break;
break; case SKIPPED_ERROR_TEXTEXTRACT:
case SKIPPED_ERROR_INDEXING: error_text++;
error_index++; break;
break; case SKIPPED_ERROR_INDEXING:
case SKIPPED_ERROR_IO: error_index++;
error_io++; break;
break; case SKIPPED_ERROR_IO:
default: error_io++;
; break;
default:
;
}
} }
} }
StringBuilder msg = new StringBuilder(); StringBuilder msg = new StringBuilder();
msg.append("<table border=0><tr><td>").append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.postIndexSummary.knowFileHeaderLbl")).append("</td><td>").append(text_ingested).append("</td></tr>"); msg.append("<table border=0><tr><td>").append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.postIndexSummary.knowFileHeaderLbl")).append("</td><td>").append(text_ingested).append("</td></tr>");
msg.append("<tr><td>").append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.postIndexSummary.fileGenStringsHead")).append("</td><td>").append(strings_ingested).append("</td></tr>"); msg.append("<tr><td>").append(NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.postIndexSummary.fileGenStringsHead")).append("</td><td>").append(strings_ingested).append("</td></tr>");
@ -393,16 +409,16 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme
private boolean extractStringsAndIndex(AbstractFile aFile) { private boolean extractStringsAndIndex(AbstractFile aFile) {
try { try {
if (stringExtractor.index(aFile)) { if (stringExtractor.index(aFile)) {
ingestStatus.put(aFile.getId(), IngestStatus.STRINGS_INGESTED); putIngestStatus(aFile.getId(), IngestStatus.STRINGS_INGESTED);
return true; return true;
} else { } else {
logger.log(Level.WARNING, "Failed to extract strings and ingest, file ''{0}'' (id: {1}).", new Object[]{aFile.getName(), aFile.getId()}); logger.log(Level.WARNING, "Failed to extract strings and ingest, file ''{0}'' (id: {1}).", new Object[]{aFile.getName(), aFile.getId()});
ingestStatus.put(aFile.getId(), IngestStatus.SKIPPED_ERROR_TEXTEXTRACT); putIngestStatus(aFile.getId(), IngestStatus.SKIPPED_ERROR_TEXTEXTRACT);
return false; return false;
} }
} catch (IngesterException ex) { } catch (IngesterException ex) {
logger.log(Level.WARNING, "Failed to extract strings and ingest, file '" + aFile.getName() + "' (id: " + aFile.getId() + ").", ex); logger.log(Level.WARNING, "Failed to extract strings and ingest, file '" + aFile.getName() + "' (id: " + aFile.getId() + ").", ex);
ingestStatus.put(aFile.getId(), IngestStatus.SKIPPED_ERROR_INDEXING); putIngestStatus(aFile.getId(), IngestStatus.SKIPPED_ERROR_INDEXING);
return false; return false;
} }
} }
@ -448,9 +464,9 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme
if ((indexContent == false || aFile.isDir() || size == 0)) { if ((indexContent == false || aFile.isDir() || size == 0)) {
try { try {
ingester.ingest(aFile, false); //meta-data only ingester.ingest(aFile, false); //meta-data only
ingestStatus.put(aFile.getId(), IngestStatus.METADATA_INGESTED); putIngestStatus(aFile.getId(), IngestStatus.METADATA_INGESTED);
} catch (IngesterException ex) { } catch (IngesterException ex) {
ingestStatus.put(aFile.getId(), IngestStatus.SKIPPED_ERROR_INDEXING); putIngestStatus(aFile.getId(), IngestStatus.SKIPPED_ERROR_INDEXING);
logger.log(Level.WARNING, "Unable to index meta-data for file: " + aFile.getId(), ex); logger.log(Level.WARNING, "Unable to index meta-data for file: " + aFile.getId(), ex);
} }
return; return;
@ -484,9 +500,9 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme
if (AbstractFileExtract.ARCHIVE_MIME_TYPES.contains(detectedFormat)) { if (AbstractFileExtract.ARCHIVE_MIME_TYPES.contains(detectedFormat)) {
try { try {
ingester.ingest(aFile, false); //meta-data only ingester.ingest(aFile, false); //meta-data only
ingestStatus.put(aFile.getId(), IngestStatus.METADATA_INGESTED); putIngestStatus(aFile.getId(), IngestStatus.METADATA_INGESTED);
} catch (IngesterException ex) { } catch (IngesterException ex) {
ingestStatus.put(aFile.getId(), IngestStatus.SKIPPED_ERROR_INDEXING); putIngestStatus(aFile.getId(), IngestStatus.SKIPPED_ERROR_INDEXING);
logger.log(Level.WARNING, "Unable to index meta-data for file: " + aFile.getId(), ex); logger.log(Level.WARNING, "Unable to index meta-data for file: " + aFile.getId(), ex);
} }
return; return;
@ -499,20 +515,20 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme
//logger.log(Level.INFO, "indexing: " + aFile.getName()); //logger.log(Level.INFO, "indexing: " + aFile.getName());
if (!extractTextAndIndex(aFile, detectedFormat)) { if (!extractTextAndIndex(aFile, detectedFormat)) {
logger.log(Level.WARNING, "Failed to extract text and ingest, file ''{0}'' (id: {1}).", new Object[]{aFile.getName(), aFile.getId()}); logger.log(Level.WARNING, "Failed to extract text and ingest, file ''{0}'' (id: {1}).", new Object[]{aFile.getName(), aFile.getId()});
ingestStatus.put(aFile.getId(), IngestStatus.SKIPPED_ERROR_TEXTEXTRACT); putIngestStatus(aFile.getId(), IngestStatus.SKIPPED_ERROR_TEXTEXTRACT);
} else { } else {
ingestStatus.put(aFile.getId(), IngestStatus.TEXT_INGESTED); putIngestStatus(aFile.getId(), IngestStatus.TEXT_INGESTED);
wasTextAdded = true; wasTextAdded = true;
} }
} catch (IngesterException e) { } catch (IngesterException e) {
logger.log(Level.INFO, "Could not extract text with Tika, " + aFile.getId() + ", " logger.log(Level.INFO, "Could not extract text with Tika, " + aFile.getId() + ", "
+ aFile.getName(), e); + aFile.getName(), e);
ingestStatus.put(aFile.getId(), IngestStatus.SKIPPED_ERROR_INDEXING); putIngestStatus(aFile.getId(), IngestStatus.SKIPPED_ERROR_INDEXING);
} catch (Exception e) { } catch (Exception e) {
logger.log(Level.WARNING, "Error extracting text with Tika, " + aFile.getId() + ", " logger.log(Level.WARNING, "Error extracting text with Tika, " + aFile.getId() + ", "
+ aFile.getName(), e); + aFile.getName(), e);
ingestStatus.put(aFile.getId(), IngestStatus.SKIPPED_ERROR_TEXTEXTRACT); putIngestStatus(aFile.getId(), IngestStatus.SKIPPED_ERROR_TEXTEXTRACT);
} }
} }
@ -522,5 +538,4 @@ public final class KeywordSearchIngestModule extends IngestModuleAdapter impleme
} }
} }
} }
} }

2
ScalpelCarver/src/org/sleuthkit/autopsy/scalpel/ScalpelCarverIngestModule.java
View File

@ -53,7 +53,7 @@ class ScalpelCarverIngestModule extends IngestModuleAdapter implements FileInges
private static final Logger logger = Logger.getLogger(ScalpelCarverIngestModule.class.getName()); private static final Logger logger = Logger.getLogger(ScalpelCarverIngestModule.class.getName());
private final String MODULE_OUTPUT_DIR_NAME = "ScalpelCarver"; private final String MODULE_OUTPUT_DIR_NAME = "ScalpelCarver";
private String moduleOutputDirPath; private String moduleOutputDirPath;
private String configFileName = "scalpel.conf"; private final String configFileName = "scalpel.conf";
private String configFilePath; private String configFilePath;
private boolean initialized = false; private boolean initialized = false;
private ScalpelCarver carver; private ScalpelCarver carver;

1
thunderbirdparser/src/org/sleuthkit/autopsy/thunderbirdparser/ThunderbirdMboxFileIngestModule.java
View File

@ -54,7 +54,6 @@ public final class ThunderbirdMboxFileIngestModule extends IngestModuleAdapter i
private static final Logger logger = Logger.getLogger(ThunderbirdMboxFileIngestModule.class.getName()); private static final Logger logger = Logger.getLogger(ThunderbirdMboxFileIngestModule.class.getName());
private IngestServices services = IngestServices.getInstance(); private IngestServices services = IngestServices.getInstance();
private int messageId = 0; // RJCTODO: Not thread safe
private FileManager fileManager; private FileManager fileManager;
private IngestJobContext context; private IngestJobContext context;