This commit is contained in:
adam-m 2012-09-18 15:14:29 -04:00
commit ecb2ccdc64
13 changed files with 331 additions and 89 deletions

View File

@ -57,7 +57,11 @@ public final class ExifParserFileIngestModule implements IngestModuleAbstractFil
private IngestServices services;
final String MODULE_NAME = "Exif Parser";
final public static String MODULE_NAME = "Exif Parser";
final public static String MODULE_VERSION = "1.0";
private String args;
private static final Logger logger = Logger.getLogger(ExifParserFileIngestModule.class.getName());
private static ExifParserFileIngestModule defaultInstance = null;
private static int messageId = 0;
@ -194,6 +198,23 @@ public final class ExifParserFileIngestModule implements IngestModuleAbstractFil
//module specific cleanup due to completion here
}
@Override
public String getVersion() {
return MODULE_VERSION;
}
@Override
public String getArguments() {
return args;
}
@Override
public void setArguments(String args) {
this.args = args;
}
@Override
public String getName() {
return "Exif Image Parser";

View File

@ -51,6 +51,8 @@ public class HashDbIngestModule implements IngestModuleAbstractFile {
private static HashDbIngestModule instance = null;
public final static String MODULE_NAME = "Hash Lookup";
public final static String MODULE_DESCRIPTION = "Identifies known and notables files using supplied hash databases, such as a standard NSRL database.";
final public static String MODULE_VERSION = "1.0";
private String args;
private static final Logger logger = Logger.getLogger(HashDbIngestModule.class.getName());
private Processor processor = new Processor();
private IngestServices services;
@ -68,7 +70,6 @@ public class HashDbIngestModule implements IngestModuleAbstractFile {
private Map<Integer, HashDb> knownBadSets = new HashMap<Integer, HashDb>();
private HashDbManagementPanel panel;
private HashDbIngestModule() {
count = 0;
}
@ -80,7 +81,6 @@ public class HashDbIngestModule implements IngestModuleAbstractFile {
return instance;
}
@Override
public void init(IngestModuleInit initContext) {
services = IngestServices.getDefault();
@ -99,13 +99,13 @@ public class HashDbIngestModule implements IngestModuleAbstractFile {
calcHashesIsSet = hdbxml.getCalculate();
HashDb nsrl = hdbxml.getNSRLSet();
if(nsrl != null && IndexStatus.isIngestible(nsrl.status())) {
if (nsrl != null && IndexStatus.isIngestible(nsrl.status())) {
nsrlIsSet = true;
this.nsrlSet = nsrl;
nsrlPointer = skCase.setNSRLDatabase(nsrl.getDatabasePaths().get(0));
}
for(HashDb db : hdbxml.getKnownBadSets()) {
for (HashDb db : hdbxml.getKnownBadSets()) {
IndexStatus status = db.status();
if (db.getUseForIngest() && IndexStatus.isIngestible(status)) {
knownBadIsSet = true;
@ -126,7 +126,6 @@ public class HashDbIngestModule implements IngestModuleAbstractFile {
}
}
@Override
public void complete() {
StringBuilder detailsSb = new StringBuilder();
@ -140,10 +139,10 @@ public class HashDbIngestModule implements IngestModuleAbstractFile {
detailsSb.append("<tr>");
detailsSb.append("<th>Notable databases used:</th>");
detailsSb.append("<td>Calc Time: ").append(calctime).append(" Lookup Time: " ).append(lookuptime).append("</td>");
detailsSb.append("<td>Calc Time: ").append(calctime).append(" Lookup Time: ").append(lookuptime).append("</td>");
detailsSb.append("</tr>");
for(HashDb db : knownBadSets.values()) {
for (HashDb db : knownBadSets.values()) {
detailsSb.append("<tr><th>");
detailsSb.append(db.getName());
detailsSb.append("</th><td>");
@ -160,7 +159,8 @@ public class HashDbIngestModule implements IngestModuleAbstractFile {
}
/**
* notification from manager to stop processing due to some interruption (user, error, exception)
* notification from manager to stop processing due to some interruption
* (user, error, exception)
*/
@Override
public void stop() {
@ -171,9 +171,10 @@ public class HashDbIngestModule implements IngestModuleAbstractFile {
}
/**
* get specific name of the module
* should be unique across modules, a user-friendly name of the module shown in GUI
* @return The name of this Ingest Module
* get specific name of the module should be unique across modules, a
* user-friendly name of the module shown in GUI
*
* @return The name of this Ingest Module
*/
@Override
public String getName() {
@ -185,11 +186,27 @@ public class HashDbIngestModule implements IngestModuleAbstractFile {
return MODULE_DESCRIPTION;
}
@Override
public String getVersion() {
return MODULE_VERSION;
}
@Override
public String getArguments() {
return args;
}
@Override
public void setArguments(String args) {
this.args = args;
}
/**
* Process the given AbstractFile object
*
* @param abstractFile the object to be processed
* @return ProcessResult OK if file is unknown and should be processed further, otherwise STOP_COND if file is known
* @return ProcessResult OK if file is unknown and should be processed
* further, otherwise STOP_COND if file is known
*/
@Override
public ProcessResult process(AbstractFile abstractFile) {
@ -206,7 +223,6 @@ public class HashDbIngestModule implements IngestModuleAbstractFile {
return false;
}
@Override
public boolean hasSimpleConfiguration() {
return true;
@ -244,7 +260,7 @@ public class HashDbIngestModule implements IngestModuleAbstractFile {
@Override
public void saveSimpleConfiguration() {
HashDbXML.getCurrent().save();
HashDbXML.getCurrent().save();
}
private void processBadFile(AbstractFile abstractFile, String md5Hash, String hashSetName, boolean showInboxMessage) {
@ -318,14 +334,14 @@ public class HashDbIngestModule implements IngestModuleAbstractFile {
if (md5Hash == null || md5Hash.isEmpty()) {
long calcstart = System.currentTimeMillis();
md5Hash = Hash.calculateMd5(fsContent);
calctime += (System.currentTimeMillis()-calcstart);
calctime += (System.currentTimeMillis() - calcstart);
}
TskData.FileKnown status = TskData.FileKnown.UKNOWN;
boolean foundBad = false;
for (Map.Entry<Integer, HashDb> entry : knownBadSets.entrySet()) {
long lookupstart = System.currentTimeMillis();
status = skCase.knownBadLookupMd5(md5Hash, entry.getKey());
lookuptime += (System.currentTimeMillis()-lookupstart);
lookuptime += (System.currentTimeMillis() - lookupstart);
if (status.equals(TskData.FileKnown.BAD)) {
foundBad = true;
count += 1;
@ -337,7 +353,7 @@ public class HashDbIngestModule implements IngestModuleAbstractFile {
if (!foundBad && nsrlIsSet) {
long lookupstart = System.currentTimeMillis();
status = skCase.nsrlLookupMd5(md5Hash);
lookuptime += (System.currentTimeMillis()-lookupstart);
lookuptime += (System.currentTimeMillis() - lookupstart);
if (status.equals(TskData.FileKnown.KNOWN)) {
skCase.setKnown(fsContent, status);
}
@ -353,18 +369,17 @@ public class HashDbIngestModule implements IngestModuleAbstractFile {
"Error encountered while calculating the hash value for " + name + "."));
ret = ProcessResult.ERROR;
}
} else if(processFile && calcHashesIsSet) {
} else if (processFile && calcHashesIsSet) {
String name = fsContent.getName();
try {
String md5Hash = fsContent.getMd5Hash();
if (md5Hash == null || md5Hash.isEmpty()) {
long calcstart = System.currentTimeMillis();
Hash.calculateMd5(fsContent);
calctime += (System.currentTimeMillis()-calcstart);
calctime += (System.currentTimeMillis() - calcstart);
}
ret = ProcessResult.OK;
}
catch (IOException ex) {
} catch (IOException ex) {
logger.log(Level.WARNING, "Error reading file " + name, ex);
services.postMessage(IngestMessage.createErrorMessage(++messageId, HashDbIngestModule.this, "Read Error: " + name,
"Error encountered while calculating the hash value for " + name + " without databases."));
@ -372,7 +387,5 @@ public class HashDbIngestModule implements IngestModuleAbstractFile {
}
return ret;
}
}
}

View File

@ -73,6 +73,12 @@ public interface IngestModuleAbstract {
*/
public String getName();
/**
* Gets the module version
* @return module version string
*/
public String getVersion();
/**
* Gets user-friendly description of the module
* @return module description
@ -85,6 +91,19 @@ public interface IngestModuleAbstract {
*/
public ModuleType getType();
/**
* Gets the arguments as set in XML
* @return arguments string
*/
public String getArguments();
/**
* Sets the arguments from XML
* @param args arguments string in XML
*/
public void setArguments(String args);
/**
* A module can manage and use additional threads to perform some work in the background.
* This method provides insight to the manager if the module has truly completed its work or not.

View File

@ -38,11 +38,14 @@ public class ExampleAbstractFileIngestModule implements IngestModuleAbstractFile
private static ExampleAbstractFileIngestModule instance = null;
private IngestServices services;
private static int messageId = 0;
public static final String MODULE_NAME = "Example AbstractFile Module";
public static final String MODULE_DESC = "Example AbstractFile Module description";
public static final String MODULE_VERSION = "1.0";
private String args;
//file ingest modules require a private constructor
//to ensure singleton instances
private ExampleAbstractFileIngestModule() {
}
public static synchronized ExampleAbstractFileIngestModule getDefault() {
@ -75,15 +78,28 @@ public class ExampleAbstractFileIngestModule implements IngestModuleAbstractFile
@Override
public String getName() {
return "Example AbstractFile Module";
return MODULE_NAME;
}
@Override
public String getDescription() {
return "Example AbstractFile Module description";
return MODULE_DESC;
}
@Override
public String getVersion() {
return MODULE_VERSION;
}
@Override
public String getArguments() {
return args;
}
@Override
public void setArguments(String args) {
this.args = args;
}
@Override
public void init(IngestModuleInit initContext) {
@ -131,7 +147,6 @@ public class ExampleAbstractFileIngestModule implements IngestModuleAbstractFile
return false;
}
@Override
public void saveAdvancedConfiguration() {
}

View File

@ -38,6 +38,10 @@ public final class ExampleImageIngestModule implements IngestModuleImage {
private static ExampleImageIngestModule defaultInstance = null;
private IngestServices services;
private static int messageId = 0;
public static final String MODULE_NAME = "Example Image Module";
public static final String MODULE_DESC = "Example Image Module description";
public static final String MODULE_VERSION = "1.0";
private String args;
//public constructor is required
//as multiple instances are created for processing multiple images simultenously
@ -99,12 +103,27 @@ public final class ExampleImageIngestModule implements IngestModuleImage {
@Override
public String getName() {
return "Example Image Service";
return MODULE_NAME;
}
@Override
public String getDescription() {
return "Example Image Service description";
return MODULE_DESC;
}
@Override
public String getVersion() {
return MODULE_VERSION;
}
@Override
public String getArguments() {
return args;
}
@Override
public void setArguments(String args) {
this.args = args;
}
@Override
@ -129,7 +148,7 @@ public final class ExampleImageIngestModule implements IngestModuleImage {
return ModuleType.Image;
}
@Override
@Override
public boolean hasSimpleConfiguration() {
return false;
}
@ -154,7 +173,6 @@ public final class ExampleImageIngestModule implements IngestModuleImage {
return false;
}
@Override
public void saveAdvancedConfiguration() {
}

View File

@ -91,7 +91,8 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
private static final Logger logger = Logger.getLogger(KeywordSearchIngestModule.class.getName());
public static final String MODULE_NAME = "Keyword Search";
public static final String MODULE_DESCRIPTION = "Performs file indexing and periodic search using keywords and regular expressions in lists.";
public static final String MODULE_DESCRIPTION = "Performs file indexing and periodic search using keywords and regular expressions in lists."; final public static String MODULE_VERSION = "1.0";
private String args;
private static KeywordSearchIngestModule instance = null;
private IngestServices services;
private Ingester ingester = null;
@ -326,6 +327,22 @@ public final class KeywordSearchIngestModule implements IngestModuleAbstractFile
return MODULE_DESCRIPTION;
}
@Override
public String getVersion() {
return MODULE_VERSION;
}
@Override
public String getArguments() {
return args;
}
@Override
public void setArguments(String args) {
this.args = args;
}
/**
* Initializes the module for new ingest run Sets up threads, timers,
* retrieves settings, keyword lists to run on

View File

@ -57,12 +57,32 @@ public class Chrome extends Extract implements IngestModuleImage {
private final Logger logger = Logger.getLogger(this.getClass().getName());
public int ChromeCount = 0;
final public static String MODULE_VERSION = "1.0";
private String args;
private IngestServices services;
public Chrome() {
moduleName = "Chrome";
}
@Override
public String getVersion() {
return MODULE_VERSION;
}
@Override
public String getArguments() {
return args;
}
@Override
public void setArguments(String args) {
this.args = args;
}
@Override
public void process(Image image, IngestImageWorkerController controller) {
this.getHistory(image, controller);

View File

@ -90,10 +90,31 @@ public class ExtractIE extends Extract implements IngestModuleImage {
public LinkedHashMap<String, Object> IE_OBJ;
boolean pascoFound = false;
final public static String MODULE_VERSION = "1.0";
private String args;
public ExtractIE() {
moduleName = "Internet Explorer";
}
@Override
public String getVersion() {
return MODULE_VERSION;
}
@Override
public String getArguments() {
return args;
}
@Override
public void setArguments(String args) {
this.args = args;
}
@Override
public void process(Image image, IngestImageWorkerController controller) {
this.getHistory(image, controller);

View File

@ -52,6 +52,8 @@ public class ExtractRegistry extends Extract implements IngestModuleImage {
boolean rrFound = false;
private int sysid;
private IngestServices services;
final public static String MODULE_VERSION = "1.0";
private String args;
ExtractRegistry() {
final File rrRoot = InstalledFileLocator.getDefault().locate("rr", ExtractRegistry.class.getPackage().getName(), false);
@ -78,6 +80,21 @@ public class ExtractRegistry extends Extract implements IngestModuleImage {
RR_PATH = rrHome + File.separator + "rip.exe";
}
@Override
public String getVersion() {
return MODULE_VERSION;
}
@Override
public String getArguments() {
return args;
}
@Override
public void setArguments(String args) {
this.args = args;
}
private void getregistryfiles(Image image, IngestImageWorkerController controller) {
try {
Case currentCase = Case.getCurrentCase(); // get the most updated case

View File

@ -54,12 +54,32 @@ public class Firefox extends Extract implements IngestModuleImage {
private static final String ffdownloadquery = "select target, source,(startTime/1000000) as startTime, maxBytes from moz_downloads";
public int FireFoxCount = 0;
final public static String MODULE_VERSION = "1.0";
private String args;
private IngestServices services;
public Firefox() {
moduleName = "FireFox";
}
@Override
public String getVersion() {
return MODULE_VERSION;
}
@Override
public String getArguments() {
return args;
}
@Override
public void setArguments(String args) {
this.args = args;
}
@Override
public void process(Image image, IngestImageWorkerController controller) {
this.getHistory(image, controller);

View File

@ -50,6 +50,10 @@ public final class RAImageIngestModule implements IngestModuleImage {
private ExtractIE eere = null;
private SearchEngineURLQueryAnalyzer usq = null;
final public static String MODULE_VERSION = "1.0";
private String args;
//public constructor is required
//as multiple instances are created for processing multiple images simultenously
public RAImageIngestModule() {
@ -163,6 +167,22 @@ public final class RAImageIngestModule implements IngestModuleImage {
return ModuleType.Image;
}
@Override
public String getVersion() {
return MODULE_VERSION;
}
@Override
public String getArguments() {
return args;
}
@Override
public void setArguments(String args) {
this.args = args;
}
@Override
public boolean hasSimpleConfiguration() {
return false;

View File

@ -65,8 +65,13 @@ import org.w3c.dom.NodeList;
public class SearchEngineURLQueryAnalyzer extends Extract implements IngestModuleImage {
private IngestServices services;
static final String MODULE_NAME = "Search Engine URL Query Analyzer";
public static final String XMLFile = "SEUQAMappings.xml";
public static final String MODULE_NAME = "Search Engine URL Query Analyzer";
public final static String MODULE_VERSION = "1.0";
private String args;
public static final String XMLFile = "SEQUAMappings.xml";
private static String[] searchEngineNames;
private static SearchEngine[] engines;
@ -380,7 +385,7 @@ public class SearchEngineURLQueryAnalyzer extends Extract implements IngestModul
@Override
public String getName() {
return this.moduleName;
return MODULE_NAME;
}
@Override
@ -392,6 +397,21 @@ public class SearchEngineURLQueryAnalyzer extends Extract implements IngestModul
return "Extracts search queries on the following search engines: \n" + total;
}
@Override
public String getVersion() {
return MODULE_VERSION;
}
@Override
public String getArguments() {
return args;
}
@Override
public void setArguments(String args) {
this.args = args;
}
@Override
public ModuleType getType() {
return ModuleType.Image;

View File

@ -63,8 +63,13 @@ public class ThunderbirdMboxFileIngestModule implements IngestModuleAbstractFile
private static ThunderbirdMboxFileIngestModule instance = null;
private IngestServices services;
private static int messageId = 0;
private static final String classname = "Thunderbird Parser";
private static final String MODULE_NAME = "Thunderbird Parser";
private final String hashDBModuleName = "Hash Lookup";
final public static String MODULE_VERSION = "1.0";
private String args;
private final GetIsFileKnownVisitor getIsFileKnown = new GetIsFileKnownVisitor();
public static synchronized ThunderbirdMboxFileIngestModule getDefault() {
@ -201,18 +206,18 @@ public class ThunderbirdMboxFileIngestModule implements IngestModuleAbstractFile
bcc = ((propertyMap.get(Metadata.MESSAGE_BCC) != null) ? propertyMap.get(Metadata.MESSAGE_BCC) : "");
Collection<BlackboardAttribute> bbattributes = new ArrayList<BlackboardAttribute>();
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_EMAIL_TO.getTypeID(), classname, to));
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_EMAIL_CC.getTypeID(), classname, cc));
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_EMAIL_BCC.getTypeID(), classname, bcc));
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_EMAIL_FROM.getTypeID(), classname, from));
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_EMAIL_CONTENT_PLAIN.getTypeID(), classname, content.replaceAll("\\<[^>]*>", "")));
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_EMAIL_CONTENT_HTML.getTypeID(), classname, content));
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_MSG_ID.getTypeID(), classname, StringEscapeUtils.escapeHtml(emailId)));
//bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_MSG_REPLY_ID.getTypeID(), classname, "",));
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_RCVD.getTypeID(), classname, date));
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_SENT.getTypeID(), classname, date));
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_SUBJECT.getTypeID(), classname, subject));
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PATH.getTypeID(), classname, folderPath));
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_EMAIL_TO.getTypeID(), MODULE_NAME, to));
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_EMAIL_CC.getTypeID(), MODULE_NAME, cc));
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_EMAIL_BCC.getTypeID(), MODULE_NAME, bcc));
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_EMAIL_FROM.getTypeID(), MODULE_NAME, from));
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_EMAIL_CONTENT_PLAIN.getTypeID(), MODULE_NAME, content.replaceAll("\\<[^>]*>", "")));
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_EMAIL_CONTENT_HTML.getTypeID(), MODULE_NAME, content));
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_MSG_ID.getTypeID(), MODULE_NAME, StringEscapeUtils.escapeHtml(emailId)));
//bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_MSG_REPLY_ID.getTypeID(), MODULE_NAME, "",));
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_RCVD.getTypeID(), MODULE_NAME, date));
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_SENT.getTypeID(), MODULE_NAME, date));
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_SUBJECT.getTypeID(), MODULE_NAME, subject));
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PATH.getTypeID(), MODULE_NAME, folderPath));
BlackboardArtifact bbart;
try {
bbart = abstractFile.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_EMAIL_MSG);
@ -220,7 +225,7 @@ public class ThunderbirdMboxFileIngestModule implements IngestModuleAbstractFile
} catch (TskCoreException ex) {
Logger.getLogger(ThunderbirdMboxFileIngestModule.class.getName()).log(Level.WARNING, null, ex);
}
services.fireModuleDataEvent(new ModuleDataEvent(classname, BlackboardArtifact.ARTIFACT_TYPE.TSK_EMAIL_MSG));
services.fireModuleDataEvent(new ModuleDataEvent(MODULE_NAME, BlackboardArtifact.ARTIFACT_TYPE.TSK_EMAIL_MSG));
}
} catch (FileNotFoundException ex) {
Logger.getLogger(ThunderbirdMboxFileIngestModule.class.getName()).log(Level.WARNING, null, ex);
@ -246,14 +251,30 @@ public class ThunderbirdMboxFileIngestModule implements IngestModuleAbstractFile
@Override
public String getName() {
return "Thunderbird Parser";
return MODULE_NAME;
}
@Override
public String getDescription() {
return "This class parses through a file to determine if it is an mbox file and if so, populates an email artifact for it in the blackboard.";
return "This module detects and parses mbox Thunderbird files and populates email artifacts in the blackboard.";
}
@Override
public String getVersion() {
return MODULE_VERSION;
}
@Override
public String getArguments() {
return args;
}
@Override
public void setArguments(String args) {
this.args = args;
}
@Override
public void init(IngestModuleInit initContext) {
logger.log(Level.INFO, "init()");