mstoeck3/autopsy-flatpak
1
0
Fork 0
mirror of https://github.com/overcuriousity/autopsy-flatpak.git synced 2025-07-17 10:17:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

Start adding ingest pipeline infrastructure

Browse Source
This commit is contained in:
Richard Cordovano 2014-02-21 17:46:11 -05:00
parent 2155d2567c
commit 751982bfa1
3 changed files with 295 additions and 246 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
Core/src/org/sleuthkit/autopsy/ingest/IngestManager.java
View File

@ -1303,4 +1303,19 @@ public class IngestManager {
scheduler.getDataSourceScheduler().empty(); scheduler.getDataSourceScheduler().empty();
} }
} }
private class FileIngester implements Runnable {
@Override
public void run() {
final IngestScheduler.FileScheduler fileScheduler = scheduler.getFileScheduler();
while (fileScheduler.hasNext()) {
final FileTask fileTask = fileScheduler.next();
final DataSourceTask<IngestModuleAbstractFile> dataSourceTask = fileTask.getDataSourceTask();
final AbstractFile fileToProcess = fileTask.getFile();
fileToProcess.close();
}
logger.log(Level.INFO, "IngestManager: Finished processing files");
}
}
} }

483
Core/src/org/sleuthkit/autopsy/ingest/IngestModuleLoader.java
View File

@ -471,48 +471,49 @@ import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil;
private void autodiscover() throws IngestModuleLoaderException { private void autodiscover() throws IngestModuleLoaderException {
Collection<? extends IngestModuleFactory> factories = Lookup.getDefault().lookupAll(IngestModuleFactory.class); Collection<? extends IngestModuleFactory> factories = Lookup.getDefault().lookupAll(IngestModuleFactory.class);
moduleFactories.addAll(factories); for (IngestModuleFactory factory : factories) {
logger.log(Level.INFO, "Loaded ingest module factory: name = " + factory.getModuleDisplayName() + ", version = " + factory.getModuleVersionNumber());
// moduleFactories moduleFactories.add(factory);
// Use Lookup to find the other NBM modules. We'll later search them for ingest modules
Collection<? extends ModuleInfo> moduleInfos = Lookup.getDefault().lookupAll(ModuleInfo.class);
logger.log(Level.INFO, "Autodiscovery, found #platform modules: " + moduleInfos.size());
Set<URL> urls = getJarPaths(moduleInfos);
ArrayList<Reflections> reflectionsSet = new ArrayList<>();
for (final ModuleInfo moduleInfo : moduleInfos) {
if (moduleInfo.isEnabled()) {
/* NOTE: We have an assumption here that the modules in an NBM will
* have the same package name as the NBM name. This means that
* an NBM can have only one package with modules in it. */
String basePackageName = moduleInfo.getCodeNameBase();
// skip the standard ones
if (basePackageName.startsWith("org.netbeans")
|| basePackageName.startsWith("org.openide")) {
continue;
}
logger.log(Level.INFO, "Found module: " + moduleInfo.getDisplayName() + " " + basePackageName
+ " Build version: " + moduleInfo.getBuildVersion()
+ " Spec version: " + moduleInfo.getSpecificationVersion()
+ " Impl version: " + moduleInfo.getImplementationVersion());
ConfigurationBuilder cb = new ConfigurationBuilder();
cb.filterInputsBy(new FilterBuilder().include(FilterBuilder.prefix(basePackageName)));
cb.setUrls(urls);
cb.setScanners(new SubTypesScanner(), new ResourcesScanner());
reflectionsSet.add(new Reflections(cb));
}
else {
// log if we have our own modules disabled
if (moduleInfo.getCodeNameBase().startsWith("org.sleuthkit")) {
logger.log(Level.WARNING, "Sleuth Kit Module not enabled: " + moduleInfo.getDisplayName());
}
}
} }
// // Use Lookup to find the other NBM modules. We'll later search them for ingest modules
// Collection<? extends ModuleInfo> moduleInfos = Lookup.getDefault().lookupAll(ModuleInfo.class);
// logger.log(Level.INFO, "Autodiscovery, found #platform modules: " + moduleInfos.size());
//
// Set<URL> urls = getJarPaths(moduleInfos);
// ArrayList<Reflections> reflectionsSet = new ArrayList<>();
//
// for (final ModuleInfo moduleInfo : moduleInfos) {
// if (moduleInfo.isEnabled()) {
// /* NOTE: We have an assumption here that the modules in an NBM will
// * have the same package name as the NBM name. This means that
// * an NBM can have only one package with modules in it. */
// String basePackageName = moduleInfo.getCodeNameBase();
//
// // skip the standard ones
// if (basePackageName.startsWith("org.netbeans")
// || basePackageName.startsWith("org.openide")) {
// continue;
// }
//
// logger.log(Level.INFO, "Found module: " + moduleInfo.getDisplayName() + " " + basePackageName
// + " Build version: " + moduleInfo.getBuildVersion()
// + " Spec version: " + moduleInfo.getSpecificationVersion()
// + " Impl version: " + moduleInfo.getImplementationVersion());
//
// ConfigurationBuilder cb = new ConfigurationBuilder();
// cb.filterInputsBy(new FilterBuilder().include(FilterBuilder.prefix(basePackageName)));
// cb.setUrls(urls);
// cb.setScanners(new SubTypesScanner(), new ResourcesScanner());
// reflectionsSet.add(new Reflections(cb));
// }
// else {
// // log if we have our own modules disabled
// if (moduleInfo.getCodeNameBase().startsWith("org.sleuthkit")) {
// logger.log(Level.WARNING, "Sleuth Kit Module not enabled: " + moduleInfo.getDisplayName());
// }
// }
// }
/* This area is used to load the example modules. They are not found via lookup since they /* This area is used to load the example modules. They are not found via lookup since they
* are in this NBM module. * are in this NBM module.
@ -526,111 +527,104 @@ import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil;
reflectionsSet.add(new Reflections(cb)); reflectionsSet.add(new Reflections(cb));
*/ */
for (Reflections reflections : reflectionsSet) { // for (Reflections reflections : reflectionsSet) {
//
Set<?> fileModules = reflections.getSubTypesOf(IngestModuleAbstractFile.class); // Set<?> fileModules = reflections.getSubTypesOf(IngestModuleAbstractFile.class);
Iterator<?> it = fileModules.iterator(); // Iterator<?> it = fileModules.iterator();
while (it.hasNext()) { // while (it.hasNext()) {
logger.log(Level.INFO, "Found file ingest module in: " + reflections.getClass().getSimpleName() + ": " + it.next().toString()); // logger.log(Level.INFO, "Found file ingest module in: " + reflections.getClass().getSimpleName() + ": " + it.next().toString());
} // }
//
Set<?> dataSourceModules = reflections.getSubTypesOf(IngestModuleDataSource.class); // Set<?> dataSourceModules = reflections.getSubTypesOf(IngestModuleDataSource.class);
it = dataSourceModules.iterator(); // it = dataSourceModules.iterator();
while (it.hasNext()) { // while (it.hasNext()) {
logger.log(Level.INFO, "Found DataSource ingest module in: " + reflections.getClass().getSimpleName() + ": " + it.next().toString()); // logger.log(Level.INFO, "Found DataSource ingest module in: " + reflections.getClass().getSimpleName() + ": " + it.next().toString());
} // }
//
if ((fileModules.isEmpty()) && (dataSourceModules.isEmpty())) { // if ((fileModules.isEmpty()) && (dataSourceModules.isEmpty())) {
logger.log(Level.INFO, "Module has no ingest modules: " + reflections.getClass().getSimpleName()); // logger.log(Level.INFO, "Module has no ingest modules: " + reflections.getClass().getSimpleName());
continue; // continue;
} // }
//
//find out which modules to add // //find out which modules to add
//TODO check which modules to remove (which modules were uninstalled) // //TODO check which modules to remove (which modules were uninstalled)
boolean modulesChanged = false; // boolean modulesChanged = false;
//
it = fileModules.iterator(); // it = fileModules.iterator();
while (it.hasNext()) { // while (it.hasNext()) {
boolean exists = false; // boolean exists = false;
Class<IngestModuleAbstractFile> foundClass = (Class<IngestModuleAbstractFile>) it.next(); // Class<IngestModuleAbstractFile> foundClass = (Class<IngestModuleAbstractFile>) it.next();
//
for (IngestModuleLoader.XmlPipelineRaw rawP : pipelinesXML) { // for (IngestModuleLoader.XmlPipelineRaw rawP : pipelinesXML) {
if (!rawP.type.equals(IngestModuleLoader.XmlPipelineRaw.PIPELINE_TYPE.FILE_ANALYSIS.toString())) { // if (!rawP.type.equals(IngestModuleLoader.XmlPipelineRaw.PIPELINE_TYPE.FILE_ANALYSIS.toString())) {
continue; //skip // continue; //skip
} // }
//
for (IngestModuleLoader.XmlModuleRaw rawM : rawP.modules) { // for (IngestModuleLoader.XmlModuleRaw rawM : rawP.modules) {
//logger.log(Level.INFO, "CLASS NAME : " + foundClass.getName()); // //logger.log(Level.INFO, "CLASS NAME : " + foundClass.getName());
if (foundClass.getName().equals(rawM.location)) { // if (foundClass.getName().equals(rawM.location)) {
exists = true; // exists = true;
break; // break;
} // }
} // }
if (exists == true) { // if (exists == true) {
break; // break;
} // }
} // }
//
if (exists == false) { // if (exists == false) {
logger.log(Level.INFO, "Discovered a new file module to load: " + foundClass.getName()); // logger.log(Level.INFO, "Discovered a new file module to load: " + foundClass.getName());
//ADD MODULE // //ADD MODULE
addModuleToRawPipeline(foundClass, IngestModuleLoader.XmlPipelineRaw.PIPELINE_TYPE.FILE_ANALYSIS); // addModuleToRawPipeline(foundClass, IngestModuleLoader.XmlPipelineRaw.PIPELINE_TYPE.FILE_ANALYSIS);
modulesChanged = true; // modulesChanged = true;
} // }
//
} // }
//
it = dataSourceModules.iterator(); // it = dataSourceModules.iterator();
while (it.hasNext()) { // while (it.hasNext()) {
boolean exists = false; // boolean exists = false;
Class<IngestModuleDataSource> foundClass = (Class<IngestModuleDataSource>) it.next(); // Class<IngestModuleDataSource> foundClass = (Class<IngestModuleDataSource>) it.next();
//
for (IngestModuleLoader.XmlPipelineRaw rawP : pipelinesXML) { // for (IngestModuleLoader.XmlPipelineRaw rawP : pipelinesXML) {
if (!rawP.type.equals(IngestModuleLoader.XmlPipelineRaw.PIPELINE_TYPE.DATA_SOURCE_ANALYSIS.toString())) { // if (!rawP.type.equals(IngestModuleLoader.XmlPipelineRaw.PIPELINE_TYPE.DATA_SOURCE_ANALYSIS.toString())) {
continue; //skip // continue; //skip
} // }
//
//
for (IngestModuleLoader.XmlModuleRaw rawM : rawP.modules) { // for (IngestModuleLoader.XmlModuleRaw rawM : rawP.modules) {
//logger.log(Level.INFO, "CLASS NAME : " + foundClass.getName()); // //logger.log(Level.INFO, "CLASS NAME : " + foundClass.getName());
if (foundClass.getName().equals(rawM.location)) { // if (foundClass.getName().equals(rawM.location)) {
exists = true; // exists = true;
break; // break;
} // }
} // }
if (exists == true) { // if (exists == true) {
break; // break;
} // }
} // }
//
if (exists == false) { // if (exists == false) {
logger.log(Level.INFO, "Discovered a new DataSource module to load: " + foundClass.getName()); // logger.log(Level.INFO, "Discovered a new DataSource module to load: " + foundClass.getName());
//ADD MODULE // //ADD MODULE
addModuleToRawPipeline(foundClass, IngestModuleLoader.XmlPipelineRaw.PIPELINE_TYPE.DATA_SOURCE_ANALYSIS); // addModuleToRawPipeline(foundClass, IngestModuleLoader.XmlPipelineRaw.PIPELINE_TYPE.DATA_SOURCE_ANALYSIS);
modulesChanged = true; // modulesChanged = true;
} // }
//
} // }
//
if (modulesChanged) { // if (modulesChanged) {
save(); // save();
//
try { // try {
pcs.firePropertyChange(IngestModuleLoader.Event.ModulesReloaded.toString(), 0, 1); // pcs.firePropertyChange(IngestModuleLoader.Event.ModulesReloaded.toString(), 0, 1);
} // }
catch (Exception e) { // catch (Exception e) {
logger.log(Level.SEVERE, "IngestModuleLoader listener threw exception", e); // logger.log(Level.SEVERE, "IngestModuleLoader listener threw exception", e);
MessageNotifyUtil.Notify.show("Module Error", "A module caused an error listening to IngestModuleLoader updates. See log to determine which module. Some data could be incomplete.", MessageNotifyUtil.MessageType.ERROR); // MessageNotifyUtil.Notify.show("Module Error", "A module caused an error listening to IngestModuleLoader updates. See log to determine which module. Some data could be incomplete.", MessageNotifyUtil.MessageType.ERROR);
} // }
} // }
// }
/*
//Enumeration<URL> resources = moduleClassLoader.getResources(basePackageName);
Enumeration<URL> resources = classLoader.getResources(basePackageName);
while (resources.hasMoreElements()) {
System.out.println(resources.nextElement());
} */
}
} }
/** /**
@ -756,100 +750,97 @@ import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil;
private void instantiate() throws IngestModuleLoaderException { private void instantiate() throws IngestModuleLoaderException {
//clear current //clear current
filePipeline.clear(); // filePipeline.clear();
dataSourcePipeline.clear(); // dataSourcePipeline.clear();
//add autodiscovered modules to pipelinesXML //add autodiscovered modules to pipelinesXML
autodiscover(); autodiscover();
//validate all modules: from XML + just autodiscovered //validate all modules: from XML + just autodiscovered
validate(); // validate();
//
for (IngestModuleLoader.XmlPipelineRaw pRaw : pipelinesXML) { // for (IngestModuleLoader.XmlPipelineRaw pRaw : pipelinesXML) {
if (pRaw.valid == false) { // if (pRaw.valid == false) {
//skip invalid pipelines // //skip invalid pipelines
continue; // continue;
} // }
//
//sort modules by order parameter, in case XML order is different // //sort modules by order parameter, in case XML order is different
Collections.sort(pRaw.modules, new Comparator<IngestModuleLoader.XmlModuleRaw>() { // Collections.sort(pRaw.modules, new Comparator<IngestModuleLoader.XmlModuleRaw>() {
@Override // @Override
public int compare(IngestModuleLoader.XmlModuleRaw o1, IngestModuleLoader.XmlModuleRaw o2) { // public int compare(IngestModuleLoader.XmlModuleRaw o1, IngestModuleLoader.XmlModuleRaw o2) {
return Integer.valueOf(o1.order).compareTo(Integer.valueOf(o2.order)); // return Integer.valueOf(o1.order).compareTo(Integer.valueOf(o2.order));
} // }
}); // });
//
//check pipelineType, add to right pipeline collection // //check pipelineType, add to right pipeline collection
IngestModuleLoader.XmlPipelineRaw.PIPELINE_TYPE pType = IngestModuleLoader.XmlPipelineRaw.getPipelineType(pRaw.type); // IngestModuleLoader.XmlPipelineRaw.PIPELINE_TYPE pType = IngestModuleLoader.XmlPipelineRaw.getPipelineType(pRaw.type);
//
for (IngestModuleLoader.XmlModuleRaw pMod : pRaw.modules) { // for (IngestModuleLoader.XmlModuleRaw pMod : pRaw.modules) {
try { // try {
if (pMod.valid == false) { // if (pMod.valid == false) {
//skip invalid modules // //skip invalid modules
continue; // continue;
} // }
//
//add to right pipeline // //add to right pipeline
switch (pType) { // switch (pType) {
case FILE_ANALYSIS: // case FILE_ANALYSIS:
IngestModuleAbstractFile fileModuleInstance = null; // IngestModuleAbstractFile fileModuleInstance = null;
final Class<IngestModuleAbstractFile> fileModuleClass = // final Class<IngestModuleAbstractFile> fileModuleClass =
(Class<IngestModuleAbstractFile>) Class.forName(pMod.location, true, classLoader); // (Class<IngestModuleAbstractFile>) Class.forName(pMod.location, true, classLoader);
try { // try {
Method getDefaultMethod = fileModuleClass.getMethod("getDefault"); // Method getDefaultMethod = fileModuleClass.getMethod("getDefault");
if (getDefaultMethod != null) { // if (getDefaultMethod != null) {
fileModuleInstance = (IngestModuleAbstractFile) getDefaultMethod.invoke(null); // fileModuleInstance = (IngestModuleAbstractFile) getDefaultMethod.invoke(null);
} // }
} catch (NoSuchMethodException ex) { // } catch (NoSuchMethodException ex) {
logger.log(Level.WARNING, "Validated module, but not public getDefault() found: " + pMod.location); // logger.log(Level.WARNING, "Validated module, but not public getDefault() found: " + pMod.location);
pMod.valid = false; //prevent from trying to load again // pMod.valid = false; //prevent from trying to load again
} catch (SecurityException ex) { // } catch (SecurityException ex) {
logger.log(Level.WARNING, "Validated module, but not public getDefault() found: " + pMod.location); // logger.log(Level.WARNING, "Validated module, but not public getDefault() found: " + pMod.location);
pMod.valid = false; //prevent from trying to load again // pMod.valid = false; //prevent from trying to load again
} catch (IllegalAccessException ex) { // } catch (IllegalAccessException ex) {
logger.log(Level.WARNING, "Validated module, but not public getDefault() found: " + pMod.location); // logger.log(Level.WARNING, "Validated module, but not public getDefault() found: " + pMod.location);
pMod.valid = false; //prevent from trying to load again // pMod.valid = false; //prevent from trying to load again
} catch (InvocationTargetException ex) { // } catch (InvocationTargetException ex) {
logger.log(Level.WARNING, "Validated module, but not public getDefault() found: " + pMod.location); // logger.log(Level.WARNING, "Validated module, but not public getDefault() found: " + pMod.location);
pMod.valid = false; //prevent from trying to load again // pMod.valid = false; //prevent from trying to load again
} // }
//
filePipeline.add(fileModuleInstance); // filePipeline.add(fileModuleInstance);
break; // break;
case DATA_SOURCE_ANALYSIS: // case DATA_SOURCE_ANALYSIS:
final Class<IngestModuleDataSource> dataSourceModuleClass = // final Class<IngestModuleDataSource> dataSourceModuleClass =
(Class<IngestModuleDataSource>) Class.forName(pMod.location, true, classLoader); // (Class<IngestModuleDataSource>) Class.forName(pMod.location, true, classLoader);
//
try { // try {
Constructor<IngestModuleDataSource> constr = dataSourceModuleClass.getConstructor(); // Constructor<IngestModuleDataSource> constr = dataSourceModuleClass.getConstructor();
IngestModuleDataSource dataSourceModuleInstance = constr.newInstance(); // IngestModuleDataSource dataSourceModuleInstance = constr.newInstance();
//
if (dataSourceModuleInstance != null) { // if (dataSourceModuleInstance != null) {
dataSourcePipeline.add(dataSourceModuleInstance); // dataSourcePipeline.add(dataSourceModuleInstance);
} // }
//
} catch (NoSuchMethodException | SecurityException | InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException ex) { // } catch (NoSuchMethodException | SecurityException | InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException ex) {
logger.log(Level.WARNING, "Validated module, could not initialize, check for bugs in the module: " + pMod.location, ex); // logger.log(Level.WARNING, "Validated module, could not initialize, check for bugs in the module: " + pMod.location, ex);
pMod.valid = false; // pMod.valid = false;
} // }
//
//
break; // break;
default: // default:
logger.log(Level.SEVERE, "Unexpected pipeline type to add module to: " + pType); // logger.log(Level.SEVERE, "Unexpected pipeline type to add module to: " + pType);
} // }
//
//
} catch (ClassNotFoundException ex) { // } catch (ClassNotFoundException ex) {
logger.log(Level.SEVERE, "Validated module, but could not load (shouldn't happen): " + pMod.location); // logger.log(Level.SEVERE, "Validated module, but could not load (shouldn't happen): " + pMod.location);
} // }
} // }
//
} //end instantiating modules in XML // } //end instantiating modules in XML
} }
/** /**
@ -957,18 +948,18 @@ import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil;
* @throws IngestModuleLoaderException * @throws IngestModuleLoaderException
*/ */
public synchronized void init() throws IngestModuleLoaderException { public synchronized void init() throws IngestModuleLoaderException {
absFilePath = PlatformUtil.getUserConfigDirectory() + File.separator + PIPELINE_CONFIG_XML; // absFilePath = PlatformUtil.getUserConfigDirectory() + File.separator + PIPELINE_CONFIG_XML;
ClassLoader parentClassLoader = Lookup.getDefault().lookup(ClassLoader.class); // ClassLoader parentClassLoader = Lookup.getDefault().lookup(ClassLoader.class);
classLoader = new CustomClassLoader(parentClassLoader); // classLoader = new CustomClassLoader(parentClassLoader);
//
try { // try {
boolean extracted = PlatformUtil.extractResourceToUserConfigDir(IngestModuleLoader.class, PIPELINE_CONFIG_XML); // boolean extracted = PlatformUtil.extractResourceToUserConfigDir(IngestModuleLoader.class, PIPELINE_CONFIG_XML);
} catch (IOException ex) { // } catch (IOException ex) {
logger.log(Level.SEVERE, "Error copying default pipeline configuration to user dir ", ex); // logger.log(Level.SEVERE, "Error copying default pipeline configuration to user dir ", ex);
} // }
//
//load the pipeline config // //load the pipeline config
loadRawPipeline(); // loadRawPipeline();
instantiate(); instantiate();

43
Core/src/org/sleuthkit/autopsy/ingest/IngestPipelines.java Executable file
View File

@ -0,0 +1,43 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2014 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.ingest;
import java.util.ArrayList;
import java.util.List;
/**
*
* @author rcordovano
*/
public class IngestPipelines {
private List<IngestModuleTemplate> fileIngestPipeline = new ArrayList<>();
private List<IngestModuleTemplate> dataSourceIngestPipeline = new ArrayList<>();
IngestPipelines(List<IngestModuleTemplate> moduleTemplates) {
for (IngestModuleTemplate moduleTemplate : moduleTemplates) {
if (moduleTemplate.getIngestModuleFactory().isFileIngestModuleFactory()) {
fileIngestPipeline.add(moduleTemplate);
}
else {
dataSourceIngestPipeline.add(moduleTemplate);
}
}
}
}