Merge pull request #6236 from markmckinnon/6727-iLeapp-module-parser-with-minimal-parsing-of-plugins

6727 i leapp module parser with minimal parsing of plugins
This commit is contained in:
Richard Cordovano 2020-09-14 16:14:24 -04:00 committed by GitHub
commit 039f409732
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 938 additions and 5 deletions

View File

@ -48,6 +48,11 @@
<copy todir="${basedir}/release/gstreamer" >
<fileset dir="${thirdparty.dir}/gstreamer"/>
</copy>
<!--Copy iLeapp to release-->
<copy todir="${basedir}/release/iLeapp" >
<fileset dir="${thirdparty.dir}/iLeapp"/>
</copy>
<!--Copy 7-Zip to release-->
<copy todir="${basedir}/release/7-Zip" >

View File

@ -0,0 +1,3 @@
ILeappAnalyzerIngestModule.init.exception.msg=Unable to find {0}.
ILeappAnalyzerIngestModule.processing.file=Processing file {0}
ILeappAnalyzerIngestModule.parsing.file=Parsing file {0}

View File

@ -0,0 +1,29 @@
ILeappAnalyzerIngestModule.completed=iLeapp Processing Completed
ILeappAnalyzerIngestModule.error.creating.output.dir=Error creating iLeapp module output directory.
ILeappAnalyzerIngestModule.error.ileapp.file.processor.init=Failure to initialize ILeappProcessFile
ILeappAnalyzerIngestModule.error.running.iLeapp=Error running iLeapp, see log file.
ILeappAnalyzerIngestModule.executable.not.found=iLeapp Executable Not Found.
ILeappAnalyzerIngestModule.has.run=iLeapp
ILeappAnalyzerIngestModule.iLeapp.cancelled=iLeapp run was canceled
ILeappAnalyzerIngestModule.init.exception.msg=Unable to find {0}.
ILeappAnalyzerIngestModule.processing.file=Processing file {0}
ILeappAnalyzerIngestModule.parsing.file=Parsing file {0}
ILeappAnalyzerIngestModule.report.name=iLeapp Html Report
ILeappAnalyzerIngestModule.requires.windows=iLeapp module requires windows.
ILeappAnalyzerIngestModule.running.iLeapp=Running iLeapp
ILeappAnalyzerIngestModule.starting.iLeapp=Starting iLeapp
ILeappAnalyzerModuleFactory_moduleDesc=Runs iLeapp against files.
ILeappAnalyzerModuleFactory_moduleName=ILeapp Analyzer
ILeappFileProcessor.cannot.load.artifact.xml=Cannor load xml artifact file.
ILeappFileProcessor.cannotBuildXmlParser=Cannot buld an XML parser.
ILeappFileProcessor.completed=iLeapp Processing Completed
ILeappFileProcessor.error.creating.new.artifacts=Error creating new artifacts.
ILeappFileProcessor.error.creating.output.dir=Error creating iLeapp module output directory.
ILeappFileProcessor.error.reading.iLeapp.directory=Error reading iLeapp Output Directory
ILeappFileProcessor.error.running.iLeapp=Error running iLeapp, see log file.
ILeappFileProcessor.has.run=iLeapp
ILeappFileProcessor.iLeapp.cancelled=iLeapp run was canceled
ILeappFileProcessor.postartifacts_error=Error posting Blackboard Artifact
ILeappFileProcessor.running.iLeapp=Running iLeapp
ILeappFileProcessor.starting.iLeapp=Starting iLeapp
ILeappFileProcessor_cannotParseXml=Cannot Parse XML file.

View File

@ -0,0 +1,259 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2020 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.modules.ileappanalyzer;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.text.SimpleDateFormat;
import java.util.List;
import java.util.ArrayList;
import java.util.Locale;
import java.util.logging.Level;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.openide.modules.InstalledFileLocator;
import org.openide.util.NbBundle;
import org.sleuthkit.autopsy.casemodule.Case;
import static org.sleuthkit.autopsy.casemodule.Case.getCurrentCase;
import org.sleuthkit.autopsy.casemodule.services.FileManager;
import org.sleuthkit.autopsy.coreutils.ExecUtil;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.coreutils.PlatformUtil;
import org.sleuthkit.autopsy.ingest.DataSourceIngestModule;
import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProcessTerminator;
import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProgress;
import org.sleuthkit.autopsy.ingest.IngestJobContext;
import org.sleuthkit.autopsy.ingest.IngestMessage;
import org.sleuthkit.autopsy.ingest.IngestServices;
import org.sleuthkit.autopsy.ingest.IngestModule.IngestModuleException;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.Content;
import org.sleuthkit.datamodel.TskCoreException;
/**
* Data source ingest module that runs iLeapp against logical iOS files.
*/
public class ILeappAnalyzerIngestModule implements DataSourceIngestModule {
private static final Logger logger = Logger.getLogger(ILeappAnalyzerIngestModule.class.getName());
private static final String MODULE_NAME = ILeappAnalyzerModuleFactory.getModuleName();
private static final String ILEAPP = "iLeapp"; //NON-NLS
private static final String ILEAPP_EXECUTABLE = "ileapp.exe";//NON-NLS
private File iLeappExecutable;
private IngestJobContext context;
private ILeappFileProcessor iLeappFileProcessor;
ILeappAnalyzerIngestModule() {
// This constructor is intentionally empty. Nothing special is needed here.
}
@NbBundle.Messages({
"ILeappAnalyzerIngestModule.executable.not.found=iLeapp Executable Not Found.",
"ILeappAnalyzerIngestModule.requires.windows=iLeapp module requires windows.",
"ILeappAnalyzerIngestModule.error.ileapp.file.processor.init=Failure to initialize ILeappProcessFile"})
@Override
public void startUp(IngestJobContext context) throws IngestModuleException {
this.context = context;
if (false == PlatformUtil.isWindowsOS()) {
throw new IngestModuleException(Bundle.ILeappAnalyzerIngestModule_requires_windows());
}
try {
iLeappFileProcessor = new ILeappFileProcessor();
} catch (IOException | IngestModuleException ex) {
throw new IngestModuleException(Bundle.ILeappAnalyzerIngestModule_error_ileapp_file_processor_init(), ex);
}
try {
iLeappExecutable = locateExecutable(ILEAPP_EXECUTABLE);
} catch (FileNotFoundException exception) {
logger.log(Level.WARNING, "iLeapp executable not found.", exception); //NON-NLS
throw new IngestModuleException(Bundle.ILeappAnalyzerIngestModule_executable_not_found(), exception);
}
}
@NbBundle.Messages({
"ILeappAnalyzerIngestModule.error.running.iLeapp=Error running iLeapp, see log file.",
"ILeappAnalyzerIngestModule.error.creating.output.dir=Error creating iLeapp module output directory.",
"ILeappAnalyzerIngestModule.starting.iLeapp=Starting iLeapp",
"ILeappAnalyzerIngestModule.running.iLeapp=Running iLeapp",
"ILeappAnalyzerIngestModule.has.run=iLeapp",
"ILeappAnalyzerIngestModule.iLeapp.cancelled=iLeapp run was canceled",
"ILeappAnalyzerIngestModule.completed=iLeapp Processing Completed",
"ILeappAnalyzerIngestModule.report.name=iLeapp Html Report"})
@Override
public ProcessResult process(Content dataSource, DataSourceIngestModuleProgress statusHelper) {
statusHelper.progress(Bundle.ILeappAnalyzerIngestModule_starting_iLeapp(), 0);
List<AbstractFile> iLeappFilesToProcess = findiLeappFilesToProcess(dataSource);
statusHelper.switchToDeterminate(iLeappFilesToProcess.size());
Integer filesProcessedCount = 0;
Case currentCase = Case.getCurrentCase();
for (AbstractFile iLeappFile : iLeappFilesToProcess) {
String currentTime = new SimpleDateFormat("yyyy-MM-dd HH-mm-ss z", Locale.US).format(System.currentTimeMillis());//NON-NLS
Path moduleOutputPath = Paths.get(currentCase.getModuleDirectory(), ILEAPP, currentTime);
try {
Files.createDirectories(moduleOutputPath);
} catch (IOException ex) {
logger.log(Level.SEVERE, String.format("Error creating iLeapp output directory %s", moduleOutputPath.toString()), ex);
return ProcessResult.ERROR;
}
statusHelper.progress(NbBundle.getMessage(this.getClass(), "ILeappAnalyzerIngestModule.processing.file", iLeappFile.getName()), filesProcessedCount);
ProcessBuilder iLeappCommand = buildiLeappCommand(moduleOutputPath, iLeappFile.getLocalAbsPath(), iLeappFile.getNameExtension());
try {
int result = ExecUtil.execute(iLeappCommand, new DataSourceIngestModuleProcessTerminator(context));
if (result != 0) {
logger.log(Level.SEVERE, String.format("Error running iLeapp, error code returned %d", result)); //NON-NLS
return ProcessResult.ERROR;
}
addILeappReportToReports(moduleOutputPath, currentCase);
} catch (IOException ex) {
logger.log(Level.SEVERE, String.format("Error when trying to execute iLeapp program against file %s", iLeappFile.getLocalAbsPath()), ex);
return ProcessResult.ERROR;
}
if (context.dataSourceIngestIsCancelled()) {
logger.log(Level.INFO, "ILeapp Analyser ingest module run was canceled"); //NON-NLS
return ProcessResult.OK;
}
ProcessResult fileProcessorResult = iLeappFileProcessor.processFiles(dataSource, moduleOutputPath, iLeappFile);
if (fileProcessorResult == ProcessResult.ERROR) {
return ProcessResult.ERROR;
}
filesProcessedCount++;
}
IngestMessage message = IngestMessage.createMessage(IngestMessage.MessageType.DATA,
Bundle.ILeappAnalyzerIngestModule_has_run(),
Bundle.ILeappAnalyzerIngestModule_completed());
IngestServices.getInstance().postMessage(message);
return ProcessResult.OK;
}
/**
* Find the files that will be processed by the iLeapp program
*
* @param dataSource
*
* @return List of abstract files to process.
*/
private List<AbstractFile> findiLeappFilesToProcess(Content dataSource) {
List<AbstractFile> iLeappFiles = new ArrayList<>();
FileManager fileManager = getCurrentCase().getServices().getFileManager();
// findFiles use the SQL wildcard % in the file name
try {
iLeappFiles = fileManager.findFiles(dataSource, "%", "/"); //NON-NLS
} catch (TskCoreException ex) {
logger.log(Level.WARNING, "No files found to process"); //NON-NLS
return iLeappFiles;
}
List<AbstractFile> iLeappFilesToProcess = new ArrayList<>();
for (AbstractFile iLeappFile : iLeappFiles) {
if ((iLeappFile.getName().toLowerCase().contains(".zip") || (iLeappFile.getName().toLowerCase().contains(".tar"))
|| iLeappFile.getName().toLowerCase().contains(".tgz"))) {
iLeappFilesToProcess.add(iLeappFile);
}
}
return iLeappFilesToProcess;
}
private ProcessBuilder buildiLeappCommand(Path moduleOutputPath, String sourceFilePath, String iLeappFileSystemType) {
ProcessBuilder processBuilder = buildProcessWithRunAsInvoker(
"\"" + iLeappExecutable + "\"", //NON-NLS
"-t", iLeappFileSystemType, //NON-NLS
"-i", sourceFilePath, //NON-NLS
"-o", moduleOutputPath.toString()
);
processBuilder.redirectError(moduleOutputPath.resolve("iLeapp_err.txt").toFile()); //NON-NLS
processBuilder.redirectOutput(moduleOutputPath.resolve("iLeapp_out.txt").toFile()); //NON-NLS
return processBuilder;
}
static private ProcessBuilder buildProcessWithRunAsInvoker(String... commandLine) {
ProcessBuilder processBuilder = new ProcessBuilder(commandLine);
/*
* Add an environment variable to force log2timeline/psort to run with
* the same permissions Autopsy uses.
*/
processBuilder.environment().put("__COMPAT_LAYER", "RunAsInvoker"); //NON-NLS
return processBuilder;
}
private static File locateExecutable(String executableName) throws FileNotFoundException {
String executableToFindName = Paths.get(ILEAPP, executableName).toString();
File exeFile = InstalledFileLocator.getDefault().locate(executableToFindName, ILeappAnalyzerIngestModule.class.getPackage().getName(), false);
if (null == exeFile || exeFile.canExecute() == false) {
throw new FileNotFoundException(executableName + " executable not found.");
}
return exeFile;
}
/**
* Find the index.html file in the iLeapp output directory so it can be
* added to reports
*/
private void addILeappReportToReports(Path iLeappOutputDir, Case currentCase) {
List<String> allIndexFiles = new ArrayList<>();
try (Stream<Path> walk = Files.walk(iLeappOutputDir)) {
allIndexFiles = walk.map(x -> x.toString())
.filter(f -> f.toLowerCase().endsWith("index.html")).collect(Collectors.toList());
if (!allIndexFiles.isEmpty()) {
currentCase.addReport(allIndexFiles.get(0), MODULE_NAME, Bundle.ILeappAnalyzerIngestModule_report_name());
}
} catch (IOException | UncheckedIOException | TskCoreException ex) {
// catch the error and continue on as report is not added
logger.log(Level.WARNING, String.format("Error finding index file in path %s", iLeappOutputDir.toString()), ex);
}
}
}

View File

@ -0,0 +1,67 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2020 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.modules.ileappanalyzer;
import org.openide.util.NbBundle;
import org.openide.util.lookup.ServiceProvider;
import org.sleuthkit.autopsy.coreutils.Version;
import org.sleuthkit.autopsy.ingest.DataSourceIngestModule;
import org.sleuthkit.autopsy.ingest.IngestModuleFactory;
import org.sleuthkit.autopsy.ingest.IngestModuleFactoryAdapter;
import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettings;
/**
* A factory that creates data source ingest modules that will run iLeapp
* against logical files and saves the output to module output.
*/
@ServiceProvider(service = IngestModuleFactory.class)
public class ILeappAnalyzerModuleFactory extends IngestModuleFactoryAdapter {
@NbBundle.Messages({"ILeappAnalyzerModuleFactory_moduleName=ILeapp Analyzer"})
static String getModuleName() {
return Bundle.ILeappAnalyzerModuleFactory_moduleName();
}
@Override
public String getModuleDisplayName() {
return getModuleName();
}
@NbBundle.Messages({"ILeappAnalyzerModuleFactory_moduleDesc=Runs iLeapp against files."})
@Override
public String getModuleDescription() {
return Bundle.ILeappAnalyzerModuleFactory_moduleDesc();
}
@Override
public String getModuleVersionNumber() {
return Version.getVersion();
}
@Override
public boolean isDataSourceIngestModuleFactory() {
return true;
}
@Override
public DataSourceIngestModule createDataSourceIngestModule(IngestModuleIngestJobSettings ingestJobOptions) {
return new ILeappAnalyzerIngestModule();
}
}

View File

@ -0,0 +1,446 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2020 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.modules.ileappanalyzer;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.List;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Date;
import java.util.HashMap;
import static java.util.Locale.US;
import java.util.Map;
import java.util.logging.Level;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import org.apache.commons.io.FilenameUtils;
import org.openide.util.NbBundle;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.coreutils.PlatformUtil;
import org.sleuthkit.autopsy.ingest.IngestModule.IngestModuleException;
import org.sleuthkit.autopsy.ingest.IngestModule.ProcessResult;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.Blackboard;
import org.sleuthkit.datamodel.BlackboardArtifact;
import org.sleuthkit.datamodel.BlackboardAttribute;
import org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE;
import org.sleuthkit.datamodel.Content;
import org.sleuthkit.datamodel.TskCoreException;
import org.sleuthkit.datamodel.TskException;
import org.w3c.dom.Document;
import org.w3c.dom.NamedNodeMap;
import org.w3c.dom.NodeList;
import org.xml.sax.SAXException;
/**
* Find and process output from iLeapp program and bring into Autopsy
*/
public final class ILeappFileProcessor {
private static final Logger logger = Logger.getLogger(ILeappFileProcessor.class.getName());
private static final String MODULE_NAME = ILeappAnalyzerModuleFactory.getModuleName();
private static final String XMLFILE = "ileap-artifact-attribute-reference.xml"; //NON-NLS
private final Map<String, String> tsvFiles;
private final Map<String, String> tsvFileArtifacts;
private final Map<String, String> tsvFileArtifactComments;
private final Map<String, List<List<String>>> tsvFileAttributes;
public ILeappFileProcessor() throws IOException, IngestModuleException {
this.tsvFiles = new HashMap<>();
this.tsvFileArtifacts = new HashMap<>();
this.tsvFileArtifactComments = new HashMap<>();
this.tsvFileAttributes = new HashMap<>();
configExtractor();
loadConfigFile();
}
@NbBundle.Messages({
"ILeappFileProcessor.error.running.iLeapp=Error running iLeapp, see log file.",
"ILeappFileProcessor.error.creating.output.dir=Error creating iLeapp module output directory.",
"ILeappFileProcessor.starting.iLeapp=Starting iLeapp",
"ILeappFileProcessor.running.iLeapp=Running iLeapp",
"ILeappFileProcessor.has.run=iLeapp",
"ILeappFileProcessor.iLeapp.cancelled=iLeapp run was canceled",
"ILeappFileProcessor.completed=iLeapp Processing Completed",
"ILeappFileProcessor.error.reading.iLeapp.directory=Error reading iLeapp Output Directory"})
public ProcessResult processFiles(Content dataSource, Path moduleOutputPath, AbstractFile iLeappFile) {
try {
List<String> iLeappTsvOutputFiles = findTsvFiles(moduleOutputPath);
processiLeappFiles(iLeappTsvOutputFiles, iLeappFile);
} catch (IOException | IngestModuleException ex) {
logger.log(Level.SEVERE, String.format("Error trying to process iLeapp output files in directory %s. ", moduleOutputPath.toString()), ex); //NON-NLS
return ProcessResult.ERROR;
}
return ProcessResult.OK;
}
/**
* Find the tsv files in the iLeapp output directory and match them to files
* we know we want to process and return the list to process those files.
*/
private List<String> findTsvFiles(Path iLeappOutputDir) throws IngestModuleException {
List<String> allTsvFiles = new ArrayList<>();
List<String> foundTsvFiles = new ArrayList<>();
try (Stream<Path> walk = Files.walk(iLeappOutputDir)) {
allTsvFiles = walk.map(x -> x.toString())
.filter(f -> f.toLowerCase().endsWith(".tsv")).collect(Collectors.toList());
for (String tsvFile : allTsvFiles) {
if (tsvFiles.containsKey(FilenameUtils.getName(tsvFile))) {
foundTsvFiles.add(tsvFile);
}
}
} catch (IOException | UncheckedIOException e) {
throw new IngestModuleException(Bundle.ILeappFileProcessor_error_reading_iLeapp_directory() + iLeappOutputDir.toString(), e);
}
return foundTsvFiles;
}
/**
* Process the iLeapp files that were found that match the xml mapping file
*
* @param iLeappFilesToProcess List of files to process
* @param iLeappImageFile Abstract file to create artifact for
* @param statusHelper progress bar update
*
* @throws FileNotFoundException
* @throws IOException
*/
private void processiLeappFiles(List<String> iLeappFilesToProcess, AbstractFile iLeappImageFile) throws FileNotFoundException, IOException, IngestModuleException {
List<BlackboardArtifact> bbartifacts = new ArrayList<>();
for (String iLeappFileName : iLeappFilesToProcess) {
String fileName = FilenameUtils.getName(iLeappFileName);
File iLeappFile = new File(iLeappFileName);
if (tsvFileAttributes.containsKey(fileName)) {
List<List<String>> attrList = tsvFileAttributes.get(fileName);
try {
BlackboardArtifact.Type artifactType = Case.getCurrentCase().getSleuthkitCase().getArtifactType(tsvFileArtifacts.get(fileName));
processFile(iLeappFile, attrList, fileName, artifactType, bbartifacts, iLeappImageFile);
} catch (TskCoreException ex) {
// check this
throw new IngestModuleException(String.format("Error getting Blackboard Artifact Type for %s", tsvFileArtifacts.get(fileName)), ex);
}
}
}
if (!bbartifacts.isEmpty()) {
postArtifacts(bbartifacts);
}
}
private void processFile(File iLeappFile, List<List<String>> attrList, String fileName, BlackboardArtifact.Type artifactType,
List<BlackboardArtifact> bbartifacts, AbstractFile iLeappImageFile) throws FileNotFoundException, IOException, IngestModuleException {
try (BufferedReader reader = new BufferedReader(new FileReader(iLeappFile))) {
String line = reader.readLine();
// Check first line, if it is null then no heading so nothing to match to, close and go to next file.
if (line != null) {
Map<Integer, String> columnNumberToProcess = findColumnsToProcess(line, attrList);
line = reader.readLine();
while (line != null) {
Collection<BlackboardAttribute> bbattributes = processReadLine(line, columnNumberToProcess, fileName);
if (!bbattributes.isEmpty()) {
BlackboardArtifact bbartifact = createArtifactWithAttributes(artifactType.getTypeID(), iLeappImageFile, bbattributes);
if (bbartifact != null) {
bbartifacts.add(bbartifact);
}
}
line = reader.readLine();
}
}
}
}
/**
* Process the line read and create the necessary attributes for it
*
* @param line a tsv line to process that was read
* @param columnNumberToProcess Which columns to process in the tsv line
*
* @return
*/
private Collection<BlackboardAttribute> processReadLine(String line, Map<Integer, String> columnNumberToProcess, String fileName) throws IngestModuleException {
String[] columnValues = line.split("\\t");
Collection<BlackboardAttribute> bbattributes = new ArrayList<BlackboardAttribute>();
for (Map.Entry<Integer, String> columnToProcess : columnNumberToProcess.entrySet()) {
Integer columnNumber = columnToProcess.getKey();
String attributeName = columnToProcess.getValue();
try {
BlackboardAttribute.Type attributeType = Case.getCurrentCase().getSleuthkitCase().getAttributeType(attributeName.toUpperCase());
if (attributeType == null) {
break;
}
String attrType = attributeType.getValueType().getLabel().toUpperCase();
checkAttributeType(bbattributes, attrType, columnValues, columnNumber, attributeType);
} catch (TskCoreException ex) {
throw new IngestModuleException(String.format("Error getting Attribute type for Attribute Name %s", attributeName), ex); //NON-NLS
}
}
if (tsvFileArtifactComments.containsKey(fileName)) {
bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_COMMENT, MODULE_NAME, tsvFileArtifactComments.get(fileName)));
}
return bbattributes;
}
private void checkAttributeType(Collection<BlackboardAttribute> bbattributes, String attrType, String[] columnValues, Integer columnNumber, BlackboardAttribute.Type attributeType) {
if (attrType.matches("STRING")) {
bbattributes.add(new BlackboardAttribute(attributeType, MODULE_NAME, columnValues[columnNumber]));
} else if (attrType.matches("INTEGER")) {
bbattributes.add(new BlackboardAttribute(attributeType, MODULE_NAME, Integer.valueOf(columnValues[columnNumber])));
} else if (attrType.matches("LONG")) {
bbattributes.add(new BlackboardAttribute(attributeType, MODULE_NAME, Long.valueOf(columnValues[columnNumber])));
} else if (attrType.matches("DOUBLE")) {
bbattributes.add(new BlackboardAttribute(attributeType, MODULE_NAME, Double.valueOf(columnValues[columnNumber])));
} else if (attrType.matches("BYTE")) {
bbattributes.add(new BlackboardAttribute(attributeType, MODULE_NAME, Byte.valueOf(columnValues[columnNumber])));
} else if (attrType.matches("DATETIME")) {
// format of data should be the same in all the data and the format is 2020-03-28 01:00:17
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-d HH:mm:ss", US);
Long dateLong = Long.valueOf(0);
try {
Date newDate = dateFormat.parse(columnValues[columnNumber]);
dateLong = newDate.getTime() / 1000;
bbattributes.add(new BlackboardAttribute(attributeType, MODULE_NAME, dateLong));
} catch (ParseException ex) {
// catching error and displaying date that could not be parsed
// we set the timestamp to 0 and continue on processing
logger.log(Level.WARNING, String.format("Failed to parse date/time %s for attribute.", columnValues[columnNumber]), ex); //NON-NLS
}
} else if (attrType.matches("JSON")) {
bbattributes.add(new BlackboardAttribute(attributeType, MODULE_NAME, columnValues[columnNumber]));
} else {
// Log this and continue on with processing
logger.log(Level.WARNING, String.format("Attribute Type %s not defined.", attrType)); //NON-NLS
}
}
/**
* Process the first line of the tsv file which has the headings. Match the
* headings to the columns in the XML mapping file so we know which columns
* to process.
*
* @param line a tsv heading line of the columns in the file
* @param attrList the list of headings we want to process
*
* @return the numbered column(s) and attribute(s) we want to use for the
* column(s)
*/
private Map<Integer, String> findColumnsToProcess(String line, List<List<String>> attrList) {
String[] columnNames = line.split("\\t");
HashMap<Integer, String> columnsToProcess = new HashMap<>();
Integer columnPosition = 0;
for (String columnName : columnNames) {
// for some reason the first column of the line has unprintable characters so removing them
String cleanColumnName = columnName.replaceAll("[^\\n\\r\\t\\p{Print}]", "");
for (List<String> atList : attrList) {
if (atList.contains(cleanColumnName.toLowerCase())) {
columnsToProcess.put(columnPosition, atList.get(0));
break;
}
}
columnPosition++;
}
return columnsToProcess;
}
@NbBundle.Messages({
"ILeappFileProcessor.cannot.load.artifact.xml=Cannor load xml artifact file.",
"ILeappFileProcessor.cannotBuildXmlParser=Cannot buld an XML parser.",
"ILeappFileProcessor_cannotParseXml=Cannot Parse XML file.",
"ILeappFileProcessor.postartifacts_error=Error posting Blackboard Artifact",
"ILeappFileProcessor.error.creating.new.artifacts=Error creating new artifacts."
})
/**
* Read the XML config file and load the mappings into maps
*/
private void loadConfigFile() throws IngestModuleException {
Document xmlinput;
try {
String path = PlatformUtil.getUserConfigDirectory() + File.separator + XMLFILE;
File f = new File(path);
DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
DocumentBuilder db = dbf.newDocumentBuilder();
xmlinput = db.parse(f);
} catch (IOException e) {
throw new IngestModuleException(Bundle.ILeappFileProcessor_cannot_load_artifact_xml() + e.getLocalizedMessage(), e); //NON-NLS
} catch (ParserConfigurationException pce) {
throw new IngestModuleException(Bundle.ILeappFileProcessor_cannotBuildXmlParser() + pce.getLocalizedMessage(), pce); //NON-NLS
} catch (SAXException sxe) {
throw new IngestModuleException(Bundle.ILeappFileProcessor_cannotParseXml() + sxe.getLocalizedMessage(), sxe); //NON-NLS
}
getFileNode(xmlinput);
getArtifactNode(xmlinput);
getAttributeNodes(xmlinput);
}
private void getFileNode(Document xmlinput) {
NodeList nlist = xmlinput.getElementsByTagName("FileName"); //NON-NLS
for (int i = 0; i < nlist.getLength(); i++) {
NamedNodeMap nnm = nlist.item(i).getAttributes();
tsvFiles.put(nnm.getNamedItem("filename").getNodeValue(), nnm.getNamedItem("description").getNodeValue());
}
}
private void getArtifactNode(Document xmlinput) {
NodeList artifactNlist = xmlinput.getElementsByTagName("ArtifactName"); //NON-NLS
for (int k = 0; k < artifactNlist.getLength(); k++) {
NamedNodeMap nnm = artifactNlist.item(k).getAttributes();
String artifactName = nnm.getNamedItem("artifactname").getNodeValue();
String comment = nnm.getNamedItem("comment").getNodeValue();
String parentName = artifactNlist.item(k).getParentNode().getAttributes().getNamedItem("filename").getNodeValue();
tsvFileArtifacts.put(parentName, artifactName);
if (!comment.toLowerCase().matches("null")) {
tsvFileArtifactComments.put(parentName, comment);
}
}
}
private void getAttributeNodes(Document xmlinput) {
NodeList attributeNlist = xmlinput.getElementsByTagName("AttributeName"); //NON-NLS
for (int k = 0; k < attributeNlist.getLength(); k++) {
List<String> attributeList = new ArrayList<>();
NamedNodeMap nnm = attributeNlist.item(k).getAttributes();
String attributeName = nnm.getNamedItem("attributename").getNodeValue();
if (!attributeName.toLowerCase().matches("null")) {
String columnName = nnm.getNamedItem("columnName").getNodeValue();
String required = nnm.getNamedItem("required").getNodeValue();
String parentName = attributeNlist.item(k).getParentNode().getParentNode().getAttributes().getNamedItem("filename").getNodeValue();
attributeList.add(attributeName.toLowerCase());
attributeList.add(columnName.toLowerCase());
attributeList.add(required.toLowerCase());
if (tsvFileAttributes.containsKey(parentName)) {
List<List<String>> attrList = tsvFileAttributes.get(parentName);
attrList.add(attributeList);
tsvFileAttributes.replace(parentName, attrList);
} else {
List<List<String>> attrList = new ArrayList<>();
attrList.add(attributeList);
tsvFileAttributes.put(parentName, attrList);
}
}
}
}
/**
* Generic method for creating a blackboard artifact with attributes
*
* @param type is a blackboard.artifact_type enum to determine
* which type the artifact should be
* @param content is the Content object that needs to have the
* artifact added for it
* @param bbattributes is the collection of blackboard attributes that
* need to be added to the artifact after the
* artifact has been created
*
* @return The newly-created artifact, or null on error
*/
private BlackboardArtifact createArtifactWithAttributes(int type, AbstractFile abstractFile, Collection<BlackboardAttribute> bbattributes) {
try {
BlackboardArtifact bbart = abstractFile.newArtifact(type);
bbart.addAttributes(bbattributes);
return bbart;
} catch (TskException ex) {
logger.log(Level.WARNING, Bundle.ILeappFileProcessor_error_creating_new_artifacts(), ex); //NON-NLS
}
return null;
}
/**
* Method to post a list of BlackboardArtifacts to the blackboard.
*
* @param artifacts A list of artifacts. IF list is empty or null, the
* function will return.
*/
void postArtifacts(Collection<BlackboardArtifact> artifacts) {
if (artifacts == null || artifacts.isEmpty()) {
return;
}
try {
Case.getCurrentCase().getSleuthkitCase().getBlackboard().postArtifacts(artifacts, MODULE_NAME);
} catch (Blackboard.BlackboardException ex) {
logger.log(Level.SEVERE, Bundle.ILeappFileProcessor_postartifacts_error(), ex); //NON-NLS
}
}
/**
* Extract the iLeapp config xml file to the user directory to process
*
* @throws org.sleuthkit.autopsy.ingest.IngestModule.IngestModuleException
*/
private void configExtractor() throws IOException {
PlatformUtil.extractResourceToUserConfigDir(ILeappFileProcessor.class, XMLFILE, true);
}
}

View File

@ -0,0 +1,105 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!---
This file contains the parameters for how to map iLeapp plugin output to attributes inside Autopsy for the Ileapp Analyser module.
Each FileName node corresponds to a tab seperated values (tsv) file that is produced from iLeapp.
A FileName will have an associated TSK artifact assigned to it.\
Each TSK artifact may have multiple attributes that correspond to the columns of the output from the iLeapp program tsv file.
FileName:
filename: The iLeapp TSV file that you want to process.
description: A description of the tsv file name, this is defined in the iLeapp plugin for each tsv file.
ArtifactName:
artifactname: The artifact that is to be created for the data in the tsv file.
comment: This will be the data that will be added to the TSK_COMMENT attribute for each artifact. If the artifact
does not need/require a comment then make the value null, a null comment will be ignored.
AttributeName:
attributeName: The TSK attribute that the data corresponds to in the TSV file. If the data has no corresponding TSK attribute then
make the value null, this will make sure the data in this column is ignored.
columnName: This is the column name that is defined in the tsv file and what the attributeName corresponds to.
required: whether the attribute is required or not (yes or no)
-->
<iLeap_Files_To_Process>
<FileName filename="Safari Browser History.tsv" description="Safari Browser">
<ArtifactName artifactname="TSK_WEB_HISTORY" comment="null">
<AttributeName attributename="TSK_DATETIME_ACCESSED" columnName="Visit Time" required="yes" />
<AttributeName attributename="TSK_URL" columnName="URL" required="yes" />
<AttributeName attributename="null" columnName="Visit Count" required="no" />
<AttributeName attributename="TSK_TITLE" columnName="Title" required="yes" />
<AttributeName attributename="null" columnName="iCloud Sync" required="no" />
<AttributeName attributename="null" columnName="Load Successful" required="no" />
<AttributeName attributename="null" columnName="Visit ID" required="no" />
<AttributeName attributename="TSK_REFERRER" columnName="Redirect Source" required="yes" />
<AttributeName attributename="null" columnName="Redirect Destination" required="no" />
<AttributeName attributename="null" columnName="History Item ID" required="no" />
</ArtifactName>
</FileName>
<FileName filename="Safari Web Search.tsv" description="Safari Browser">
<ArtifactName artifactname="TSK_WEB_SEARCH_QUERY" comment="null">
<AttributeName attributename="TSK_DATETIME_ACCESSED" columnName="Visit Time" required="yes" />
<AttributeName attributename="TSK_TEXT" columnName="Search Term" required="yes" />
<AttributeName attributename="TSK_URL" columnName="URL" required="yes" />
<AttributeName attributename="null" columnName="Visit Count" required="no" />
<AttributeName attributename="TSK_TITLE" columnName="Title" required="yes" />
<AttributeName attributename="null" columnName="iCloud Sync" required="no" />
<AttributeName attributename="null" columnName="Load Successful" required="no" />
<AttributeName attributename="null" columnName="Visit ID" required="no" />
<AttributeName attributename="TSK_REFERRER" columnName="Redirect Source" required="yes" />
<AttributeName attributename="null" columnName="Redirect Destination" required="no" />
<AttributeName attributename="null" columnName="History Item ID" required="no" />
</ArtifactName>
</FileName>
<FileName filename="KnowledgeC Bluetooth.tsv" description="KnowledgeC Bluetooth Connections">
<ArtifactName artifactname="TSK_BLUETOOTH_PAIRING" comment="null">
<AttributeName attributename="TSK_DATETIME_START" columnName="Start" required="yes" />
<AttributeName attributename="TSK_DATETIME_END" columnName="End" required="yes" />
<AttributeName attributename="TSK_MAC_ADDRESS" columnName="Bluetooth Address" required="yes" />
<AttributeName attributename="TSK_DEVICE_NAME" columnName="Bluetooth Name" required="yes" />
<AttributeName attributename="null" columnName="Usage in Seconds" required="no" />
<AttributeName attributename="null" columnName="Usage in Minutes" required="no" />
<AttributeName attributename="null" columnName="Day of the week" required="no" />
<AttributeName attributename="null" columnName="GMT Offset" required="no" />
<AttributeName attributename="null" columnName="Entry Creation" required="no" />
<AttributeName attributename="null" columnName="UUID" required="no" />
<AttributeName attributename="null" columnName="Zobject Table ID" required="no" />
</ArtifactName>
</FileName>
<FileName filename="Bluetooth paired.tsv" description="Bluetooth Paired">
<ArtifactName artifactname="TSK_BLUETOOTH_PAIRING" comment="null">
<AttributeName attributename="TSK_DEVICE_ID" columnName="UUID" required="yes" />
<AttributeName attributename="TSK_NAME" columnName="Name" required="yes" />
<AttributeName attributename="null" columnName="Name Origin" required="no" />
<AttributeName attributename="null" columnName="Address" required="no" />
<AttributeName attributename="null" columnName="Resolved Address" required="no" />
<AttributeName attributename="null" columnName="Last Seen TIme" required="yes" />
<AttributeName attributename="null" columnName="Last Connection Time" required="yes" />
</ArtifactName>
</FileName>
<FileName filename="KnowledgeC App Usage.tsv" description="KnowledgeC App Usage">
<ArtifactName artifactname="TSK_PROG_RUN" comment="KnowledgeC App Usage">
<AttributeName attributename="TSK_DATETIME" columnName="Start" required="yes" />
<AttributeName attributename="null" columnName="End" required="no" />
<AttributeName attributename="TSK_PROG_NAME" columnName="Bundle ID" required="yes" />
<AttributeName attributename="null" columnName="Usage in Seconds" required="no" />
<AttributeName attributename="null" columnName="Usage in Minutes" required="no" />
<AttributeName attributename="null" columnName="Device ID" required="no" />
<AttributeName attributename="null" columnName="Day of the week" required="no" />
<AttributeName attributename="null" columnName="GMT Offset" required="no" />
<AttributeName attributename="null" columnName="Entry Creation" required="no" />
<AttributeName attributename="null" columnName="UUID" required="no" />
<AttributeName attributename="null" columnName="Zobject Table ID" required="no" />
</ArtifactName>
</FileName>
</iLeap_Files_To_Process>

View File

@ -205,9 +205,7 @@ DeleteCaseTask.progress.parsingManifest=Parsing manifest file {0}...
DeleteCaseTask.progress.releasingManifestLock=Releasing lock on the manifest file {0}...
DeleteCaseTask.progress.startMessage=Starting deletion...
DeleteOrphanCaseNodesAction.progressDisplayName=Cleanup Case Znodes
# {0} - item count
DeleteOrphanCaseNodesDialog.additionalInit.lblNodeCount.text=Znodes found: {0}
# {0} - item count
DeleteOrphanCaseNodesDialog.additionalInit.znodesTextArea.countMessage=ZNODES FOUND: {0}
DeleteOrphanCaseNodesTask.progress.connectingToCoordSvc=Connecting to the coordination service
# {0} - node path

View File

@ -36,7 +36,7 @@ KeywordSearchResultFactory.createNodeForKey.noResultsFound.text=No results found
KeywordSearchResultFactory.query.exception.msg=Could not perform the query
OpenIDE-Module-Display-Category=Ingest Module
OpenIDE-Module-Long-Description=Keyword Search ingest module.\n\nThe module indexes files found in the disk image at ingest time.\nIt then periodically runs the search on the indexed files using one or more keyword lists (containing pure words and/or regular expressions) and posts results.\n\n\The module also contains additional tools integrated in the main GUI, such as keyword list configuration, keyword search bar in the top-right corner, extracted text viewer and search results viewer showing highlighted keywords found.
OpenIDE-Module-Long-Description=Keyword Search ingest module.\n\nThe module indexes files found in the disk image at ingest time.\nIt then periodically runs the search on the indexed files using one or more keyword lists (containing pure words and/or regular expressions) and posts results.\n\nThe module also contains additional tools integrated in the main GUI, such as keyword list configuration, keyword search bar in the top-right corner, extracted text viewer and search results viewer showing highlighted keywords found.
OpenIDE-Module-Name=KeywordSearch
OptionsCategory_Name_KeywordSearchOptions=Keyword Search
OptionsCategory_Keywords_KeywordSearchOptions=Keyword Search

View File

@ -1,5 +1,5 @@
#Updated by build script
#Wed, 08 Jul 2020 15:15:46 -0400
#Wed, 09 Sep 2020 10:39:20 -0400
LBL_splash_window_title=Starting Autopsy
SPLASH_HEIGHT=314
SPLASH_WIDTH=538

View File

@ -1,4 +1,4 @@
#Updated by build script
#Wed, 08 Jul 2020 15:15:46 -0400
#Wed, 09 Sep 2020 10:39:20 -0400
CTL_MainWindow_Title=Autopsy 4.16.0
CTL_MainWindow_Title_No_Project=Autopsy 4.16.0

21
thirdparty/iLeapp/LICENSE vendored Normal file
View File

@ -0,0 +1,21 @@
MIT License
Copyright (c) 2020 Brigs
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

BIN
thirdparty/iLeapp/ileapp.exe vendored Normal file

Binary file not shown.