Merge pull request #6626 from gdicristofaro/7217-errorHandling

7217 error handling
This commit is contained in:
Richard Cordovano 2021-01-15 11:33:56 -05:00 committed by GitHub
commit 1ff4c00c54
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

View File

@ -39,10 +39,12 @@ import static java.util.Locale.US;
import java.util.Map;
import java.util.logging.Level;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import java.util.stream.Stream;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import org.apache.commons.collections4.MapUtils;
import org.apache.commons.io.FilenameUtils;
import org.openide.util.NbBundle;
import org.sleuthkit.autopsy.casemodule.Case;
@ -69,6 +71,52 @@ import org.xml.sax.SAXException;
*/
public final class LeappFileProcessor {
/**
* Represents metadata for a particular column in a tsv file.
*/
private static class TsvColumn {
private final String attributeName;
private final String columnName;
private final boolean required;
/**
* Main constructor.
*
* @param attributeName The BlackboardAttribute name or null if not
* used.
* @param columnName The name of the column in the tsv file.
* @param required Whether or not this attribute is required to be
* present.
*/
TsvColumn(String attributeName, String columnName, boolean required) {
this.attributeName = attributeName;
this.columnName = columnName;
this.required = required;
}
/**
* @return The BlackboardAttribute name or null if not used.
*/
String getAttributeName() {
return attributeName;
}
/**
* @return The name of the column in the tsv file.
*/
String getColumnName() {
return columnName;
}
/**
* @return Whether or not this attribute is required to be present.
*/
boolean isRequired() {
return required;
}
}
private static final Logger logger = Logger.getLogger(LeappFileProcessor.class.getName());
private static final String MODULE_NAME = ILeappAnalyzerModuleFactory.getModuleName();
@ -77,7 +125,7 @@ public final class LeappFileProcessor {
private final Map<String, String> tsvFiles;
private final Map<String, String> tsvFileArtifacts;
private final Map<String, String> tsvFileArtifactComments;
private final Map<String, List<List<String>>> tsvFileAttributes;
private final Map<String, List<TsvColumn>> tsvFileAttributes;
Blackboard blkBoard;
@ -104,9 +152,7 @@ public final class LeappFileProcessor {
"LeappFileProcessor.Leapp.cancelled=Leapp run was canceled",
"LeappFileProcessor.completed=Leapp Processing Completed",
"LeappFileProcessor.error.reading.Leapp.directory=Error reading Leapp Output Directory"})
public ProcessResult processFiles(Content dataSource, Path moduleOutputPath, AbstractFile LeappFile) {
try {
List<String> LeappTsvOutputFiles = findTsvFiles(moduleOutputPath);
processLeappFiles(LeappTsvOutputFiles, LeappFile);
@ -123,7 +169,7 @@ public final class LeappFileProcessor {
try {
List<String> LeappTsvOutputFiles = findTsvFiles(moduleOutputPath);
processLeappFiles(LeappTsvOutputFiles, dataSource);
} catch (IOException | IngestModuleException ex) {
} catch (IngestModuleException ex) {
logger.log(Level.SEVERE, String.format("Error trying to process Leapp output files in directory %s. ", moduleOutputPath.toString()), ex); //NON-NLS
return ProcessResult.ERROR;
}
@ -174,7 +220,7 @@ public final class LeappFileProcessor {
String fileName = FilenameUtils.getName(LeappFileName);
File LeappFile = new File(LeappFileName);
if (tsvFileAttributes.containsKey(fileName)) {
List<List<String>> attrList = tsvFileAttributes.get(fileName);
List<TsvColumn> attrList = tsvFileAttributes.get(fileName);
try {
BlackboardArtifact.Type artifactType = Case.getCurrentCase().getSleuthkitCase().getArtifactType(tsvFileArtifacts.get(fileName));
@ -202,21 +248,29 @@ public final class LeappFileProcessor {
* @throws FileNotFoundException
* @throws IOException
*/
private void processLeappFiles(List<String> LeappFilesToProcess, Content dataSource) throws FileNotFoundException, IOException, IngestModuleException {
private void processLeappFiles(List<String> LeappFilesToProcess, Content dataSource) throws IngestModuleException {
List<BlackboardArtifact> bbartifacts = new ArrayList<>();
for (String LeappFileName : LeappFilesToProcess) {
String fileName = FilenameUtils.getName(LeappFileName);
File LeappFile = new File(LeappFileName);
if (tsvFileAttributes.containsKey(fileName)) {
List<List<String>> attrList = tsvFileAttributes.get(fileName);
List<TsvColumn> attrList = tsvFileAttributes.get(fileName);
BlackboardArtifact.Type artifactType = null;
try {
BlackboardArtifact.Type artifactType = Case.getCurrentCase().getSleuthkitCase().getArtifactType(tsvFileArtifacts.get(fileName));
processFile(LeappFile, attrList, fileName, artifactType, bbartifacts, dataSource);
artifactType = Case.getCurrentCase().getSleuthkitCase().getArtifactType(tsvFileArtifacts.get(fileName));
} catch (TskCoreException ex) {
throw new IngestModuleException(String.format("Error getting Blackboard Artifact Type for %s", tsvFileArtifacts.get(fileName)), ex);
logger.log(Level.SEVERE, String.format("Error getting Blackboard Artifact Type for %s", tsvFileArtifacts.get(fileName)), ex);
}
if (artifactType == null) {
continue;
}
try {
processFile(LeappFile, attrList, fileName, artifactType, bbartifacts, dataSource);
} catch (TskCoreException | IOException ex) {
logger.log(Level.SEVERE, String.format("Error processing file at %s", LeappFile.toString()), ex);
}
}
@ -228,26 +282,34 @@ public final class LeappFileProcessor {
}
private void processFile(File LeappFile, List<List<String>> attrList, String fileName, BlackboardArtifact.Type artifactType,
private void processFile(File LeappFile, List<TsvColumn> attrList, String fileName, BlackboardArtifact.Type artifactType,
List<BlackboardArtifact> bbartifacts, Content dataSource) throws FileNotFoundException, IOException, IngestModuleException,
TskCoreException {
if (LeappFile == null || !LeappFile.exists() || fileName == null) {
logger.log(Level.WARNING, String.format("Leap file: %s is null or does not exist", LeappFile == null ? LeappFile.toString() : "<null>"));
return;
} else if (attrList == null || artifactType == null || dataSource == null) {
logger.log(Level.WARNING, String.format("attribute list, artifact type or dataSource not provided for %s", LeappFile == null ? LeappFile.toString() : "<null>"));
return;
}
try (BufferedReader reader = new BufferedReader(new FileReader(LeappFile))) {
String line = reader.readLine();
String header = reader.readLine();
// Check first line, if it is null then no heading so nothing to match to, close and go to next file.
if (line != null) {
Map<Integer, String> columnNumberToProcess = findColumnsToProcess(line, attrList);
line = reader.readLine();
if (header != null) {
Map<Integer, String> columnNumberToProcess = findColumnsToProcess(fileName, header, attrList);
String line = reader.readLine();
while (line != null) {
Collection<BlackboardAttribute> bbattributes = processReadLine(line, columnNumberToProcess, fileName);
if (artifactType == null) {
logger.log(Level.SEVERE, "Error trying to process Leapp output files in directory . "); //NON-NLS
}
if (!bbattributes.isEmpty() && !blkBoard.artifactExists(dataSource, BlackboardArtifact.ARTIFACT_TYPE.fromID(artifactType.getTypeID()), bbattributes)) {
BlackboardArtifact bbartifact = createArtifactWithAttributes(artifactType.getTypeID(), dataSource, bbattributes);
if (bbartifact != null) {
bbartifacts.add(bbartifact);
}
}
line = reader.readLine();
}
}
@ -265,10 +327,16 @@ public final class LeappFileProcessor {
* @return
*/
private Collection<BlackboardAttribute> processReadLine(String line, Map<Integer, String> columnNumberToProcess, String fileName) throws IngestModuleException {
if (MapUtils.isEmpty(columnNumberToProcess)) {
return Collections.emptyList();
} else if (line == null) {
logger.log(Level.WARNING, "Line is null. Returning empty list for attributes.");
return Collections.emptyList();
}
String[] columnValues;
// Check to see if the 2 values are equal, they may not be equal if there is no corresponding data in the line
// Check to see if the 2 values are equal, they may not be equal if there is no corresponding data in the line.
// or if the size of the line to split is not equal to the column numbers we are looking to process. This
// can happen when the last value of the tsv line has no data in it.
// If this happens then adding an empty value(s) for each columnValue where data does not exist
@ -289,7 +357,7 @@ public final class LeappFileProcessor {
try {
BlackboardAttribute.Type attributeType = Case.getCurrentCase().getSleuthkitCase().getAttributeType(attributeName.toUpperCase());
if (attributeType == null) {
break;
continue;
}
String attrType = attributeType.getValueType().getLabel().toUpperCase();
checkAttributeType(bbattributes, attrType, columnValues, columnNumber, attributeType, fileName);
@ -307,34 +375,60 @@ public final class LeappFileProcessor {
}
private void checkAttributeType(Collection<BlackboardAttribute> bbattributes, String attrType, String[] columnValues, Integer columnNumber, BlackboardAttribute.Type attributeType,
private void checkAttributeType(Collection<BlackboardAttribute> bbattributes, String attrType, String[] columnValues, int columnNumber, BlackboardAttribute.Type attributeType,
String fileName) {
if (columnValues == null || columnNumber < 0 || columnNumber > columnValues.length || columnValues[columnNumber] == null) {
logger.log(Level.WARNING, String.format("Unable to determine column value at index %d in columnValues: %s",
columnNumber,
columnValues == null ? "<null>" : "[" + String.join(", ", columnValues) + "]"));
return;
}
String columnValue = columnValues[columnNumber];
if (attrType.matches("STRING")) {
bbattributes.add(new BlackboardAttribute(attributeType, MODULE_NAME, columnValues[columnNumber]));
bbattributes.add(new BlackboardAttribute(attributeType, MODULE_NAME, columnValue));
} else if (attrType.matches("INTEGER")) {
bbattributes.add(new BlackboardAttribute(attributeType, MODULE_NAME, Integer.valueOf(columnValues[columnNumber])));
try {
bbattributes.add(new BlackboardAttribute(attributeType, MODULE_NAME, Integer.valueOf(columnValue)));
} catch (NumberFormatException ex) {
logger.log(Level.WARNING, String.format("Unable to format %s as an integer.", columnValue), ex);
}
} else if (attrType.matches("LONG")) {
bbattributes.add(new BlackboardAttribute(attributeType, MODULE_NAME, Long.valueOf(columnValues[columnNumber])));
try {
bbattributes.add(new BlackboardAttribute(attributeType, MODULE_NAME, Long.valueOf(columnValue)));
} catch (NumberFormatException ex) {
logger.log(Level.WARNING, String.format("Unable to format %s as an long.", columnValue), ex);
}
} else if (attrType.matches("DOUBLE")) {
bbattributes.add(new BlackboardAttribute(attributeType, MODULE_NAME, Double.valueOf(columnValues[columnNumber])));
try {
bbattributes.add(new BlackboardAttribute(attributeType, MODULE_NAME, Double.valueOf(columnValue)));
} catch (NumberFormatException ex) {
logger.log(Level.WARNING, String.format("Unable to format %s as an double.", columnValue), ex);
}
} else if (attrType.matches("BYTE")) {
bbattributes.add(new BlackboardAttribute(attributeType, MODULE_NAME, Byte.valueOf(columnValues[columnNumber])));
try {
bbattributes.add(new BlackboardAttribute(attributeType, MODULE_NAME, Byte.valueOf(columnValue)));
} catch (NumberFormatException ex) {
logger.log(Level.WARNING, String.format("Unable to format %s as an byte.", columnValue), ex);
}
} else if (attrType.matches("DATETIME")) {
// format of data should be the same in all the data and the format is 2020-03-28 01:00:17
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-d HH:mm:ss", US);
Long dateLong = Long.valueOf(0);
try {
Date newDate = dateFormat.parse(columnValues[columnNumber]);
Date newDate = dateFormat.parse(columnValue);
dateLong = newDate.getTime() / 1000;
bbattributes.add(new BlackboardAttribute(attributeType, MODULE_NAME, dateLong));
} catch (ParseException ex) {
// catching error and displaying date that could not be parsed
// we set the timestamp to 0 and continue on processing
logger.log(Level.WARNING, String.format("Failed to parse date/time %s for attribute type %s in file %s.", columnValues[columnNumber], attributeType.getDisplayName(), fileName)); //NON-NLS
logger.log(Level.WARNING, String.format("Failed to parse date/time %s for attribute type %s in file %s.", columnValue, attributeType.getDisplayName(), fileName)); //NON-NLS
}
} else if (attrType.matches("JSON")) {
bbattributes.add(new BlackboardAttribute(attributeType, MODULE_NAME, columnValues[columnNumber]));
bbattributes.add(new BlackboardAttribute(attributeType, MODULE_NAME, columnValue));
} else {
// Log this and continue on with processing
logger.log(Level.WARNING, String.format("Attribute Type %s not defined.", attrType)); //NON-NLS
@ -347,29 +441,43 @@ public final class LeappFileProcessor {
* headings to the columns in the XML mapping file so we know which columns
* to process.
*
* @param fileName The name of the file in which these column headers exist.
* @param line a tsv heading line of the columns in the file
* @param attrList the list of headings we want to process
*
* @return the numbered column(s) and attribute(s) we want to use for the
* column(s)
*/
private Map<Integer, String> findColumnsToProcess(String line, List<List<String>> attrList) {
private Map<Integer, String> findColumnsToProcess(String fileName, String line, List<TsvColumn> attrList) {
String[] columnNames = line.split("\\t");
HashMap<Integer, String> columnsToProcess = new HashMap<>();
Integer columnPosition = 0;
for (String columnName : columnNames) {
// for some reason the first column of the line has unprintable characters so removing them
String cleanColumnName = columnName.replaceAll("[^\\n\\r\\t\\p{Print}]", "");
for (List<String> atList : attrList) {
if (atList.contains(cleanColumnName.toLowerCase())) {
columnsToProcess.put(columnPosition, atList.get(0));
String cleanColumnName = columnName.trim().replaceAll("[^\\n\\r\\t\\p{Print}]", "");
for (TsvColumn tsvColumn : attrList) {
if (cleanColumnName.equalsIgnoreCase(tsvColumn.getColumnName())) {
columnsToProcess.put(columnPosition, tsvColumn.getAttributeName());
break;
}
}
columnPosition++;
}
if (columnsToProcess.size() != attrList.size()) {
String missingColumns = IntStream.range(0, attrList.size())
.filter((idx) -> !columnsToProcess.containsKey(attrList.get(idx).getAttributeName()))
.mapToObj((idx) -> String.format("'%s'", attrList.get(idx).getColumnName() == null ? "<null>" : attrList.get(idx).getColumnName()))
.collect(Collectors.joining(", "));
logger.log(Level.WARNING, String.format("Columns size expected not found in file %s based on xml from %s. Column Keys Missing = [%s]; Header Line = '%s'.",
this.xmlFile == null ? "<null>" : this.xmlFile,
fileName,
missingColumns,
line));
}
return columnsToProcess;
}
@ -428,6 +536,18 @@ public final class LeappFileProcessor {
String comment = nnm.getNamedItem("comment").getNodeValue();
String parentName = artifactNlist.item(k).getParentNode().getAttributes().getNamedItem("filename").getNodeValue();
BlackboardArtifact.Type foundArtifactType = null;
try {
foundArtifactType = Case.getCurrentCase().getSleuthkitCase().getArtifactType(artifactName);
} catch (TskCoreException ex) {
logger.log(Level.SEVERE, String.format("There was an issue that arose while trying to fetch artifact type for %s.", artifactName), ex);
}
if (foundArtifactType == null) {
logger.log(Level.SEVERE, String.format("No known artifact mapping found for [artifact: %s, %s]",
artifactName, getXmlFileIdentifier(parentName)));
}
tsvFileArtifacts.put(parentName, artifactName);
if (!comment.toLowerCase().matches("null")) {
@ -437,29 +557,66 @@ public final class LeappFileProcessor {
}
private String getXmlFileIdentifier(String fileName) {
return String.format("file: %s, filename: %s",
this.xmlFile == null ? "<null>" : this.xmlFile,
fileName == null ? "<null>" : fileName);
}
private String getXmlAttrIdentifier(String fileName, String attributeName) {
return String.format("attribute: %s %s",
attributeName == null ? "<null>" : attributeName,
getXmlFileIdentifier(fileName));
}
private void getAttributeNodes(Document xmlinput) {
NodeList attributeNlist = xmlinput.getElementsByTagName("AttributeName"); //NON-NLS
for (int k = 0; k < attributeNlist.getLength(); k++) {
List<String> attributeList = new ArrayList<>();
NamedNodeMap nnm = attributeNlist.item(k).getAttributes();
String attributeName = nnm.getNamedItem("attributename").getNodeValue();
if (!attributeName.toLowerCase().matches("null")) {
String columnName = nnm.getNamedItem("columnName").getNodeValue();
String required = nnm.getNamedItem("required").getNodeValue();
String parentName = attributeNlist.item(k).getParentNode().getParentNode().getAttributes().getNamedItem("filename").getNodeValue();
attributeList.add(attributeName.toLowerCase());
attributeList.add(columnName.toLowerCase());
attributeList.add(required.toLowerCase());
BlackboardAttribute.Type foundAttrType = null;
try {
foundAttrType = Case.getCurrentCase().getSleuthkitCase().getAttributeType(attributeName.toUpperCase());
} catch (TskCoreException ex) {
logger.log(Level.SEVERE, String.format("There was an issue that arose while trying to fetch attribute type for %s.", attributeName), ex);
}
if (foundAttrType == null) {
logger.log(Level.SEVERE, String.format("No known attribute mapping found for [%s]", getXmlAttrIdentifier(parentName, attributeName)));
}
if (required != null && required.compareToIgnoreCase("yes") != 0 && required.compareToIgnoreCase("no") != 0) {
logger.log(Level.SEVERE, String.format("Required value %s did not match 'yes' or 'no' for [%s]",
required, getXmlAttrIdentifier(parentName, attributeName)));
}
if (columnName == null) {
logger.log(Level.SEVERE, String.format("No column name provided for [%s]", getXmlAttrIdentifier(parentName, attributeName)));
} else if (columnName.trim().length() != columnName.length()) {
logger.log(Level.SEVERE, String.format("Column name '%s' starts or ends with whitespace for [%s]", columnName, getXmlAttrIdentifier(parentName, attributeName)));
} else if (columnName.matches("[^ \\S]")) {
logger.log(Level.SEVERE, String.format("Column name '%s' contains invalid characters [%s]", columnName, getXmlAttrIdentifier(parentName, attributeName)));
}
TsvColumn thisCol = new TsvColumn(
attributeName.toLowerCase(),
columnName.toLowerCase(),
"yes".compareToIgnoreCase(required) == 0);
if (tsvFileAttributes.containsKey(parentName)) {
List<List<String>> attrList = tsvFileAttributes.get(parentName);
attrList.add(attributeList);
List<TsvColumn> attrList = tsvFileAttributes.get(parentName);
attrList.add(thisCol);
tsvFileAttributes.replace(parentName, attrList);
} else {
List<List<String>> attrList = new ArrayList<>();
attrList.add(attributeList);
List<TsvColumn> attrList = new ArrayList<>();
attrList.add(thisCol);
tsvFileAttributes.put(parentName, attrList);
}
}
@ -537,7 +694,8 @@ public final class LeappFileProcessor {
* @throws org.sleuthkit.autopsy.ingest.IngestModule.IngestModuleException
*/
private void configExtractor() throws IOException {
PlatformUtil.extractResourceToUserConfigDir(LeappFileProcessor.class, xmlFile, true);
PlatformUtil.extractResourceToUserConfigDir(LeappFileProcessor.class,
xmlFile, true);
}
}