diff --git a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/LeappFileProcessor.java b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/LeappFileProcessor.java index b73b127284..f4e24aca39 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/LeappFileProcessor.java +++ b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/LeappFileProcessor.java @@ -39,10 +39,12 @@ import static java.util.Locale.US; import java.util.Map; import java.util.logging.Level; import java.util.stream.Collectors; +import java.util.stream.IntStream; import java.util.stream.Stream; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; +import org.apache.commons.collections4.MapUtils; import org.apache.commons.io.FilenameUtils; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.casemodule.Case; @@ -69,6 +71,52 @@ import org.xml.sax.SAXException; */ public final class LeappFileProcessor { + /** + * Represents metadata for a particular column in a tsv file. + */ + private static class TsvColumn { + + private final String attributeName; + private final String columnName; + private final boolean required; + + /** + * Main constructor. + * + * @param attributeName The BlackboardAttribute name or null if not + * used. + * @param columnName The name of the column in the tsv file. + * @param required Whether or not this attribute is required to be + * present. + */ + TsvColumn(String attributeName, String columnName, boolean required) { + this.attributeName = attributeName; + this.columnName = columnName; + this.required = required; + } + + /** + * @return The BlackboardAttribute name or null if not used. + */ + String getAttributeName() { + return attributeName; + } + + /** + * @return The name of the column in the tsv file. + */ + String getColumnName() { + return columnName; + } + + /** + * @return Whether or not this attribute is required to be present. + */ + boolean isRequired() { + return required; + } + } + private static final Logger logger = Logger.getLogger(LeappFileProcessor.class.getName()); private static final String MODULE_NAME = ILeappAnalyzerModuleFactory.getModuleName(); @@ -77,7 +125,7 @@ public final class LeappFileProcessor { private final Map tsvFiles; private final Map tsvFileArtifacts; private final Map tsvFileArtifactComments; - private final Map>> tsvFileAttributes; + private final Map> tsvFileAttributes; Blackboard blkBoard; @@ -104,9 +152,7 @@ public final class LeappFileProcessor { "LeappFileProcessor.Leapp.cancelled=Leapp run was canceled", "LeappFileProcessor.completed=Leapp Processing Completed", "LeappFileProcessor.error.reading.Leapp.directory=Error reading Leapp Output Directory"}) - public ProcessResult processFiles(Content dataSource, Path moduleOutputPath, AbstractFile LeappFile) { - try { List LeappTsvOutputFiles = findTsvFiles(moduleOutputPath); processLeappFiles(LeappTsvOutputFiles, LeappFile); @@ -123,7 +169,7 @@ public final class LeappFileProcessor { try { List LeappTsvOutputFiles = findTsvFiles(moduleOutputPath); processLeappFiles(LeappTsvOutputFiles, dataSource); - } catch (IOException | IngestModuleException ex) { + } catch (IngestModuleException ex) { logger.log(Level.SEVERE, String.format("Error trying to process Leapp output files in directory %s. ", moduleOutputPath.toString()), ex); //NON-NLS return ProcessResult.ERROR; } @@ -174,7 +220,7 @@ public final class LeappFileProcessor { String fileName = FilenameUtils.getName(LeappFileName); File LeappFile = new File(LeappFileName); if (tsvFileAttributes.containsKey(fileName)) { - List> attrList = tsvFileAttributes.get(fileName); + List attrList = tsvFileAttributes.get(fileName); try { BlackboardArtifact.Type artifactType = Case.getCurrentCase().getSleuthkitCase().getArtifactType(tsvFileArtifacts.get(fileName)); @@ -202,21 +248,29 @@ public final class LeappFileProcessor { * @throws FileNotFoundException * @throws IOException */ - private void processLeappFiles(List LeappFilesToProcess, Content dataSource) throws FileNotFoundException, IOException, IngestModuleException { + private void processLeappFiles(List LeappFilesToProcess, Content dataSource) throws IngestModuleException { List bbartifacts = new ArrayList<>(); for (String LeappFileName : LeappFilesToProcess) { String fileName = FilenameUtils.getName(LeappFileName); File LeappFile = new File(LeappFileName); if (tsvFileAttributes.containsKey(fileName)) { - List> attrList = tsvFileAttributes.get(fileName); + List attrList = tsvFileAttributes.get(fileName); + BlackboardArtifact.Type artifactType = null; try { - BlackboardArtifact.Type artifactType = Case.getCurrentCase().getSleuthkitCase().getArtifactType(tsvFileArtifacts.get(fileName)); - - processFile(LeappFile, attrList, fileName, artifactType, bbartifacts, dataSource); - + artifactType = Case.getCurrentCase().getSleuthkitCase().getArtifactType(tsvFileArtifacts.get(fileName)); } catch (TskCoreException ex) { - throw new IngestModuleException(String.format("Error getting Blackboard Artifact Type for %s", tsvFileArtifacts.get(fileName)), ex); + logger.log(Level.SEVERE, String.format("Error getting Blackboard Artifact Type for %s", tsvFileArtifacts.get(fileName)), ex); + } + + if (artifactType == null) { + continue; + } + + try { + processFile(LeappFile, attrList, fileName, artifactType, bbartifacts, dataSource); + } catch (TskCoreException | IOException ex) { + logger.log(Level.SEVERE, String.format("Error processing file at %s", LeappFile.toString()), ex); } } @@ -228,26 +282,34 @@ public final class LeappFileProcessor { } - private void processFile(File LeappFile, List> attrList, String fileName, BlackboardArtifact.Type artifactType, + private void processFile(File LeappFile, List attrList, String fileName, BlackboardArtifact.Type artifactType, List bbartifacts, Content dataSource) throws FileNotFoundException, IOException, IngestModuleException, TskCoreException { + + if (LeappFile == null || !LeappFile.exists() || fileName == null) { + logger.log(Level.WARNING, String.format("Leap file: %s is null or does not exist", LeappFile == null ? LeappFile.toString() : "")); + return; + } else if (attrList == null || artifactType == null || dataSource == null) { + logger.log(Level.WARNING, String.format("attribute list, artifact type or dataSource not provided for %s", LeappFile == null ? LeappFile.toString() : "")); + return; + } + try (BufferedReader reader = new BufferedReader(new FileReader(LeappFile))) { - String line = reader.readLine(); + String header = reader.readLine(); // Check first line, if it is null then no heading so nothing to match to, close and go to next file. - if (line != null) { - Map columnNumberToProcess = findColumnsToProcess(line, attrList); - line = reader.readLine(); + if (header != null) { + Map columnNumberToProcess = findColumnsToProcess(fileName, header, attrList); + String line = reader.readLine(); while (line != null) { Collection bbattributes = processReadLine(line, columnNumberToProcess, fileName); - if (artifactType == null) { - logger.log(Level.SEVERE, "Error trying to process Leapp output files in directory . "); //NON-NLS - } + if (!bbattributes.isEmpty() && !blkBoard.artifactExists(dataSource, BlackboardArtifact.ARTIFACT_TYPE.fromID(artifactType.getTypeID()), bbattributes)) { BlackboardArtifact bbartifact = createArtifactWithAttributes(artifactType.getTypeID(), dataSource, bbattributes); if (bbartifact != null) { bbartifacts.add(bbartifact); } } + line = reader.readLine(); } } @@ -265,10 +327,16 @@ public final class LeappFileProcessor { * @return */ private Collection processReadLine(String line, Map columnNumberToProcess, String fileName) throws IngestModuleException { + if (MapUtils.isEmpty(columnNumberToProcess)) { + return Collections.emptyList(); + } else if (line == null) { + logger.log(Level.WARNING, "Line is null. Returning empty list for attributes."); + return Collections.emptyList(); + } String[] columnValues; - // Check to see if the 2 values are equal, they may not be equal if there is no corresponding data in the line + // Check to see if the 2 values are equal, they may not be equal if there is no corresponding data in the line. // or if the size of the line to split is not equal to the column numbers we are looking to process. This // can happen when the last value of the tsv line has no data in it. // If this happens then adding an empty value(s) for each columnValue where data does not exist @@ -289,7 +357,7 @@ public final class LeappFileProcessor { try { BlackboardAttribute.Type attributeType = Case.getCurrentCase().getSleuthkitCase().getAttributeType(attributeName.toUpperCase()); if (attributeType == null) { - break; + continue; } String attrType = attributeType.getValueType().getLabel().toUpperCase(); checkAttributeType(bbattributes, attrType, columnValues, columnNumber, attributeType, fileName); @@ -307,34 +375,60 @@ public final class LeappFileProcessor { } - private void checkAttributeType(Collection bbattributes, String attrType, String[] columnValues, Integer columnNumber, BlackboardAttribute.Type attributeType, + private void checkAttributeType(Collection bbattributes, String attrType, String[] columnValues, int columnNumber, BlackboardAttribute.Type attributeType, String fileName) { + + if (columnValues == null || columnNumber < 0 || columnNumber > columnValues.length || columnValues[columnNumber] == null) { + logger.log(Level.WARNING, String.format("Unable to determine column value at index %d in columnValues: %s", + columnNumber, + columnValues == null ? "" : "[" + String.join(", ", columnValues) + "]")); + return; + } + + String columnValue = columnValues[columnNumber]; + if (attrType.matches("STRING")) { - bbattributes.add(new BlackboardAttribute(attributeType, MODULE_NAME, columnValues[columnNumber])); + bbattributes.add(new BlackboardAttribute(attributeType, MODULE_NAME, columnValue)); } else if (attrType.matches("INTEGER")) { - bbattributes.add(new BlackboardAttribute(attributeType, MODULE_NAME, Integer.valueOf(columnValues[columnNumber]))); + try { + bbattributes.add(new BlackboardAttribute(attributeType, MODULE_NAME, Integer.valueOf(columnValue))); + } catch (NumberFormatException ex) { + logger.log(Level.WARNING, String.format("Unable to format %s as an integer.", columnValue), ex); + } } else if (attrType.matches("LONG")) { - bbattributes.add(new BlackboardAttribute(attributeType, MODULE_NAME, Long.valueOf(columnValues[columnNumber]))); + try { + bbattributes.add(new BlackboardAttribute(attributeType, MODULE_NAME, Long.valueOf(columnValue))); + } catch (NumberFormatException ex) { + logger.log(Level.WARNING, String.format("Unable to format %s as an long.", columnValue), ex); + } } else if (attrType.matches("DOUBLE")) { - bbattributes.add(new BlackboardAttribute(attributeType, MODULE_NAME, Double.valueOf(columnValues[columnNumber]))); + try { + bbattributes.add(new BlackboardAttribute(attributeType, MODULE_NAME, Double.valueOf(columnValue))); + } catch (NumberFormatException ex) { + logger.log(Level.WARNING, String.format("Unable to format %s as an double.", columnValue), ex); + } } else if (attrType.matches("BYTE")) { - bbattributes.add(new BlackboardAttribute(attributeType, MODULE_NAME, Byte.valueOf(columnValues[columnNumber]))); + try { + bbattributes.add(new BlackboardAttribute(attributeType, MODULE_NAME, Byte.valueOf(columnValue))); + } catch (NumberFormatException ex) { + logger.log(Level.WARNING, String.format("Unable to format %s as an byte.", columnValue), ex); + } } else if (attrType.matches("DATETIME")) { // format of data should be the same in all the data and the format is 2020-03-28 01:00:17 SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-d HH:mm:ss", US); Long dateLong = Long.valueOf(0); try { - Date newDate = dateFormat.parse(columnValues[columnNumber]); + Date newDate = dateFormat.parse(columnValue); dateLong = newDate.getTime() / 1000; bbattributes.add(new BlackboardAttribute(attributeType, MODULE_NAME, dateLong)); } catch (ParseException ex) { // catching error and displaying date that could not be parsed // we set the timestamp to 0 and continue on processing - logger.log(Level.WARNING, String.format("Failed to parse date/time %s for attribute type %s in file %s.", columnValues[columnNumber], attributeType.getDisplayName(), fileName)); //NON-NLS + logger.log(Level.WARNING, String.format("Failed to parse date/time %s for attribute type %s in file %s.", columnValue, attributeType.getDisplayName(), fileName)); //NON-NLS } } else if (attrType.matches("JSON")) { - bbattributes.add(new BlackboardAttribute(attributeType, MODULE_NAME, columnValues[columnNumber])); + bbattributes.add(new BlackboardAttribute(attributeType, MODULE_NAME, columnValue)); } else { // Log this and continue on with processing logger.log(Level.WARNING, String.format("Attribute Type %s not defined.", attrType)); //NON-NLS @@ -347,29 +441,43 @@ public final class LeappFileProcessor { * headings to the columns in the XML mapping file so we know which columns * to process. * + * @param fileName The name of the file in which these column headers exist. * @param line a tsv heading line of the columns in the file * @param attrList the list of headings we want to process * * @return the numbered column(s) and attribute(s) we want to use for the * column(s) */ - private Map findColumnsToProcess(String line, List> attrList) { + private Map findColumnsToProcess(String fileName, String line, List attrList) { String[] columnNames = line.split("\\t"); HashMap columnsToProcess = new HashMap<>(); Integer columnPosition = 0; for (String columnName : columnNames) { // for some reason the first column of the line has unprintable characters so removing them - String cleanColumnName = columnName.replaceAll("[^\\n\\r\\t\\p{Print}]", ""); - for (List atList : attrList) { - if (atList.contains(cleanColumnName.toLowerCase())) { - columnsToProcess.put(columnPosition, atList.get(0)); + String cleanColumnName = columnName.trim().replaceAll("[^\\n\\r\\t\\p{Print}]", ""); + for (TsvColumn tsvColumn : attrList) { + if (cleanColumnName.equalsIgnoreCase(tsvColumn.getColumnName())) { + columnsToProcess.put(columnPosition, tsvColumn.getAttributeName()); break; } } columnPosition++; } + if (columnsToProcess.size() != attrList.size()) { + String missingColumns = IntStream.range(0, attrList.size()) + .filter((idx) -> !columnsToProcess.containsKey(attrList.get(idx).getAttributeName())) + .mapToObj((idx) -> String.format("'%s'", attrList.get(idx).getColumnName() == null ? "" : attrList.get(idx).getColumnName())) + .collect(Collectors.joining(", ")); + + logger.log(Level.WARNING, String.format("Columns size expected not found in file %s based on xml from %s. Column Keys Missing = [%s]; Header Line = '%s'.", + this.xmlFile == null ? "" : this.xmlFile, + fileName, + missingColumns, + line)); + } + return columnsToProcess; } @@ -428,6 +536,18 @@ public final class LeappFileProcessor { String comment = nnm.getNamedItem("comment").getNodeValue(); String parentName = artifactNlist.item(k).getParentNode().getAttributes().getNamedItem("filename").getNodeValue(); + BlackboardArtifact.Type foundArtifactType = null; + try { + foundArtifactType = Case.getCurrentCase().getSleuthkitCase().getArtifactType(artifactName); + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, String.format("There was an issue that arose while trying to fetch artifact type for %s.", artifactName), ex); + } + + if (foundArtifactType == null) { + logger.log(Level.SEVERE, String.format("No known artifact mapping found for [artifact: %s, %s]", + artifactName, getXmlFileIdentifier(parentName))); + } + tsvFileArtifacts.put(parentName, artifactName); if (!comment.toLowerCase().matches("null")) { @@ -437,29 +557,66 @@ public final class LeappFileProcessor { } + private String getXmlFileIdentifier(String fileName) { + return String.format("file: %s, filename: %s", + this.xmlFile == null ? "" : this.xmlFile, + fileName == null ? "" : fileName); + } + + private String getXmlAttrIdentifier(String fileName, String attributeName) { + return String.format("attribute: %s %s", + attributeName == null ? "" : attributeName, + getXmlFileIdentifier(fileName)); + } + private void getAttributeNodes(Document xmlinput) { NodeList attributeNlist = xmlinput.getElementsByTagName("AttributeName"); //NON-NLS for (int k = 0; k < attributeNlist.getLength(); k++) { - List attributeList = new ArrayList<>(); NamedNodeMap nnm = attributeNlist.item(k).getAttributes(); String attributeName = nnm.getNamedItem("attributename").getNodeValue(); + if (!attributeName.toLowerCase().matches("null")) { String columnName = nnm.getNamedItem("columnName").getNodeValue(); String required = nnm.getNamedItem("required").getNodeValue(); String parentName = attributeNlist.item(k).getParentNode().getParentNode().getAttributes().getNamedItem("filename").getNodeValue(); - attributeList.add(attributeName.toLowerCase()); - attributeList.add(columnName.toLowerCase()); - attributeList.add(required.toLowerCase()); + BlackboardAttribute.Type foundAttrType = null; + try { + foundAttrType = Case.getCurrentCase().getSleuthkitCase().getAttributeType(attributeName.toUpperCase()); + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, String.format("There was an issue that arose while trying to fetch attribute type for %s.", attributeName), ex); + } + + if (foundAttrType == null) { + logger.log(Level.SEVERE, String.format("No known attribute mapping found for [%s]", getXmlAttrIdentifier(parentName, attributeName))); + } + + if (required != null && required.compareToIgnoreCase("yes") != 0 && required.compareToIgnoreCase("no") != 0) { + logger.log(Level.SEVERE, String.format("Required value %s did not match 'yes' or 'no' for [%s]", + required, getXmlAttrIdentifier(parentName, attributeName))); + } + + if (columnName == null) { + logger.log(Level.SEVERE, String.format("No column name provided for [%s]", getXmlAttrIdentifier(parentName, attributeName))); + } else if (columnName.trim().length() != columnName.length()) { + logger.log(Level.SEVERE, String.format("Column name '%s' starts or ends with whitespace for [%s]", columnName, getXmlAttrIdentifier(parentName, attributeName))); + } else if (columnName.matches("[^ \\S]")) { + logger.log(Level.SEVERE, String.format("Column name '%s' contains invalid characters [%s]", columnName, getXmlAttrIdentifier(parentName, attributeName))); + } + + TsvColumn thisCol = new TsvColumn( + attributeName.toLowerCase(), + columnName.toLowerCase(), + "yes".compareToIgnoreCase(required) == 0); if (tsvFileAttributes.containsKey(parentName)) { - List> attrList = tsvFileAttributes.get(parentName); - attrList.add(attributeList); + List attrList = tsvFileAttributes.get(parentName); + attrList.add(thisCol); tsvFileAttributes.replace(parentName, attrList); } else { - List> attrList = new ArrayList<>(); - attrList.add(attributeList); + List attrList = new ArrayList<>(); + attrList.add(thisCol); tsvFileAttributes.put(parentName, attrList); } } @@ -537,7 +694,8 @@ public final class LeappFileProcessor { * @throws org.sleuthkit.autopsy.ingest.IngestModule.IngestModuleException */ private void configExtractor() throws IOException { - PlatformUtil.extractResourceToUserConfigDir(LeappFileProcessor.class, xmlFile, true); + PlatformUtil.extractResourceToUserConfigDir(LeappFileProcessor.class, + xmlFile, true); } }