diff --git a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/LeappFileProcessor.java b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/LeappFileProcessor.java index 1439a20761..c6fdb6ae1e 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/LeappFileProcessor.java +++ b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/LeappFileProcessor.java @@ -137,8 +137,8 @@ public final class LeappFileProcessor { private final Map> tsvFileAttributes; private static final Map CUSTOM_ARTIFACT_MAP = ImmutableMap.builder() - .put("TSK_IP_DHCP", "DHCP Information") - .build(); + .put("TSK_IP_DHCP", "DHCP Information") + .build(); Blackboard blkBoard; @@ -314,9 +314,10 @@ public final class LeappFileProcessor { idx -> idx, (val1, val2) -> val1)); - int lineNum = 1; + int lineNum = 2; while (iterator.hasNext()) { - Collection bbattributes = processReadLine(iterator.next(), columnIndexes, attrList, fileName, lineNum++); + List columnItems = iterator.next(); + Collection bbattributes = processReadLine(columnItems, columnIndexes, attrList, fileName, lineNum); if (!bbattributes.isEmpty()) { BlackboardArtifact bbartifact = createArtifactWithAttributes(artifactType.getTypeID(), dataSource, bbattributes); @@ -324,6 +325,8 @@ public final class LeappFileProcessor { bbartifacts.add(bbartifact); } } + + lineNum++; } } } @@ -334,7 +337,8 @@ public final class LeappFileProcessor { * * @param lineValues List of column values. * @param columnIndexes Mapping of column headers (trimmed; to lower case) - * to column index. + * to column index. All header columns and only all header columns should be + * present. * @param attrList The list of attributes as specified for the schema of * this file. * @param fileName The name of the file being processed. @@ -349,25 +353,38 @@ public final class LeappFileProcessor { if (MapUtils.isEmpty(columnIndexes) || CollectionUtils.isEmpty(lineValues) || (lineValues.size() == 1 && StringUtils.isEmpty(lineValues.get(0)))) { return Collections.emptyList(); + } else if (lineValues.size() != columnIndexes.size()) { + logger.log(Level.WARNING, String.format( + "Row at line number %d in file %s has %d columns when %d were expected based on the header row.", + lineNum, fileName, lineValues.size(), columnIndexes.size())); + return Collections.emptyList(); } List attrsToRet = new ArrayList<>(); for (TsvColumn colAttr : attrList) { if (colAttr.getAttributeType() == null) { + // this handles columns that are currently ignored. continue; } Integer columnIdx = columnIndexes.get(colAttr.getColumnName()); - String value = (columnIdx == null || columnIdx >= lineValues.size() || columnIdx < 0) ? null : lineValues.get(columnIdx); - if (value == null) { - logger.log(Level.WARNING, String.format("No value found for column %s at line %d in file %s.", colAttr.getColumnName(), lineNum, fileName)); + if (columnIdx == null) { + logger.log(Level.WARNING, String.format("No column mapping found for %s in file %s. Omitting column.", colAttr.getColumnName(), fileName)); continue; } - BlackboardAttribute attr = (value == null) ? null : getAttribute(colAttr.getAttributeType(), value, fileName); - if (attr != null) { - attrsToRet.add(attr); + String value = (columnIdx >= lineValues.size() || columnIdx < 0) ? null : lineValues.get(columnIdx); + if (value == null) { + logger.log(Level.WARNING, String.format("No value found for column %s at line %d in file %s. Omitting row.", colAttr.getColumnName(), lineNum, fileName)); + return Collections.emptyList(); } + + BlackboardAttribute attr = (value == null) ? null : getAttribute(colAttr.getAttributeType(), value, fileName); + if (attr == null) { + logger.log(Level.WARNING, String.format("Blackboard attribute could not be parsed column %s at line %d in file %s. Omitting row.", colAttr.getColumnName(), lineNum, fileName)); + return Collections.emptyList(); + } + attrsToRet.add(attr); } if (tsvFileArtifactComments.containsKey(fileName)) { @@ -458,6 +475,10 @@ public final class LeappFileProcessor { * @return The generated blackboard attribute or null if not determined. */ private BlackboardAttribute parseAttrValue(String value, BlackboardAttribute.Type attrType, String fileName, boolean blankIsNull, boolean zeroIsNull, ParseExceptionFunction valueConverter) { + // remove non-printable characters from tsv input + // https://stackoverflow.com/a/6199346 + value = value.replaceAll("\\p{C}", ""); + if (blankIsNull && StringUtils.isBlank(value)) { return null; } @@ -703,13 +724,13 @@ public final class LeappFileProcessor { return leappFilesToProcess; } - - /** + + /** * Create custom artifacts that are defined in the xLeapp xml file(s). - * + * */ private void createCustomArtifacts(Blackboard blkBoard) { - + for (Map.Entry customArtifact : CUSTOM_ARTIFACT_MAP.entrySet()) { String artifactName = customArtifact.getKey(); String artifactDescription = customArtifact.getValue(); @@ -718,8 +739,8 @@ public final class LeappFileProcessor { BlackboardArtifact.Type customArtifactType = blkBoard.getOrAddArtifactType(artifactName, artifactDescription); } catch (Blackboard.BlackboardException ex) { logger.log(Level.WARNING, String.format("Failed to create custom artifact type %s.", artifactName), ex); - } - + } + } } }