From dcda7b1a49a6d031dafda8f9c6e022c461093471 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Thu, 21 Jan 2021 14:43:30 -0500 Subject: [PATCH 1/4] remove non printable characters --- .../autopsy/modules/leappanalyzers/LeappFileProcessor.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/LeappFileProcessor.java b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/LeappFileProcessor.java index 1439a20761..d267ce39b9 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/LeappFileProcessor.java +++ b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/LeappFileProcessor.java @@ -458,6 +458,10 @@ public final class LeappFileProcessor { * @return The generated blackboard attribute or null if not determined. */ private BlackboardAttribute parseAttrValue(String value, BlackboardAttribute.Type attrType, String fileName, boolean blankIsNull, boolean zeroIsNull, ParseExceptionFunction valueConverter) { + // remove non-printable characters from tsv input + // https://stackoverflow.com/a/6199346 + value = value.replaceAll("\\p{C}", ""); + if (blankIsNull && StringUtils.isBlank(value)) { return null; } From 3c0acca7e386be6ffe4696994b8f2228a26f2f9a Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Thu, 21 Jan 2021 15:26:38 -0500 Subject: [PATCH 2/4] skip jagged rows --- .../leappanalyzers/LeappFileProcessor.java | 50 ++++++++++++------- 1 file changed, 33 insertions(+), 17 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/LeappFileProcessor.java b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/LeappFileProcessor.java index d267ce39b9..71c6169dce 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/LeappFileProcessor.java +++ b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/LeappFileProcessor.java @@ -137,8 +137,8 @@ public final class LeappFileProcessor { private final Map> tsvFileAttributes; private static final Map CUSTOM_ARTIFACT_MAP = ImmutableMap.builder() - .put("TSK_IP_DHCP", "DHCP Information") - .build(); + .put("TSK_IP_DHCP", "DHCP Information") + .build(); Blackboard blkBoard; @@ -314,9 +314,10 @@ public final class LeappFileProcessor { idx -> idx, (val1, val2) -> val1)); - int lineNum = 1; + int lineNum = 2; while (iterator.hasNext()) { - Collection bbattributes = processReadLine(iterator.next(), columnIndexes, attrList, fileName, lineNum++); + List columnItems = iterator.next(); + Collection bbattributes = processReadLine(columnItems, columnIndexes, attrList, fileName, lineNum); if (!bbattributes.isEmpty()) { BlackboardArtifact bbartifact = createArtifactWithAttributes(artifactType.getTypeID(), dataSource, bbattributes); @@ -324,6 +325,8 @@ public final class LeappFileProcessor { bbartifacts.add(bbartifact); } } + + lineNum++; } } } @@ -334,7 +337,8 @@ public final class LeappFileProcessor { * * @param lineValues List of column values. * @param columnIndexes Mapping of column headers (trimmed; to lower case) - * to column index. + * to column index. All header columns and only all header columns should be + * present. * @param attrList The list of attributes as specified for the schema of * this file. * @param fileName The name of the file being processed. @@ -349,25 +353,37 @@ public final class LeappFileProcessor { if (MapUtils.isEmpty(columnIndexes) || CollectionUtils.isEmpty(lineValues) || (lineValues.size() == 1 && StringUtils.isEmpty(lineValues.get(0)))) { return Collections.emptyList(); + } else if (lineValues.size() != columnIndexes.size()) { + logger.log(Level.WARNING, String.format( + "Row at line number %d in file %s has %d columns when %d were expected based on the header row.", + lineNum, fileName, lineValues.size(), columnIndexes.size())); } List attrsToRet = new ArrayList<>(); for (TsvColumn colAttr : attrList) { if (colAttr.getAttributeType() == null) { + // this handles coluns that are currently ignored. continue; } Integer columnIdx = columnIndexes.get(colAttr.getColumnName()); - String value = (columnIdx == null || columnIdx >= lineValues.size() || columnIdx < 0) ? null : lineValues.get(columnIdx); - if (value == null) { - logger.log(Level.WARNING, String.format("No value found for column %s at line %d in file %s.", colAttr.getColumnName(), lineNum, fileName)); + if (columnIdx == null) { + logger.log(Level.WARNING, String.format("No column mapping found for %s in file %s. Omitting column.", colAttr.getColumnName(), fileName)); continue; } - BlackboardAttribute attr = (value == null) ? null : getAttribute(colAttr.getAttributeType(), value, fileName); - if (attr != null) { - attrsToRet.add(attr); + String value = (columnIdx >= lineValues.size() || columnIdx < 0) ? null : lineValues.get(columnIdx); + if (value == null) { + logger.log(Level.WARNING, String.format("No value found for column %s at line %d in file %s. Omitting row.", colAttr.getColumnName(), lineNum, fileName)); + return Collections.emptyList(); } + + BlackboardAttribute attr = (value == null) ? null : getAttribute(colAttr.getAttributeType(), value, fileName); + if (attr == null) { + logger.log(Level.WARNING, String.format("Blackboard attribute could not be parsed column %s at line %d in file %s. Omitting row.", colAttr.getColumnName(), lineNum, fileName)); + return Collections.emptyList(); + } + attrsToRet.add(attr); } if (tsvFileArtifactComments.containsKey(fileName)) { @@ -707,13 +723,13 @@ public final class LeappFileProcessor { return leappFilesToProcess; } - - /** + + /** * Create custom artifacts that are defined in the xLeapp xml file(s). - * + * */ private void createCustomArtifacts(Blackboard blkBoard) { - + for (Map.Entry customArtifact : CUSTOM_ARTIFACT_MAP.entrySet()) { String artifactName = customArtifact.getKey(); String artifactDescription = customArtifact.getValue(); @@ -722,8 +738,8 @@ public final class LeappFileProcessor { BlackboardArtifact.Type customArtifactType = blkBoard.getOrAddArtifactType(artifactName, artifactDescription); } catch (Blackboard.BlackboardException ex) { logger.log(Level.WARNING, String.format("Failed to create custom artifact type %s.", artifactName), ex); - } - + } + } } } From 0aad3d44feff1d38e95b0b523dc96606623ed3bf Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Thu, 21 Jan 2021 16:14:01 -0500 Subject: [PATCH 3/4] bug fix --- .../autopsy/modules/leappanalyzers/LeappFileProcessor.java | 1 + 1 file changed, 1 insertion(+) diff --git a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/LeappFileProcessor.java b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/LeappFileProcessor.java index 71c6169dce..35fb87e6d9 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/LeappFileProcessor.java +++ b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/LeappFileProcessor.java @@ -357,6 +357,7 @@ public final class LeappFileProcessor { logger.log(Level.WARNING, String.format( "Row at line number %d in file %s has %d columns when %d were expected based on the header row.", lineNum, fileName, lineValues.size(), columnIndexes.size())); + return Collections.emptyList(); } List attrsToRet = new ArrayList<>(); From 5707c3c061cac623e656b6f09a3e6fe745bc7b16 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Thu, 21 Jan 2021 18:25:28 -0500 Subject: [PATCH 4/4] fix misspelling --- .../autopsy/modules/leappanalyzers/LeappFileProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/LeappFileProcessor.java b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/LeappFileProcessor.java index 35fb87e6d9..c6fdb6ae1e 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/LeappFileProcessor.java +++ b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/LeappFileProcessor.java @@ -363,7 +363,7 @@ public final class LeappFileProcessor { List attrsToRet = new ArrayList<>(); for (TsvColumn colAttr : attrList) { if (colAttr.getAttributeType() == null) { - // this handles coluns that are currently ignored. + // this handles columns that are currently ignored. continue; }