From 17365015f5edf46e1b83a49bcc4f60cee5c917b6 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Wed, 31 May 2023 21:21:14 -0400 Subject: [PATCH 1/6] updates to LeappFileProcessor --- .../leappanalyzers/LeappFileProcessor.java | 29 +++++++++++-------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/LeappFileProcessor.java b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/LeappFileProcessor.java index bda9eb747a..77ea2a0c2b 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/LeappFileProcessor.java +++ b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/LeappFileProcessor.java @@ -220,6 +220,10 @@ public final class LeappFileProcessor { loadConfigFile(); } + + private static String normalizeKey(String origKey) { + return StringUtils.defaultString(origKey).trim().toLowerCase(); + } @NbBundle.Messages({ "LeappFileProcessor.error.running.Leapp=Error running Leapp, see log file.", @@ -280,7 +284,7 @@ public final class LeappFileProcessor { .filter(f -> f.toLowerCase().endsWith(".tsv")).collect(Collectors.toList()); for (String tsvFile : allTsvFiles) { - if (tsvFiles.containsKey(FilenameUtils.getName(tsvFile.toLowerCase()))) { + if (tsvFiles.containsKey(normalizeKey(FilenameUtils.getName(tsvFile)))) { foundTsvFiles.add(tsvFile); } } @@ -329,9 +333,10 @@ public final class LeappFileProcessor { progress.progress(Bundle.LeappFileProcessor_tsvProcessed(fileName), i); File LeappFile = new File(LeappFileName); - if (tsvFileAttributes.containsKey(fileName)) { - List attrList = tsvFileAttributes.get(fileName); - BlackboardArtifact.Type artifactType = tsvFileArtifacts.get(fileName); + String fileKey = fileName.toLowerCase().trim(); + if (tsvFileAttributes.containsKey(normalizeKey(fileKey))) { + List attrList = tsvFileAttributes.get(normalizeKey(fileKey)); + BlackboardArtifact.Type artifactType = tsvFileArtifacts.get(normalizeKey(fileKey)); try { processFile(LeappFile, attrList, fileName, artifactType, dataSource); @@ -940,8 +945,8 @@ public final class LeappFileProcessor { attrsToRet.add(attr); } - if (tsvFileArtifactComments.containsKey(fileName)) { - attrsToRet.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_COMMENT, moduleName, tsvFileArtifactComments.get(fileName))); + if (tsvFileArtifactComments.containsKey(normalizeKey(fileName))) { + attrsToRet.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_COMMENT, moduleName, tsvFileArtifactComments.get(normalizeKey(fileName)))); } return attrsToRet; @@ -1121,7 +1126,7 @@ public final class LeappFileProcessor { for (int i = 0; i < nlist.getLength(); i++) { NamedNodeMap nnm = nlist.item(i).getAttributes(); - tsvFiles.put(nnm.getNamedItem("filename").getNodeValue().toLowerCase(), nnm.getNamedItem("description").getNodeValue()); + tsvFiles.put(normalizeKey(nnm.getNamedItem("filename").getNodeValue()), nnm.getNamedItem("description").getNodeValue()); } @@ -1147,11 +1152,11 @@ public final class LeappFileProcessor { logger.log(Level.SEVERE, String.format("No known artifact mapping found for [artifact: %s, %s]", artifactName, getXmlFileIdentifier(parentName))); } else { - tsvFileArtifacts.put(parentName, foundArtifactType); + tsvFileArtifacts.put(normalizeKey(parentName), foundArtifactType); } if (!comment.toLowerCase().matches("null")) { - tsvFileArtifactComments.put(parentName, comment); + tsvFileArtifactComments.put(normalizeKey(parentName), comment); } } @@ -1213,14 +1218,14 @@ public final class LeappFileProcessor { columnName.trim().toLowerCase(), "yes".compareToIgnoreCase(required) == 0); - if (tsvFileAttributes.containsKey(parentName)) { - List attrList = tsvFileAttributes.get(parentName); + if (tsvFileAttributes.containsKey(normalizeKey(parentName))) { + List attrList = tsvFileAttributes.get(normalizeKey(parentName)); attrList.add(thisCol); tsvFileAttributes.replace(parentName, attrList); } else { List attrList = new ArrayList<>(); attrList.add(thisCol); - tsvFileAttributes.put(parentName, attrList); + tsvFileAttributes.put(normalizeKey(parentName), attrList); } } From 83c30d5640e3ef74aabf2eb21b7de80c274c3499 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Sat, 10 Jun 2023 20:33:50 -0400 Subject: [PATCH 2/6] parser flexibility --- .../leappanalyzers/LeappFileProcessor.java | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/LeappFileProcessor.java b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/LeappFileProcessor.java index 393b90c3bb..f00607433e 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/LeappFileProcessor.java +++ b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/LeappFileProcessor.java @@ -221,6 +221,13 @@ public final class LeappFileProcessor { } + /** + * Generates a key trimmed and case-insensitive that can be used for a + * case-insensitive lookup in a map. + * + * @param origKey The original key. + * @return The normalized key. + */ private static String normalizeKey(String origKey) { return StringUtils.defaultString(origKey).trim().toLowerCase(); } @@ -909,12 +916,13 @@ public final class LeappFileProcessor { if (MapUtils.isEmpty(columnIndexes) || CollectionUtils.isEmpty(lineValues) || (lineValues.size() == 1 && StringUtils.isEmpty(lineValues.get(0)))) { return Collections.emptyList(); - } else if (lineValues.size() != columnIndexes.size()) { - logger.log(Level.WARNING, String.format( - "Row at line number %d in file %s has %d columns when %d were expected based on the header row.", - lineNum, fileName, lineValues.size(), columnIndexes.size())); - return Collections.emptyList(); - } + } +// else if (lineValues.size() < columnIndexes.size()) { +// logger.log(Level.WARNING, String.format( +// "Row at line number %d in file %s has %d columns when %d were expected based on the header row.", +// lineNum, fileName, lineValues.size(), columnIndexes.size())); +// return Collections.emptyList(); +// } List attrsToRet = new ArrayList<>(); for (TsvColumn colAttr : attrList) { @@ -938,11 +946,12 @@ public final class LeappFileProcessor { String formattedValue = formatValueBasedOnAttrType(colAttr, value); BlackboardAttribute attr = getAttribute(colAttr.getAttributeType(), formattedValue, fileName); - if (attr == null) { + if (attr != null) { + attrsToRet.add(attr); + } else if (colAttr.isRequired()) { logger.log(Level.WARNING, String.format("Blackboard attribute could not be parsed column %s at line %d in file %s. Omitting row.", colAttr.getColumnName(), lineNum, fileName)); return Collections.emptyList(); } - attrsToRet.add(attr); } if (tsvFileArtifactComments.containsKey(normalizeKey(fileName))) { From c30c8aefef71d7274fd4ce9719eb9ddd1408d0ad Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Sat, 10 Jun 2023 20:44:47 -0400 Subject: [PATCH 3/6] updates to xml --- .../aleapp-artifact-attribute-reference.xml | 26 ++++++- .../ileapp-artifact-attribute-reference.xml | 73 ++++++++++++++++++- 2 files changed, 95 insertions(+), 4 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/aleapp-artifact-attribute-reference.xml b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/aleapp-artifact-attribute-reference.xml index 9b2a6d9cfa..329568c765 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/aleapp-artifact-attribute-reference.xml +++ b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/aleapp-artifact-attribute-reference.xml @@ -36,6 +36,14 @@ + + + + + + + + @@ -54,6 +62,14 @@ + + + + + + + + @@ -173,7 +189,7 @@ - + @@ -246,7 +262,7 @@ - + @@ -329,6 +345,12 @@ + + + + + + diff --git a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/ileapp-artifact-attribute-reference.xml b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/ileapp-artifact-attribute-reference.xml index e2091ac2aa..af4b229b31 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/ileapp-artifact-attribute-reference.xml +++ b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/ileapp-artifact-attribute-reference.xml @@ -65,6 +65,15 @@ + + + + + + + + + @@ -77,6 +86,18 @@ + + + + + + + + + + + + @@ -712,8 +733,24 @@ + + + + + + + + + + + + + + + + - + @@ -744,7 +781,7 @@ - + @@ -779,4 +816,36 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 0bc7606ff4ec301ec4390cc639cc1a866b83d5ae Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Sun, 11 Jun 2023 15:07:06 -0400 Subject: [PATCH 4/6] updates for ileapp discrepancies --- .../leappanalyzers/LeappFileProcessor.java | 21 ++-- .../ileapp-artifact-attribute-reference.xml | 102 ++++++++---------- 2 files changed, 54 insertions(+), 69 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/LeappFileProcessor.java b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/LeappFileProcessor.java index f00607433e..3fab3e483c 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/LeappFileProcessor.java +++ b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/LeappFileProcessor.java @@ -913,19 +913,15 @@ public final class LeappFileProcessor { private Collection processReadLine(List lineValues, Map columnIndexes, List attrList, String fileName, int lineNum) throws IngestModuleException { + // if no attributes, return an empty row if (MapUtils.isEmpty(columnIndexes) || CollectionUtils.isEmpty(lineValues) || (lineValues.size() == 1 && StringUtils.isEmpty(lineValues.get(0)))) { return Collections.emptyList(); - } -// else if (lineValues.size() < columnIndexes.size()) { -// logger.log(Level.WARNING, String.format( -// "Row at line number %d in file %s has %d columns when %d were expected based on the header row.", -// lineNum, fileName, lineValues.size(), columnIndexes.size())); -// return Collections.emptyList(); -// } + } List attrsToRet = new ArrayList<>(); for (TsvColumn colAttr : attrList) { + // if no matching attribute type, keep going if (colAttr.getAttributeType() == null) { // this handles columns that are currently ignored. continue; @@ -939,8 +935,15 @@ public final class LeappFileProcessor { String value = (columnIdx >= lineValues.size() || columnIdx < 0) ? null : lineValues.get(columnIdx); if (value == null) { - logger.log(Level.WARNING, String.format("No value found for column %s at line %d in file %s. Omitting row.", colAttr.getColumnName(), lineNum, fileName)); - return Collections.emptyList(); + // if column is required, return empty for this row if no value + if (colAttr.isRequired()) { + logger.log(Level.WARNING, String.format("No value found for required column %s at line %d in file %s. Omitting row.", colAttr.getColumnName(), lineNum, fileName)); + return Collections.emptyList(); + } else { + // otherwise, continue to next column + logger.log(Level.WARNING, String.format("No value found for column %s at line %d in file %s. Omitting column.", colAttr.getColumnName(), lineNum, fileName)); + continue; + } } String formattedValue = formatValueBasedOnAttrType(colAttr, value); diff --git a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/ileapp-artifact-attribute-reference.xml b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/ileapp-artifact-attribute-reference.xml index af4b229b31..44ad9d3d37 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/ileapp-artifact-attribute-reference.xml +++ b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/ileapp-artifact-attribute-reference.xml @@ -65,24 +65,14 @@ - - - - - - - - - - - - - - - + + + + + @@ -93,8 +83,7 @@ - - + @@ -113,7 +102,8 @@ - + + @@ -736,32 +726,21 @@ - + - - - - - - - - - - + - + + - - - - - - - + + + + @@ -783,20 +762,26 @@ - - - + + - - - - - - - - - + + + + + + + + + + + + + + + + @@ -834,17 +819,14 @@ - - - - - - - - - - - + + + + + + + + From 5526b254b75ac19a65318d841feba99ad6a35f97 Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Sun, 11 Jun 2023 20:25:41 -0400 Subject: [PATCH 5/6] aleapp updates --- .../aleapp-artifact-attribute-reference.xml | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/aleapp-artifact-attribute-reference.xml b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/aleapp-artifact-attribute-reference.xml index 329568c765..88359a0b4c 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/aleapp-artifact-attribute-reference.xml +++ b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/aleapp-artifact-attribute-reference.xml @@ -201,11 +201,12 @@ - + - + + @@ -252,12 +253,12 @@ - + - - + + @@ -333,7 +334,7 @@ - + @@ -353,11 +354,13 @@ - + + + From 3384520a6f8da529c244a17d927bfdfa9806dd4b Mon Sep 17 00:00:00 2001 From: Greg DiCristofaro Date: Sun, 11 Jun 2023 22:20:37 -0400 Subject: [PATCH 6/6] fixes for ileapp parsing --- .../ileapp-artifact-attribute-reference.xml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/ileapp-artifact-attribute-reference.xml b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/ileapp-artifact-attribute-reference.xml index 44ad9d3d37..3580e29061 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/ileapp-artifact-attribute-reference.xml +++ b/Core/src/org/sleuthkit/autopsy/modules/leappanalyzers/ileapp-artifact-attribute-reference.xml @@ -67,8 +67,8 @@ - - + + @@ -766,10 +766,10 @@ - + - +