From 81c5db99713ca84a210f13b573daaee8063e7a26 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Fri, 1 Nov 2019 17:07:10 -0400 Subject: [PATCH] Implemented and tested xry report reading --- .../autopsy/xryparser/XRYRecordParser.java | 34 +++ .../autopsy/xryparser/XRYReport.java | 131 ++++++++++ .../autopsy/xryparser/XRYReportExtractor.java | 105 ++++++++ .../xryparser/XRYReportExtractorTest.java | 228 ++++++++++++++++++ .../autopsy/xryparser/XRYReportTest.java | 112 +++++++++ 5 files changed, 610 insertions(+) create mode 100755 Core/src/org/sleuthkit/autopsy/xryparser/XRYRecordParser.java create mode 100755 Core/src/org/sleuthkit/autopsy/xryparser/XRYReport.java create mode 100755 Core/src/org/sleuthkit/autopsy/xryparser/XRYReportExtractor.java create mode 100755 Core/test/unit/src/org/sleuthkit/autopsy/xryparser/XRYReportExtractorTest.java create mode 100755 Core/test/unit/src/org/sleuthkit/autopsy/xryparser/XRYReportTest.java diff --git a/Core/src/org/sleuthkit/autopsy/xryparser/XRYRecordParser.java b/Core/src/org/sleuthkit/autopsy/xryparser/XRYRecordParser.java new file mode 100755 index 0000000000..da1d79ce66 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/xryparser/XRYRecordParser.java @@ -0,0 +1,34 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2019 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.xryparser; + +import org.sleuthkit.datamodel.BlackboardArtifact; + +/** + * + */ +public interface XRYRecordParser { + + /** + * + * @param xryRecord + * @return + */ + public BlackboardArtifact makeArtifact(String xryRecord); +} diff --git a/Core/src/org/sleuthkit/autopsy/xryparser/XRYReport.java b/Core/src/org/sleuthkit/autopsy/xryparser/XRYReport.java new file mode 100755 index 0000000000..e38586e15a --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/xryparser/XRYReport.java @@ -0,0 +1,131 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2019 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.xryparser; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Optional; +import java.util.stream.Stream; + +/** + * + */ +public class XRYReport { + + //Number of lines that make up the header of the report file. + private final static int LINES_IN_HEADER = 5; + + //Header line number that corresponds to the report type. + private final static int LINE_WITH_REPORT_TYPE = 3; + + //Encoding of the XRY Report file. + private final static Charset REPORT_ENCODING = StandardCharsets.UTF_16LE; + + //Path to the physical report file + private final Path reportPath; + + //XRY Report type (Calls, Messages, etc) + private final String reportType; + + /** + * + * @param reportPath + * @throws IOException + */ + public XRYReport(Path reportPath) throws IOException { + this.reportPath = reportPath; + this.reportType = parseType(reportPath); + } + + /** + * + * @return + */ + public String getType() { + return reportType; + } + + /** + * + * @return + */ + public Path getPath() { + return reportPath; + } + + /** + * + * @return + */ + public Charset getEncoding() { + return REPORT_ENCODING; + } + + /** + * + * @return + */ + public int getTotalLinesInHeader() { + return LINES_IN_HEADER; + } + + /** + * + * @return + */ + public int getReportTypeLineNumber() { + return LINE_WITH_REPORT_TYPE; + } + + /** + * + * @param report + * @return + * @throws IOException + */ + private String parseType(Path report) throws IOException { + try { + BufferedReader reader = Files.newBufferedReader(report, this.getEncoding()); + + //Limit this stream to only the length of the header + //and skip to the line just before the type information. + Stream xryReportHeader = reader.lines() + .limit(this.getTotalLinesInHeader()) + .skip(this.getReportTypeLineNumber() - 1); + + Optional type = xryReportHeader.findFirst(); + if(!type.isPresent()) { + throw new IllegalArgumentException("Report did not have a type."); + } + + if(type.get().isEmpty()) { + throw new IllegalArgumentException("Report did not have a type."); + } + + return type.get(); + } catch (UncheckedIOException ex) { + throw ex.getCause(); + } + } +} \ No newline at end of file diff --git a/Core/src/org/sleuthkit/autopsy/xryparser/XRYReportExtractor.java b/Core/src/org/sleuthkit/autopsy/xryparser/XRYReportExtractor.java new file mode 100755 index 0000000000..9dd8c90de4 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/xryparser/XRYReportExtractor.java @@ -0,0 +1,105 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2019 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.xryparser; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Stream; +import org.sleuthkit.datamodel.BlackboardArtifact; + +/** + * This class is responsible for extracting XRY records from a specified + * XRYReport. XRY records are the numbered, blank line separated 'groups' in an + * XRY report. + * + * Example: + * + * Calls # 1 + * Call Type: Missed + * Time: 1/2/2019 1:23:45 PM (Device) + * From + * Tel: 12345678 + * + */ +public final class XRYReportExtractor { + + private final XRYReport xryReport; + + /** + * Creates an XRYReportExtractor. + * + * @param report Report to be extracted. + */ + public XRYReportExtractor(XRYReport report) { + this.xryReport = report; + } + + /** + * + * @param parser + * @throws IOException + */ + public List extract(XRYRecordParser parser) throws IOException { + try { + BufferedReader reader = Files.newBufferedReader(xryReport.getPath(), xryReport.getEncoding()); + + //Get a stream of all lines in the file. Skip the first n header lines. + Stream xryReportStream = reader.lines().skip(xryReport.getTotalLinesInHeader()); + + StringBuilder xryRecord = new StringBuilder(); + List artifacts = new ArrayList<>(); + xryReportStream.forEach((line) -> { + if (this.isEndOfXRYRecord(line)) { + //Pass only non empty XRY records to the parser. + if (xryRecord.length() > 0) { + artifacts.add(parser.makeArtifact(xryRecord.toString())); + xryRecord.setLength(0); + } + } else { + xryRecord.append(line).append("\n"); + } + }); + + //The file may have ended without a blank line (which is used to delimit + //records). The last XRY record would not have been processed. + if (xryRecord.length() > 0) { + artifacts.add(parser.makeArtifact(xryRecord.toString())); + } + + return artifacts; + } catch (UncheckedIOException ex) { + throw ex.getCause(); + } + } + + /** + * Determines if the line encountered during file reading signifies the end + * of an XRYRecord. + * + * @param line + * @return + */ + private boolean isEndOfXRYRecord(String line) { + return line.isEmpty(); + } +} diff --git a/Core/test/unit/src/org/sleuthkit/autopsy/xryparser/XRYReportExtractorTest.java b/Core/test/unit/src/org/sleuthkit/autopsy/xryparser/XRYReportExtractorTest.java new file mode 100755 index 0000000000..19a4ee747a --- /dev/null +++ b/Core/test/unit/src/org/sleuthkit/autopsy/xryparser/XRYReportExtractorTest.java @@ -0,0 +1,228 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2019 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.xryparser; + +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashSet; +import java.util.Set; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.sleuthkit.datamodel.BlackboardArtifact; +import static org.junit.Assert.*; + +/** + * + */ +public class XRYReportExtractorTest { + + private final Path reportDirectory = Paths.get("C:", "Users", "dsmyda", "Downloads", "2019-10-23-XRYSamples", "files"); + + public XRYReportExtractorTest() { + } + + @BeforeClass + public static void setUpClass() { + } + + @AfterClass + public static void tearDownClass() { + } + + @Before + public void setUp() { + } + + @After + public void tearDown() { + } + + @Test + public void testCallLogsSample() throws IOException { + Path reportPath = reportDirectory.resolve("Calls.txt"); + XRYReportExtractor extractor = new XRYReportExtractor(new XRYReport(reportPath)); + Set expectation = new HashSet() { + { + add("Calls # 1\n" + + "Call Type: Missed\n" + + "Time: 1/2/2019 1:23:45 PM (Device)\n" + + "From\n" + + "Tel: 12345678\n"); + add("Calls # 2\n" + + "Call Type: Dialed\n" + + "Time: 1/2/2019 2:34:56 PM (Device)\n" + + "Duration: 00:00:05\n" + + "To\n" + + "Tel: 23456789\n"); + add("Calls # 3\n" + + "Call Type: Last Dialed\n" + + "Number: 1234\n" + + "Storage: SIM\n" + + "To\n"); + add("Calls # 4\n" + + "Call Type: Received\n" + + "Time: 1/2/2019 2:34:56 AM (Device)\n" + + "Duration: 00:00:20\n" + + "From\n" + + "Tel: 34567890\n"); + } + }; + MockXRYRecordParser mockParser = new MockXRYRecordParser(expectation); + extractor.extract(mockParser); + assertEquals(expectation.size(), mockParser.getCount()); + } + + @Test + public void testMessagesSample() throws IOException { + Path reportPath = reportDirectory.resolve("Messages-SMS.txt"); + XRYReportExtractor extractor = new XRYReportExtractor(new XRYReport(reportPath)); + Set expectation = new HashSet() { + { + add("Messages-SMS # 1\n" + + "Text: Hello, this is my message. \n" + + "It has multiple lines.\n" + + "Time: 1/23/2019 1:23:45 PM UTC (Network)\n" + + "Type: Deliver\n" + + "Reference Number: 22\n" + + "Segment Number: 1\n" + + "Segments: 1\n" + + "From\n" + + "Tel: 12345678\n"); + add("Messages-SMS # 2\n" + + "Text: Hello, this is another message. one line.\n" + + "Time: 1/2/2019 1:33:44 PM (Device)\n" + + "Type: Submit\n" + + "To\n" + + "Tel: 1234\n"); + add("Messages-SMS # 3\n" + + "Text: Text goes here\n" + + "Time: 1/3/2019 2:33:22 PM (Device)\n" + + "Type: Status Report\n" + + "Participant\n" + + "Tel: 12345\n"); + } + }; + MockXRYRecordParser mockParser = new MockXRYRecordParser(expectation); + extractor.extract(mockParser); + assertEquals(expectation.size(), mockParser.getCount()); + } + + @Test + public void testContactsSample() throws IOException { + Path reportPath = reportDirectory.resolve("Contacts-Contacts.txt"); + XRYReportExtractor extractor = new XRYReportExtractor(new XRYReport(reportPath)); + Set expectation = new HashSet() { + { + add("Contacts-Contacts # 1\n" + + "Name: Abc\n" + + "Tel: +123456\n" + + "Storage: Device\n"); + add("Contacts-Contacts # 2\n" + + "Name: Xyz\n" + + "Tel: +34567\n" + + "Storage: SIM\n"); + } + }; + MockXRYRecordParser mockParser = new MockXRYRecordParser(expectation); + extractor.extract(mockParser); + assertEquals(expectation.size(), mockParser.getCount()); + } + + @Test + public void testWebBookmarksSample() throws IOException { + Path reportPath = reportDirectory.resolve("Web-Bookmarks.txt"); + XRYReportExtractor extractor = new XRYReportExtractor(new XRYReport(reportPath)); + Set expectation = new HashSet() { + { + add("Web-Bookmarks # 1\n" + + "Web Address: http://www.google.com\n" + + "Domain: Google Search\n"); + } + }; + MockXRYRecordParser mockParser = new MockXRYRecordParser(expectation); + extractor.extract(mockParser); + assertEquals(expectation.size(), mockParser.getCount()); + } + + @Test + public void testDeviceSample() throws IOException { + Path reportPath = reportDirectory.resolve("Device-General Information.txt"); + XRYReportExtractor extractor = new XRYReportExtractor(new XRYReport(reportPath)); + Set expectation = new HashSet() { + { + add("Device-General Information # 1\n" + + "Data: c:\\Path To Something I forget what though\\Maybe the Storage folder\n"); + add("Device-General Information # 2\n" + + "Data: Nokia XYZ\n" + + "Attribute: Device Name\n"); + add("Device-General Information # 3\n" + + "Data: Phone\n" + + "Attribute: Device Family\n"); + add("Device-General Information # 4\n" + + "Data: XYZ\n" + + "Attribute: Device Type\n"); + add("Device-General Information # 5\n" + + "Data: 123456\n" + + "Attribute: Mobile Id (IMEI)\n"); + add("Device-General Information # 6\n" + + "Data: 12345\n" + + "Attribute: Security Code\n"); + add("Device-General Information # 7\n" + + "Data: SIM Card\n" + + "Attribute: Device Name\n"); + } + }; + MockXRYRecordParser mockParser = new MockXRYRecordParser(expectation); + extractor.extract(mockParser); + assertEquals(expectation.size(), mockParser.getCount()); + } + + /** + * Mock XRYRecordParser. Rather than creating BlackboardArtifacts, we are + * instead verifying that the XRY Records are being parsed out correctly. + */ + private class MockXRYRecordParser implements XRYRecordParser { + + private final Set allRecords; + private int recordCount; + + public MockXRYRecordParser(Set allRecords) { + this.allRecords = allRecords; + recordCount = 0; + } + + @Override + public BlackboardArtifact makeArtifact(String xryRecord) { + assertNotNull(allRecords); + assertNotNull(xryRecord); + recordCount++; + assertTrue("More records than expected: " + recordCount + ". Expected at most: "+allRecords.size(), recordCount <= allRecords.size()); + assertTrue("Did not find the following record: " + xryRecord, allRecords.contains(xryRecord)); + return null; + } + + public int getCount() { + return recordCount; + } + } +} \ No newline at end of file diff --git a/Core/test/unit/src/org/sleuthkit/autopsy/xryparser/XRYReportTest.java b/Core/test/unit/src/org/sleuthkit/autopsy/xryparser/XRYReportTest.java new file mode 100755 index 0000000000..85c791b7db --- /dev/null +++ b/Core/test/unit/src/org/sleuthkit/autopsy/xryparser/XRYReportTest.java @@ -0,0 +1,112 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2019 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.xryparser; + +import java.util.ArrayList; +import java.io.IOException; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.List; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import static org.junit.Assert.*; + +/** + * + */ +public class XRYReportTest { + + private final Path reportDirectory = Paths.get("C:", "Users", "dsmyda", "Downloads", "2019-10-23-XRYSamples", "files"); + + public XRYReportTest() { + } + + @BeforeClass + public static void setUpClass() { + } + + @AfterClass + public static void tearDownClass() { + } + + @Before + public void setUp() { + } + + @After + public void tearDown() { + } + + @Test + public void testParseType() { + List reportTestFiles = new ArrayList() { + { + add(reportDirectory.resolve("Calls.txt")); + add(reportDirectory.resolve("Messages-SMS.txt")); + add(reportDirectory.resolve("Contacts-Contacts.txt")); + add(reportDirectory.resolve("Web-Bookmarks.txt")); + add(reportDirectory.resolve("Device-General Information.txt")); + } + }; + + List expectedTypes = new ArrayList() { + { + add("Calls"); + add("Messages/SMS"); + add("Contacts/Contacts"); + add("Web/Bookmarks"); + add("Device/General Information"); + } + }; + + List actualTypes = new ArrayList<>(); + + reportTestFiles.forEach((Path reportFile) -> { + try { + XRYReport reportObj = new MockXRYReport(reportFile); + actualTypes.add(reportObj.getType()); + } catch (IOException ex) { + fail(ex.getMessage()); + } + }); + + assertArrayEquals("Types did not match.", expectedTypes.toArray(), actualTypes.toArray()); + } + + /** + * Mock the valid XRY Report encoding to UTF-8 so that the test files can be + * run unmodified (they are currently UTF-8). + */ + private class MockXRYReport extends XRYReport { + + public MockXRYReport(Path reportPath) throws IOException { + super(reportPath); + } + + @Override + public Charset getEncoding() { + return StandardCharsets.UTF_8; + } + } +}