From 080c5370aef7732b44831b06e37113e961e491ac Mon Sep 17 00:00:00 2001 From: Kelly Kelly Date: Thu, 9 Jan 2020 13:12:24 -0500 Subject: [PATCH] Updated to use Artifact helper to create TSK_GEO_TRACK artifacts --- .../autopsy/modules/drones/DATDumper.java | 32 ++--- .../autopsy/modules/drones/DATExtractor.java | 129 +++++++++--------- .../modules/drones/DroneExtractor.java | 128 ++--------------- .../modules/drones/DroneIngestModule.java | 1 + 4 files changed, 92 insertions(+), 198 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/drones/DATDumper.java b/Core/src/org/sleuthkit/autopsy/modules/drones/DATDumper.java index d6a34c0e27..a5f45e34c4 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/drones/DATDumper.java +++ b/Core/src/org/sleuthkit/autopsy/modules/drones/DATDumper.java @@ -42,18 +42,20 @@ final class DATDumper { /** * Dump the DJI DAT file to a csv file. - * + * * @param datFilePath Path to input DAT file * @param outputFilePath Output file path * @param overWriteExisting True to overwrite an existing csv file with * outputFilePath - * - * @throws DroneIngestException + * + * @throws DroneIngestException */ void dumpDATFile(String datFilePath, String outputFilePath, boolean overWriteExisting) throws DroneIngestException { // Validate the input and output file paths. validateOutputFile(outputFilePath, overWriteExisting); - validateDATFile(datFilePath); + if (!isDATFile(datFilePath)) { + throw new DroneIngestException(String.format("Not a DAT file! DAT = %s", datFilePath)); //NON-NLS + } DatFile datFile = null; try (CsvWriter writer = new CsvWriter(outputFilePath)) { @@ -66,12 +68,12 @@ final class DATDumper { // Creates a version specific ConvertDat object ConvertDat convertDat = datFile.createConVertDat(); - + // The lower the sample rate the smaller the output csv file will be // however the date will be less precise. For our purposes we are going // a sample rate of 1. convertDat.sampleRate = 1; - + // Setting the tickRangeLower and upper values reduces some of the // noise invalid data in the output file. if (datFile.gpsLockTick != -1) { @@ -81,17 +83,17 @@ final class DATDumper { if (datFile.lastMotorStopTick != -1) { convertDat.tickRangeUpper = datFile.lastMotorStopTick; } - + convertDat.setCsvWriter(writer); convertDat.createRecordParsers(); datFile.reset(); - + // Analyze does the work of parsing the data, everything prior was // setup convertDat.analyze(true); } catch (IOException | NotDatFile | FileEnd ex) { - throw new DroneIngestException(String.format("Failed to dump DAT file to csv. DAT = %s, CSV = %s", datFilePath, outputFilePath), ex); + throw new DroneIngestException(String.format("Failed to dump DAT file to csv. DAT = %s, CSV = %s", datFilePath, outputFilePath), ex); //NON-NLS } finally { if (datFile != null) { datFile.close(); @@ -117,7 +119,7 @@ final class DATDumper { if (overWriteExisting) { csvFile.delete(); } else { - throw new DroneIngestException(String.format("Unable to dump DAT file. overWriteExsiting is false and DAT output csv file exists: %s", outputFileName)); + throw new DroneIngestException(String.format("Unable to dump DAT file. overWriteExsiting is false and DAT output csv file exists: %s", outputFileName)); //NON-NLS } } } @@ -130,19 +132,17 @@ final class DATDumper { * * @throws DroneIngestException */ - private void validateDATFile(String datFilePath) throws DroneIngestException { + public boolean isDATFile(String datFilePath) throws DroneIngestException { File datFile = new File(datFilePath); if (!datFile.exists()) { - throw new DroneIngestException(String.format("Unable to dump DAT file DAT file does not exist: %s", datFilePath)); + throw new DroneIngestException(String.format("Unable to dump DAT file DAT file does not exist: %s", datFilePath)); //NON-NLS } try { - if (!DatFile.isDatFile(datFilePath) && !DJIAssistantFile.isDJIDat(datFile)) { - throw new DroneIngestException(String.format("Unable to dump DAT file. File is not a DAT file: %s", datFilePath)); - } + return DatFile.isDatFile(datFilePath) || DJIAssistantFile.isDJIDat(datFile); } catch (FileNotFoundException ex) { - throw new DroneIngestException(String.format("Unable to dump DAT file. File not found %s", datFilePath), ex); + throw new DroneIngestException(String.format("Unable to dump DAT file. File not found %s", datFilePath), ex); //NON-NLS } } } diff --git a/Core/src/org/sleuthkit/autopsy/modules/drones/DATExtractor.java b/Core/src/org/sleuthkit/autopsy/modules/drones/DATExtractor.java index 50db7723ba..6bbb10437e 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/drones/DATExtractor.java +++ b/Core/src/org/sleuthkit/autopsy/modules/drones/DATExtractor.java @@ -23,14 +23,11 @@ import java.io.File; import java.io.FileReader; import java.io.IOException; import java.nio.file.Paths; -import java.time.LocalDateTime; import java.time.ZoneOffset; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; -import java.time.temporal.TemporalAccessor; import java.util.ArrayList; -import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -43,14 +40,16 @@ import org.sleuthkit.autopsy.coreutils.FileUtil; import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProgress; import org.sleuthkit.autopsy.ingest.IngestJobContext; import org.sleuthkit.datamodel.AbstractFile; -import org.sleuthkit.datamodel.BlackboardArtifact; -import org.sleuthkit.datamodel.BlackboardAttribute; import org.sleuthkit.datamodel.Content; +import org.sleuthkit.datamodel.blackboardutils.GeoArtifactHelper.GeoTrackPoint; +import org.sleuthkit.datamodel.blackboardutils.GeoArtifactHelper.GeoTrackPoints; +import org.sleuthkit.datamodel.blackboardutils.GeoArtifactHelper; import org.sleuthkit.datamodel.TskCoreException; +import org.sleuthkit.datamodel.Blackboard.BlackboardException; /** * Extract drone position data from DJI Phantom drones. - * + * * Module uses DatCon.jar to dump FLYXXX.DAT file to a CSV file which is stored * in the case temp directory. Artifacts are created by parsing the csv file. * @@ -59,24 +58,23 @@ final class DATExtractor extends DroneExtractor { private static final Logger logger = Logger.getLogger(DATExtractor.class.getName()); - private static final String HEADER_LONG = "IMU_ATTI(0):Longitude"; - private static final String HEADER_LAT = "IMU_ATTI(0):Latitude"; - private static final String HEADER_VELOCITY = "IMU_ATTI(0):velComposite"; - private static final String HEADER_DATETILE = "GPS:dateTimeStamp"; - private static final String HEADER_ALTITUDE = "GPS(0):heightMSL"; - private static final String HEADER_DISTANCE_FROM_HP = "IMU_ATTI(0):distanceHP"; - private static final String HEADER_DISTANCE_TRAVELED = "IMU_ATTI(0):distanceTravelled"; + private static final String HEADER_LONG = "IMU_ATTI(0):Longitude"; //NON-NLS + private static final String HEADER_LAT = "IMU_ATTI(0):Latitude"; //NON-NLS + private static final String HEADER_VELOCITY = "IMU_ATTI(0):velComposite"; //NON-NLS + private static final String HEADER_DATETILE = "GPS:dateTimeStamp"; //NON-NLS + private static final String HEADER_ALTITUDE = "GPS(0):heightMSL"; //NON-NLS + private static final String HEADER_DISTANCE_FROM_HP = "IMU_ATTI(0):distanceHP"; //NON-NLS + private static final String HEADER_DISTANCE_TRAVELED = "IMU_ATTI(0):distanceTravelled"; //NON-NLS /** * Construct a DATExtractor. - * - * @throws DroneIngestException + * + * @throws DroneIngestException */ DATExtractor() throws DroneIngestException { super(); } - @Messages({ "DATExtractor_process_message=Processing DJI DAT file: %s" }) @@ -91,27 +89,38 @@ final class DATExtractor extends DroneExtractor { if (context.dataSourceIngestIsCancelled()) { break; } - + progressBar.progress(String.format(Bundle.DATExtractor_process_message(), DATFile.getName())); - + // Copy the DAT file into the case temp folder File tempDATFile = getTemporaryFile(context, DATFile); - + // Create a path for the csv file String csvFilePath = getCSVPathForDAT(DATFile); try { + if (!dumper.isDATFile(tempDATFile.getAbsolutePath())) { + logger.log(Level.WARNING, String.format("%s is not a valid DAT file", DATFile.getName())); //NON-NLS + continue; + } // Dump the DAT file to a csv file dumper.dumpDATFile(tempDATFile.getAbsolutePath(), csvFilePath, true); if (context.dataSourceIngestIsCancelled()) { break; } - + // Process the csv file - processCSVFile(context, DATFile, csvFilePath); - } catch (DroneIngestException ex) { - logger.log(Level.WARNING, String.format("Exception thrown while processing DAT file %s", DATFile.getName()), ex); + List trackPoints = processCSVFile(context, DATFile, csvFilePath); + + if (trackPoints != null && !trackPoints.isEmpty()) { + (new GeoArtifactHelper(getSleuthkitCase(), getName(), DATFile)).addTrack(DATFile.getName(), new GeoTrackPoints(trackPoints)); + } else { + logger.log(Level.INFO, String.format("No trackpoints with valid longitude or latitude found in %s", DATFile.getName())); //NON-NLS + } + + } catch (TskCoreException | BlackboardException ex) { + logger.log(Level.WARNING, String.format("Exception thrown while processing DAT file %s", DATFile.getName()), ex); //NON-NLS } finally { tempDATFile.delete(); (new File(csvFilePath)).delete(); @@ -148,9 +157,9 @@ final class DATExtractor extends DroneExtractor { // findFiles use the SQL wildcard # in the file name try { - fileList = fileManager.findFiles(dataSource, "FLY___.DAT"); + fileList = fileManager.findFiles(dataSource, "FLY___.DAT"); //NON-NLS } catch (TskCoreException ex) { - throw new DroneIngestException("Unable to find drone DAT files.", ex); + throw new DroneIngestException("Unable to find drone DAT files.", ex); //NON-NLS } return fileList; @@ -164,49 +173,45 @@ final class DATExtractor extends DroneExtractor { * @return Absolute csv file path */ private String getCSVPathForDAT(AbstractFile file) { - String tempFileName = file.getName() + file.getId() + ".csv"; + String tempFileName = file.getName() + file.getId() + ".csv"; //NON-NLS return Paths.get(getExtractorTempPath().toString(), tempFileName).toString(); } /** * Process the csv dump of the drone DAT file. - * + * * Create artifacts for all rows that have a valid longitude and latitude. - * - * @param context current case job context - * @param DATFile Original DAT file + * + * @param context current case job context + * @param DATFile Original DAT file * @param csvFilePath Path of csv file to process - * - * @throws DroneIngestException + * + * @throws DroneIngestException */ - private void processCSVFile(IngestJobContext context, AbstractFile DATFile, String csvFilePath) throws DroneIngestException { + private List processCSVFile(IngestJobContext context, AbstractFile DATFile, String csvFilePath) throws DroneIngestException { + List trackPoints = new ArrayList<>(); try (BufferedReader reader = new BufferedReader(new FileReader(new File(csvFilePath)))) { // First read in the header line and process String line = reader.readLine(); - Map headerMap = makeHeaderMap(line.split(",")); - Collection artifacts = new ArrayList<>(); + Map headerMap = makeHeaderMap(line.split(",")); //NON-NLS while ((line = reader.readLine()) != null) { if (context.dataSourceIngestIsCancelled()) { - break; + return null; } - - String[] values = line.split(","); - Collection attributes = buildAttributes(headerMap, values); - if (attributes != null) { - artifacts.add(makeWaypointArtifact(DATFile, attributes)); - } - } - - if (context.dataSourceIngestIsCancelled()) { - return; - } - postArtifacts(artifacts); + String[] values = line.split(","); //NON-NLS + GeoTrackPoint point = createTrackPoint(headerMap, values); + if (point != null) { + trackPoints.add(point); + } + } } catch (IOException ex) { - throw new DroneIngestException(String.format("Failed to read DAT csvFile %s", csvFilePath), ex); + throw new DroneIngestException(String.format("Failed to read DAT csvFile %s", csvFilePath), ex); //NON-NLS } + + return trackPoints; } /** @@ -234,14 +239,14 @@ final class DATExtractor extends DroneExtractor { * interesting and return a null collection. * * @param columnLookup column header lookup map - * @param values Row data + * @param values Row data * - * @return Collection of BlackboardAttributes for row or null collection if - * longitude or latitude was not valid + * @return Collection of BlackboardAttributes for row or null collection if + * longitude or latitude was not valid * * @throws DroneIngestException */ - private Collection buildAttributes(Map columnLookup, String[] values) throws DroneIngestException { + private GeoTrackPoint createTrackPoint(Map columnLookup, String[] values) throws DroneIngestException { Double latitude = getDoubleValue(columnLookup.get(HEADER_LAT), values); Double longitude = getDoubleValue(columnLookup.get(HEADER_LONG), values); @@ -251,13 +256,13 @@ final class DATExtractor extends DroneExtractor { return null; } - return makeWaypointAttributes(latitude, + return new GeoArtifactHelper.GeoTrackPoint(latitude, longitude, getDoubleValue(columnLookup.get(HEADER_ALTITUDE), values), - getDateTimeValue(columnLookup, values), getDoubleValue(columnLookup.get(HEADER_VELOCITY), values), getDoubleValue(columnLookup.get(HEADER_DISTANCE_FROM_HP), values), - getDoubleValue(columnLookup.get(HEADER_DISTANCE_TRAVELED), values)); + getDoubleValue(columnLookup.get(HEADER_DISTANCE_TRAVELED), values), + getDateTimeValue(columnLookup, values)); } /** @@ -291,12 +296,12 @@ final class DATExtractor extends DroneExtractor { /** * Returns the string value at the given index parsed as a double. - * - * @param index Index to string array - * @param values Array of string values - * - * @return Double value or null if the index is out of bounds of the string - * array or the string value at index was not a double. + * + * @param index Index to string array + * @param values Array of string values + * + * @return Double value or null if the index is out of bounds of the string + * array or the string value at index was not a double. */ private Double getDoubleValue(Integer index, String[] values) { if (index == null || index == -1 || index > values.length) { diff --git a/Core/src/org/sleuthkit/autopsy/modules/drones/DroneExtractor.java b/Core/src/org/sleuthkit/autopsy/modules/drones/DroneExtractor.java index f1dd2ff68c..5c92319ff2 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/drones/DroneExtractor.java +++ b/Core/src/org/sleuthkit/autopsy/modules/drones/DroneExtractor.java @@ -22,27 +22,21 @@ import java.io.File; import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.Collection; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; import org.sleuthkit.autopsy.datamodel.ContentUtils; import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProgress; import org.sleuthkit.autopsy.ingest.IngestJobContext; import org.sleuthkit.datamodel.AbstractFile; -import org.sleuthkit.datamodel.Blackboard; -import org.sleuthkit.datamodel.BlackboardArtifact; -import org.sleuthkit.datamodel.BlackboardAttribute; import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.SleuthkitCase; -import org.sleuthkit.datamodel.TskCoreException; /** * Abstract base class for all Drone file extractors. */ abstract class DroneExtractor { - static private final String TEMP_FOLDER_NAME = "DroneExtractor"; + static private final String TEMP_FOLDER_NAME = "DroneExtractor"; //NON-NLS private final Case currentCase; /** @@ -54,7 +48,7 @@ abstract class DroneExtractor { try { currentCase = Case.getCurrentCaseThrows(); } catch (NoCurrentCaseException ex) { - throw new DroneIngestException("Unable to create drone extractor, no open case.", ex); + throw new DroneIngestException("Unable to create drone extractor, no open case.", ex); //NON-NLS } } @@ -80,112 +74,6 @@ abstract class DroneExtractor { return currentCase.getSleuthkitCase(); } - /** - * Return the Blackboard object. - * - * @return Current Case blackboard object. - */ - final protected Blackboard getBlackboard() { - return currentCase.getSleuthkitCase().getBlackboard(); - } - - /** - * Method to post a list of BlackboardArtifacts to the blackboard. - * - * @param artifacts A list of artifacts. IF list is empty or null, the - * function will return. - */ - void postArtifacts(Collection artifacts) throws DroneIngestException { - if (artifacts == null || artifacts.isEmpty()) { - return; - } - - try { - getBlackboard().postArtifacts(artifacts, getName()); - } catch (Blackboard.BlackboardException ex) { - throw new DroneIngestException(String.format("Failed to post Drone artifacts to blackboard."), ex); - } - } - - /** - * Create a TSK_WAYPOINT artifact with the given list of attributes. - * - * @param DATFile DAT file - * @param attributes List of BlackboardAttributes - * - * @return TSK_WAYPOINT BlackboardArtifact - * - * @throws DroneIngestException - */ - protected BlackboardArtifact makeWaypointArtifact(AbstractFile DATFile, Collection attributes) throws DroneIngestException { - try { - BlackboardArtifact artifact = DATFile.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_GPS_TRACKPOINT); - artifact.addAttributes(attributes); - return artifact; - } catch (TskCoreException ex) { - throw new DroneIngestException(String.format("Failed to post Drone artifacts to blackboard."), ex); - } - } - - /** - * Returns a list of BlackboardAttributes for the given parameters. - * - * Throws exception of longitude or latitude are null. - * - * @param latitude Waypoint latitude, must be non-null - * @param longitude waypoint longitude, must be non-null - * @param altitude Waypoint altitude\height - * @param dateTime Timestamp the waypoint was created (Java epoch - * seconds) - * @param velocity Velocity - * @param distanceHP Distance from home point - * @param distanceTraveled Total distance the drone has traveled - * - * @return Collection of BlackboardAttributes - * - * @throws DroneIngestException - */ - protected Collection makeWaypointAttributes(Double latitude, Double longitude, Double altitude, Long dateTime, Double velocity, Double distanceHP, Double distanceTraveled) throws DroneIngestException { - Collection attributes = new ArrayList<>(); - - if (latitude == null || longitude == null) { - throw new DroneIngestException("Invalid list of waypoint attributes, longitude or latitude was null"); - } - - attributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_GEO_LATITUDE, - DroneIngestModuleFactory.getModuleName(), latitude)); - - attributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_GEO_LONGITUDE, - DroneIngestModuleFactory.getModuleName(), longitude)); - - if (altitude != null) { - attributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_GEO_ALTITUDE, - DroneIngestModuleFactory.getModuleName(), altitude)); - } - - if (dateTime != null) { - attributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME, - DroneIngestModuleFactory.getModuleName(), dateTime)); - } - - if (velocity != null) { - attributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_GEO_VELOCITY, - DroneIngestModuleFactory.getModuleName(), velocity)); - } - - if (distanceHP != null) { - attributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_GPS_DISTANCE_FROM_HOMEPOINT, - DroneIngestModuleFactory.getModuleName(), distanceHP)); - } - - if (distanceTraveled != null) { - attributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_GPS_DISTANCE_TRAVELED, - DroneIngestModuleFactory.getModuleName(), distanceTraveled)); - } - - return attributes; - } - /** * Build the temp path and create the directory if it does not currently * exist. @@ -207,13 +95,13 @@ abstract class DroneExtractor { /** * Create a copy of file in the case temp directory. - * + * * @param context Current ingest context - * @param file File to be copied - * + * @param file File to be copied + * * @return File copy. - * - * @throws DroneIngestException + * + * @throws DroneIngestException */ protected File getTemporaryFile(IngestJobContext context, AbstractFile file) throws DroneIngestException { String tempFileName = file.getName() + file.getId() + file.getNameExtension(); @@ -223,7 +111,7 @@ abstract class DroneExtractor { try { ContentUtils.writeToFile(file, tempFilePath.toFile(), context::dataSourceIngestIsCancelled); } catch (IOException ex) { - throw new DroneIngestException(String.format("Unable to create temp file %s for abstract file %s", tempFilePath.toString(), file.getName()), ex); + throw new DroneIngestException(String.format("Unable to create temp file %s for abstract file %s", tempFilePath.toString(), file.getName()), ex); //NON-NLS } return tempFilePath.toFile(); diff --git a/Core/src/org/sleuthkit/autopsy/modules/drones/DroneIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/drones/DroneIngestModule.java index 3265c4de4c..d126761e75 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/drones/DroneIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/drones/DroneIngestModule.java @@ -56,6 +56,7 @@ public final class DroneIngestModule implements DataSourceIngestModule { } @Messages({ + "# {0} - AbstractFileName", "DroneIngestModule_process_start=Started {0}" }) @Override