mstoeck3/autopsy-flatpak
1
0
Fork 0
mirror of https://github.com/overcuriousity/autopsy-flatpak.git synced 2025-07-11 23:46:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

Updated to use Artifact helper to create TSK_GEO_TRACK artifacts

Browse Source
This commit is contained in:
Kelly Kelly 2020-01-09 13:12:24 -05:00
parent b8af7a3a9d
commit 080c5370ae
4 changed files with 92 additions and 198 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
Core/src/org/sleuthkit/autopsy/modules/drones/DATDumper.java
View File

@ -42,18 +42,20 @@ final class DATDumper {
/**
* Dump the DJI DAT file to a csv file.
*
*
* @param datFilePath Path to input DAT file
* @param outputFilePath Output file path
* @param overWriteExisting True to overwrite an existing csv file with
* outputFilePath
*
* @throws DroneIngestException
*
* @throws DroneIngestException
*/
void dumpDATFile(String datFilePath, String outputFilePath, boolean overWriteExisting) throws DroneIngestException {
// Validate the input and output file paths.
validateOutputFile(outputFilePath, overWriteExisting);
validateDATFile(datFilePath);
if (!isDATFile(datFilePath)) {
throw new DroneIngestException(String.format("Not a DAT file! DAT = %s", datFilePath)); //NON-NLS
}
DatFile datFile = null;
try (CsvWriter writer = new CsvWriter(outputFilePath)) {
@ -66,12 +68,12 @@ final class DATDumper {
// Creates a version specific ConvertDat object
ConvertDat convertDat = datFile.createConVertDat();
// The lower the sample rate the smaller the output csv file will be
// however the date will be less precise. For our purposes we are going
// a sample rate of 1.
convertDat.sampleRate = 1;
// Setting the tickRangeLower and upper values reduces some of the
// noise invalid data in the output file.
if (datFile.gpsLockTick != -1) {
@ -81,17 +83,17 @@ final class DATDumper {
if (datFile.lastMotorStopTick != -1) {
convertDat.tickRangeUpper = datFile.lastMotorStopTick;
}
convertDat.setCsvWriter(writer);
convertDat.createRecordParsers();
datFile.reset();
// Analyze does the work of parsing the data, everything prior was
// setup
convertDat.analyze(true);
} catch (IOException | NotDatFile | FileEnd ex) {
throw new DroneIngestException(String.format("Failed to dump DAT file to csv. DAT = %s, CSV = %s", datFilePath, outputFilePath), ex);
throw new DroneIngestException(String.format("Failed to dump DAT file to csv. DAT = %s, CSV = %s", datFilePath, outputFilePath), ex); //NON-NLS
} finally {
if (datFile != null) {
datFile.close();
@ -117,7 +119,7 @@ final class DATDumper {
if (overWriteExisting) {
csvFile.delete();
} else {
throw new DroneIngestException(String.format("Unable to dump DAT file. overWriteExsiting is false and DAT output csv file exists: %s", outputFileName));
throw new DroneIngestException(String.format("Unable to dump DAT file. overWriteExsiting is false and DAT output csv file exists: %s", outputFileName)); //NON-NLS
}
}
}
@ -130,19 +132,17 @@ final class DATDumper {
*
* @throws DroneIngestException
*/
private void validateDATFile(String datFilePath) throws DroneIngestException {
public boolean isDATFile(String datFilePath) throws DroneIngestException {
File datFile = new File(datFilePath);
if (!datFile.exists()) {
throw new DroneIngestException(String.format("Unable to dump DAT file DAT file does not exist: %s", datFilePath));
throw new DroneIngestException(String.format("Unable to dump DAT file DAT file does not exist: %s", datFilePath)); //NON-NLS
}
try {
if (!DatFile.isDatFile(datFilePath) && !DJIAssistantFile.isDJIDat(datFile)) {
throw new DroneIngestException(String.format("Unable to dump DAT file. File is not a DAT file: %s", datFilePath));
}
return DatFile.isDatFile(datFilePath) || DJIAssistantFile.isDJIDat(datFile);
} catch (FileNotFoundException ex) {
throw new DroneIngestException(String.format("Unable to dump DAT file. File not found %s", datFilePath), ex);
throw new DroneIngestException(String.format("Unable to dump DAT file. File not found %s", datFilePath), ex); //NON-NLS
}
}
}

129
Core/src/org/sleuthkit/autopsy/modules/drones/DATExtractor.java
View File

@ -23,14 +23,11 @@ import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.nio.file.Paths;
import java.time.LocalDateTime;
import java.time.ZoneOffset;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException;
import java.time.temporal.TemporalAccessor;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -43,14 +40,16 @@ import org.sleuthkit.autopsy.coreutils.FileUtil;
import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProgress;
import org.sleuthkit.autopsy.ingest.IngestJobContext;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.BlackboardArtifact;
import org.sleuthkit.datamodel.BlackboardAttribute;
import org.sleuthkit.datamodel.Content;
import org.sleuthkit.datamodel.blackboardutils.GeoArtifactHelper.GeoTrackPoint;
import org.sleuthkit.datamodel.blackboardutils.GeoArtifactHelper.GeoTrackPoints;
import org.sleuthkit.datamodel.blackboardutils.GeoArtifactHelper;
import org.sleuthkit.datamodel.TskCoreException;
import org.sleuthkit.datamodel.Blackboard.BlackboardException;
/**
* Extract drone position data from DJI Phantom drones.
*
*
* Module uses DatCon.jar to dump FLYXXX.DAT file to a CSV file which is stored
* in the case temp directory. Artifacts are created by parsing the csv file.
*
@ -59,24 +58,23 @@ final class DATExtractor extends DroneExtractor {
private static final Logger logger = Logger.getLogger(DATExtractor.class.getName());
private static final String HEADER_LONG = "IMU_ATTI(0):Longitude";
private static final String HEADER_LAT = "IMU_ATTI(0):Latitude";
private static final String HEADER_VELOCITY = "IMU_ATTI(0):velComposite";
private static final String HEADER_DATETILE = "GPS:dateTimeStamp";
private static final String HEADER_ALTITUDE = "GPS(0):heightMSL";
private static final String HEADER_DISTANCE_FROM_HP = "IMU_ATTI(0):distanceHP";
private static final String HEADER_DISTANCE_TRAVELED = "IMU_ATTI(0):distanceTravelled";
private static final String HEADER_LONG = "IMU_ATTI(0):Longitude"; //NON-NLS
private static final String HEADER_LAT = "IMU_ATTI(0):Latitude"; //NON-NLS
private static final String HEADER_VELOCITY = "IMU_ATTI(0):velComposite"; //NON-NLS
private static final String HEADER_DATETILE = "GPS:dateTimeStamp"; //NON-NLS
private static final String HEADER_ALTITUDE = "GPS(0):heightMSL"; //NON-NLS
private static final String HEADER_DISTANCE_FROM_HP = "IMU_ATTI(0):distanceHP"; //NON-NLS
private static final String HEADER_DISTANCE_TRAVELED = "IMU_ATTI(0):distanceTravelled"; //NON-NLS
/**
* Construct a DATExtractor.
*
* @throws DroneIngestException
*
* @throws DroneIngestException
*/
DATExtractor() throws DroneIngestException {
super();
}
@Messages({
"DATExtractor_process_message=Processing DJI DAT file: %s"
})
@ -91,27 +89,38 @@ final class DATExtractor extends DroneExtractor {
if (context.dataSourceIngestIsCancelled()) {
break;
}
progressBar.progress(String.format(Bundle.DATExtractor_process_message(), DATFile.getName()));
// Copy the DAT file into the case temp folder
File tempDATFile = getTemporaryFile(context, DATFile);
// Create a path for the csv file
String csvFilePath = getCSVPathForDAT(DATFile);
try {
if (!dumper.isDATFile(tempDATFile.getAbsolutePath())) {
logger.log(Level.WARNING, String.format("%s is not a valid DAT file", DATFile.getName())); //NON-NLS
continue;
}
// Dump the DAT file to a csv file
dumper.dumpDATFile(tempDATFile.getAbsolutePath(), csvFilePath, true);
if (context.dataSourceIngestIsCancelled()) {
break;
}
// Process the csv file
processCSVFile(context, DATFile, csvFilePath);
} catch (DroneIngestException ex) {
logger.log(Level.WARNING, String.format("Exception thrown while processing DAT file %s", DATFile.getName()), ex);
List<GeoTrackPoint> trackPoints = processCSVFile(context, DATFile, csvFilePath);
if (trackPoints != null && !trackPoints.isEmpty()) {
(new GeoArtifactHelper(getSleuthkitCase(), getName(), DATFile)).addTrack(DATFile.getName(), new GeoTrackPoints(trackPoints));
} else {
logger.log(Level.INFO, String.format("No trackpoints with valid longitude or latitude found in %s", DATFile.getName())); //NON-NLS
}
} catch (TskCoreException | BlackboardException ex) {
logger.log(Level.WARNING, String.format("Exception thrown while processing DAT file %s", DATFile.getName()), ex); //NON-NLS
} finally {
tempDATFile.delete();
(new File(csvFilePath)).delete();
@ -148,9 +157,9 @@ final class DATExtractor extends DroneExtractor {
// findFiles use the SQL wildcard # in the file name
try {
fileList = fileManager.findFiles(dataSource, "FLY___.DAT");
fileList = fileManager.findFiles(dataSource, "FLY___.DAT"); //NON-NLS
} catch (TskCoreException ex) {
throw new DroneIngestException("Unable to find drone DAT files.", ex);
throw new DroneIngestException("Unable to find drone DAT files.", ex); //NON-NLS
}
return fileList;
@ -164,49 +173,45 @@ final class DATExtractor extends DroneExtractor {
* @return Absolute csv file path
*/
private String getCSVPathForDAT(AbstractFile file) {
String tempFileName = file.getName() + file.getId() + ".csv";
String tempFileName = file.getName() + file.getId() + ".csv"; //NON-NLS
return Paths.get(getExtractorTempPath().toString(), tempFileName).toString();
}
/**
* Process the csv dump of the drone DAT file.
*
*
* Create artifacts for all rows that have a valid longitude and latitude.
*
* @param context current case job context
* @param DATFile Original DAT file
*
* @param context current case job context
* @param DATFile Original DAT file
* @param csvFilePath Path of csv file to process
*
* @throws DroneIngestException
*
* @throws DroneIngestException
*/
private void processCSVFile(IngestJobContext context, AbstractFile DATFile, String csvFilePath) throws DroneIngestException {
private List<GeoTrackPoint> processCSVFile(IngestJobContext context, AbstractFile DATFile, String csvFilePath) throws DroneIngestException {
List<GeoTrackPoint> trackPoints = new ArrayList<>();
try (BufferedReader reader = new BufferedReader(new FileReader(new File(csvFilePath)))) {
// First read in the header line and process
String line = reader.readLine();
Map<String, Integer> headerMap = makeHeaderMap(line.split(","));
Collection<BlackboardArtifact> artifacts = new ArrayList<>();
Map<String, Integer> headerMap = makeHeaderMap(line.split(",")); //NON-NLS
while ((line = reader.readLine()) != null) {
if (context.dataSourceIngestIsCancelled()) {
break;
return null;
}
String[] values = line.split(",");
Collection<BlackboardAttribute> attributes = buildAttributes(headerMap, values);
if (attributes != null) {
artifacts.add(makeWaypointArtifact(DATFile, attributes));
}
}
if (context.dataSourceIngestIsCancelled()) {
return;
}
postArtifacts(artifacts);
String[] values = line.split(","); //NON-NLS
GeoTrackPoint point = createTrackPoint(headerMap, values);
if (point != null) {
trackPoints.add(point);
}
}
} catch (IOException ex) {
throw new DroneIngestException(String.format("Failed to read DAT csvFile %s", csvFilePath), ex);
throw new DroneIngestException(String.format("Failed to read DAT csvFile %s", csvFilePath), ex); //NON-NLS
}
return trackPoints;
}
/**
@ -234,14 +239,14 @@ final class DATExtractor extends DroneExtractor {
* interesting and return a null collection.
*
* @param columnLookup column header lookup map
* @param values Row data
* @param values Row data
*
* @return Collection of BlackboardAttributes for row or null collection if
* longitude or latitude was not valid
* @return Collection of BlackboardAttributes for row or null collection if
* longitude or latitude was not valid
*
* @throws DroneIngestException
*/
private Collection<BlackboardAttribute> buildAttributes(Map<String, Integer> columnLookup, String[] values) throws DroneIngestException {
private GeoTrackPoint createTrackPoint(Map<String, Integer> columnLookup, String[] values) throws DroneIngestException {
Double latitude = getDoubleValue(columnLookup.get(HEADER_LAT), values);
Double longitude = getDoubleValue(columnLookup.get(HEADER_LONG), values);
@ -251,13 +256,13 @@ final class DATExtractor extends DroneExtractor {
return null;
}
return makeWaypointAttributes(latitude,
return new GeoArtifactHelper.GeoTrackPoint(latitude,
longitude,
getDoubleValue(columnLookup.get(HEADER_ALTITUDE), values),
getDateTimeValue(columnLookup, values),
getDoubleValue(columnLookup.get(HEADER_VELOCITY), values),
getDoubleValue(columnLookup.get(HEADER_DISTANCE_FROM_HP), values),
getDoubleValue(columnLookup.get(HEADER_DISTANCE_TRAVELED), values));
getDoubleValue(columnLookup.get(HEADER_DISTANCE_TRAVELED), values),
getDateTimeValue(columnLookup, values));
}
/**
@ -291,12 +296,12 @@ final class DATExtractor extends DroneExtractor {
/**
* Returns the string value at the given index parsed as a double.
*
* @param index Index to string array
* @param values Array of string values
*
* @return Double value or null if the index is out of bounds of the string
* array or the string value at index was not a double.
*
* @param index Index to string array
* @param values Array of string values
*
* @return Double value or null if the index is out of bounds of the string
* array or the string value at index was not a double.
*/
private Double getDoubleValue(Integer index, String[] values) {
if (index == null || index == -1 || index > values.length) {

128
Core/src/org/sleuthkit/autopsy/modules/drones/DroneExtractor.java
View File

@ -22,27 +22,21 @@ import java.io.File;
import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Collection;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException;
import org.sleuthkit.autopsy.datamodel.ContentUtils;
import org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProgress;
import org.sleuthkit.autopsy.ingest.IngestJobContext;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.Blackboard;
import org.sleuthkit.datamodel.BlackboardArtifact;
import org.sleuthkit.datamodel.BlackboardAttribute;
import org.sleuthkit.datamodel.Content;
import org.sleuthkit.datamodel.SleuthkitCase;
import org.sleuthkit.datamodel.TskCoreException;
/**
* Abstract base class for all Drone file extractors.
*/
abstract class DroneExtractor {
static private final String TEMP_FOLDER_NAME = "DroneExtractor";
static private final String TEMP_FOLDER_NAME = "DroneExtractor"; //NON-NLS
private final Case currentCase;
/**
@ -54,7 +48,7 @@ abstract class DroneExtractor {
try {
currentCase = Case.getCurrentCaseThrows();
} catch (NoCurrentCaseException ex) {
throw new DroneIngestException("Unable to create drone extractor, no open case.", ex);
throw new DroneIngestException("Unable to create drone extractor, no open case.", ex); //NON-NLS
}
}
@ -80,112 +74,6 @@ abstract class DroneExtractor {
return currentCase.getSleuthkitCase();
}
/**
* Return the Blackboard object.
*
* @return Current Case blackboard object.
*/
final protected Blackboard getBlackboard() {
return currentCase.getSleuthkitCase().getBlackboard();
}
/**
* Method to post a list of BlackboardArtifacts to the blackboard.
*
* @param artifacts A list of artifacts. IF list is empty or null, the
* function will return.
*/
void postArtifacts(Collection<BlackboardArtifact> artifacts) throws DroneIngestException {
if (artifacts == null || artifacts.isEmpty()) {
return;
}
try {
getBlackboard().postArtifacts(artifacts, getName());
} catch (Blackboard.BlackboardException ex) {
throw new DroneIngestException(String.format("Failed to post Drone artifacts to blackboard."), ex);
}
}
/**
* Create a TSK_WAYPOINT artifact with the given list of attributes.
*
* @param DATFile DAT file
* @param attributes List of BlackboardAttributes
*
* @return TSK_WAYPOINT BlackboardArtifact
*
* @throws DroneIngestException
*/
protected BlackboardArtifact makeWaypointArtifact(AbstractFile DATFile, Collection<BlackboardAttribute> attributes) throws DroneIngestException {
try {
BlackboardArtifact artifact = DATFile.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_GPS_TRACKPOINT);
artifact.addAttributes(attributes);
return artifact;
} catch (TskCoreException ex) {
throw new DroneIngestException(String.format("Failed to post Drone artifacts to blackboard."), ex);
}
}
/**
* Returns a list of BlackboardAttributes for the given parameters.
*
* Throws exception of longitude or latitude are null.
*
* @param latitude Waypoint latitude, must be non-null
* @param longitude waypoint longitude, must be non-null
* @param altitude Waypoint altitude\height
* @param dateTime Timestamp the waypoint was created (Java epoch
* seconds)
* @param velocity Velocity
* @param distanceHP Distance from home point
* @param distanceTraveled Total distance the drone has traveled
*
* @return Collection of BlackboardAttributes
*
* @throws DroneIngestException
*/
protected Collection<BlackboardAttribute> makeWaypointAttributes(Double latitude, Double longitude, Double altitude, Long dateTime, Double velocity, Double distanceHP, Double distanceTraveled) throws DroneIngestException {
Collection<BlackboardAttribute> attributes = new ArrayList<>();
if (latitude == null || longitude == null) {
throw new DroneIngestException("Invalid list of waypoint attributes, longitude or latitude was null");
}
attributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_GEO_LATITUDE,
DroneIngestModuleFactory.getModuleName(), latitude));
attributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_GEO_LONGITUDE,
DroneIngestModuleFactory.getModuleName(), longitude));
if (altitude != null) {
attributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_GEO_ALTITUDE,
DroneIngestModuleFactory.getModuleName(), altitude));
}
if (dateTime != null) {
attributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME,
DroneIngestModuleFactory.getModuleName(), dateTime));
}
if (velocity != null) {
attributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_GEO_VELOCITY,
DroneIngestModuleFactory.getModuleName(), velocity));
}
if (distanceHP != null) {
attributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_GPS_DISTANCE_FROM_HOMEPOINT,
DroneIngestModuleFactory.getModuleName(), distanceHP));
}
if (distanceTraveled != null) {
attributes.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_GPS_DISTANCE_TRAVELED,
DroneIngestModuleFactory.getModuleName(), distanceTraveled));
}
return attributes;
}
/**
* Build the temp path and create the directory if it does not currently
* exist.
@ -207,13 +95,13 @@ abstract class DroneExtractor {
/**
* Create a copy of file in the case temp directory.
*
*
* @param context Current ingest context
* @param file File to be copied
*
* @param file File to be copied
*
* @return File copy.
*
* @throws DroneIngestException
*
* @throws DroneIngestException
*/
protected File getTemporaryFile(IngestJobContext context, AbstractFile file) throws DroneIngestException {
String tempFileName = file.getName() + file.getId() + file.getNameExtension();
@ -223,7 +111,7 @@ abstract class DroneExtractor {
try {
ContentUtils.writeToFile(file, tempFilePath.toFile(), context::dataSourceIngestIsCancelled);
} catch (IOException ex) {
throw new DroneIngestException(String.format("Unable to create temp file %s for abstract file %s", tempFilePath.toString(), file.getName()), ex);
throw new DroneIngestException(String.format("Unable to create temp file %s for abstract file %s", tempFilePath.toString(), file.getName()), ex); //NON-NLS
}
return tempFilePath.toFile();

1
Core/src/org/sleuthkit/autopsy/modules/drones/DroneIngestModule.java
View File

@ -56,6 +56,7 @@ public final class DroneIngestModule implements DataSourceIngestModule {
}
@Messages({
"# {0} - AbstractFileName",
"DroneIngestModule_process_start=Started {0}"
})
@Override