mstoeck3/autopsy-flatpak
1
0
Fork 0
mirror of https://github.com/overcuriousity/autopsy-flatpak.git synced 2025-07-06 21:00:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #5684 from rcordovano/gpx-module-fix-for-api-changes

Browse Source 
6105 GPX Parser module fixes for API changes, bugs
This commit is contained in:
Richard Cordovano 2020-03-06 11:20:43 -05:00 committed by GitHub
commit febe04d9d9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 79 additions and 128 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

207
InternalPythonModules/GPX_Module/GPX_Parser_Module.py
View File

@ -37,8 +37,12 @@ from org.sleuthkit.datamodel import BlackboardArtifact
from org.sleuthkit.datamodel import BlackboardAttribute from org.sleuthkit.datamodel import BlackboardAttribute
from org.sleuthkit.datamodel import TskCoreException from org.sleuthkit.datamodel import TskCoreException
from org.sleuthkit.datamodel.blackboardutils import GeoArtifactsHelper from org.sleuthkit.datamodel.blackboardutils import GeoArtifactsHelper
from org.sleuthkit.datamodel.blackboardutils.attributes import GeoWaypoint from org.sleuthkit.datamodel.blackboardutils.attributes import TskGeoWaypointsUtil
from org.sleuthkit.datamodel.blackboardutils.attributes import GeoTrackPoints from org.sleuthkit.datamodel.blackboardutils.attributes.TskGeoWaypointsUtil import GeoWaypointList
from org.sleuthkit.datamodel.blackboardutils.attributes.TskGeoWaypointsUtil.GeoWaypointList import GeoWaypoint
from org.sleuthkit.datamodel.blackboardutils.attributes import TskGeoTrackpointsUtil
from org.sleuthkit.datamodel.blackboardutils.attributes.TskGeoTrackpointsUtil import GeoTrackPointList
from org.sleuthkit.datamodel.blackboardutils.attributes.TskGeoTrackpointsUtil.GeoTrackPointList import GeoTrackPoint
from org.sleuthkit.autopsy.datamodel import ContentUtils from org.sleuthkit.autopsy.datamodel import ContentUtils
from org.sleuthkit.autopsy.ingest import IngestModule from org.sleuthkit.autopsy.ingest import IngestModule
from org.sleuthkit.autopsy.ingest.IngestModule import IngestModuleException from org.sleuthkit.autopsy.ingest.IngestModule import IngestModuleException
@ -63,20 +67,15 @@ import gpxpy.parser
class GPXParserDataSourceIngestModuleFactory(IngestModuleFactoryAdapter): class GPXParserDataSourceIngestModuleFactory(IngestModuleFactoryAdapter):
moduleName = "GPX Parser" moduleName = "GPX Parser"
# True - Verbose debugging messages sent to log file.
# False - Verbose debugging turned off.
debuglevel = False
def getModuleDisplayName(self): def getModuleDisplayName(self):
return self.moduleName return self.moduleName
# TODO: Give it a description
def getModuleDescription(self): def getModuleDescription(self):
return "Module that extracts GEO data from GPX files." return "Module that extracts GEO data from GPX files."
def getModuleVersionNumber(self): def getModuleVersionNumber(self):
return "1.1" return "1.2"
def isDataSourceIngestModuleFactory(self): def isDataSourceIngestModuleFactory(self):
return True return True
@ -88,10 +87,11 @@ class GPXParserDataSourceIngestModuleFactory(IngestModuleFactoryAdapter):
# Data Source-level ingest module. One gets created per data source. # Data Source-level ingest module. One gets created per data source.
class GPXParserDataSourceIngestModule(DataSourceIngestModule): class GPXParserDataSourceIngestModule(DataSourceIngestModule):
_logger = Logger.getLogger(GPXParserDataSourceIngestModuleFactory.moduleName) logger = Logger.getLogger(GPXParserDataSourceIngestModuleFactory.moduleName)
writeDebugMsgs = False
def log(self, level, msg): def log(self, level, msg):
self._logger.logp(level, self.__class__.__name__, inspect.stack()[1][3], msg) self.logger.logp(level, self.__class__.__name__, inspect.stack()[1][3], msg)
def __init__(self): def __init__(self):
self.context = None self.context = None
@ -105,179 +105,130 @@ class GPXParserDataSourceIngestModule(DataSourceIngestModule):
# We don't know how much work there is yet. # We don't know how much work there is yet.
progressBar.switchToIndeterminate() progressBar.switchToIndeterminate()
# This will work in 4.0.1 and beyond.
# Use blackboard class to index blackboard artifacts for keyword search.
blackboard = Case.getCurrentCase().getServices().getBlackboard()
# Get the sleuthkitcase # Get the case database and its blackboard.
skCase = Case.getCurrentCase().getSleuthkitCase() skCase = Case.getCurrentCase().getSleuthkitCase()
blackboard = skCase.getBlackboard()
# In the name and then count and read them. # Get any files with a .gpx extension.
fileManager = Case.getCurrentCase().getServices().getFileManager() # It would perhaps be better to get these files by MIME type instead.
# RC: It would also be better if this were a file level ingest module so it could process files extracted from archives.
fileManager = Case.getCurrentCase().getServices().getFileManager()
files = fileManager.findFiles(dataSource, "%.gpx") files = fileManager.findFiles(dataSource, "%.gpx")
# TODO: Would like to change this to find files based on mimetype rather than extension.
#files = findFiles(dataSource, "text/xml")
#if (file.isMimeType('text/xml') == False):
# Update the progress bar now that we know how much work there is to do.
numFiles = len(files) numFiles = len(files)
if GPXParserDataSourceIngestModuleFactory.debuglevel: self.log(Level.INFO, "found " + str(numFiles) + " files") if self.writeDebugMsgs: self.log(Level.INFO, "Found " + str(numFiles) + " GPX files")
progressBar.switchToDeterminate(numFiles) progressBar.switchToDeterminate(numFiles)
fileCount = 0;
# Get module name for adding attributes # Get the module name, it will be needed for adding attributes
moduleName = GPXParserDataSourceIngestModuleFactory.moduleName moduleName = GPXParserDataSourceIngestModuleFactory.moduleName
# Check if a folder for this module is present in the case Temp directory.
# If not, create it.
dirName = os.path.join(Case.getCurrentCase().getTempDirectory(), "GPX_Parser_Module")
try:
os.stat(dirName)
except:
os.mkdir(dirName)
# Create a temp file name. It appears that we cannot close and delete
# this file, but we can overwrite it for each file we need to process.
fileName = os.path.join(dirName, "tmp.gpx")
fileCount = 0;
for file in files: for file in files:
# Get the GeoArtifactsHelper # Create a GeoArtifactsHelper for this file.
geoArtifactHelper = GeoArtifactsHelper(skCase, moduleName, file) geoArtifactHelper = GeoArtifactsHelper(skCase, moduleName, None, file)
# Check if the user pressed cancel while we were busy. # Check if the user pressed cancel while we were busy.
if self.context.isJobCancelled(): if self.context.isJobCancelled():
return IngestModule.ProcessResult.OK return IngestModule.ProcessResult.OK
#self.log(Level.INFO, "GPX: Processing file: " + file.getName()) if self.writeDebugMsgs: self.log(Level.INFO, "Processing " + file.getUniquePath() + " (objID = " + str(file.getId()) + ")")
fileCount += 1 fileCount += 1
# Check if module folder is present. If not, create it. # Write the file so that it can be parsed by gpxpy.
dirName = os.path.join(Case.getCurrentCase().getTempDirectory(), "GPX_Parser_Module")
try:
os.stat(dirName)
except:
os.mkdir(dirName)
fileName = os.path.join(dirName, "tmp.gpx")
# Check to see if temporary file exists. If it does, remove it.
if os.path.exists(fileName):
try:
os.remove(fileName)
if GPXParserDataSourceIngestModuleFactory.debuglevel: self.log(Level.INFO, "GPX:\t" + "FILE DELETED " + fileName )
except:
if GPXParserDataSourceIngestModuleFactory.debuglevel: self.log(Level.INFO, "GPX:\t" + "FILE NOT DELETED " + fileName)
# This writes the file to the local file system.
localFile = File(fileName) localFile = File(fileName)
ContentUtils.writeToFile(file, localFile) ContentUtils.writeToFile(file, localFile)
# Send to gpxpy for parsing. # Send the file to gpxpy for parsing.
gpxfile = open(fileName) gpxfile = open(fileName)
try: try:
gpx = gpxpy.parse(gpxfile) gpx = gpxpy.parse(gpxfile)
if GPXParserDataSourceIngestModuleFactory.debuglevel: self.log(Level.INFO, "GPX:\t" + "FILE PARSED") if self.writeDebugMsgs: self.log(Level.INFO, "Parsed " + file.getUniquePath() + " (objID = " + str(file.getId()) + ")")
except: except Exception as e:
if GPXParserDataSourceIngestModuleFactory.debuglevel: self.log(Level.SEVERE, "GPX:\t" + file.getName() + " - FILE NOT PARSED") self.log(Level.WARNING, "Error parsing file " + file.getUniquePath() + " (objID = " + str(file.getId()) + "):" + str(e))
continue continue
if gpx: if gpx:
if GPXParserDataSourceIngestModuleFactory.debuglevel: self.log(Level.INFO, "GPX: TRACKS") if self.writeDebugMsgs: self.log(Level.INFO, "Processing tracks from " + file.getUniquePath() + " (objID = " + str(file.getId()) + ")")
for track in gpx.tracks: for track in gpx.tracks:
for segment in track.segments: for segment in track.segments:
geoPointList = ArrayList() geoPointList = TskGeoTrackpointsUtil.GeoTrackPointList()
for point in segment.points: for point in segment.points:
elevation = 0 elevation = 0
if point.elevation != None: if point.elevation != None:
elevation = point.elevation elevation = point.elevation
dateTime = 0 timeStamp = 0
try: try:
if (point.time != None): if (point.time != None):
datetime = long(time.mktime(point.time.timetuple())) timeStamp = long(time.mktime(point.time.timetuple()))
except: except Exception as e:
pass self.log(Level.WARNING, "Error getting track timestamp from " + file.getUniquePath() + " (objID = " + str(file.getId()) + "):" + str(e))
geoPointList.add(GeoWaypoint.GeoTrackPoint(point.latitude, point.longitude, elevation, 0, 0, 0, dateTime)) geoPointList.addPoint(GeoTrackPoint(point.latitude, point.longitude, elevation, None, 0, 0, 0, timeStamp))
try: try:
# Add the trackpoint using the helper class geoArtifactHelper.addTrack("Track", geoPointList, None)
geoartifact = geoArtifactHelper.addTrack("Trackpoint", geoPointList)
except Blackboard.BlackboardException as e: except Blackboard.BlackboardException as e:
if GPXParserDataSourceIngestModuleFactory.debuglevel: self.log(Level.SEVERE, "GPX: Error using geo artifact helper with blackboard " ) self.log(Level.SEVERE, "Error posting GPS track artifact for " + file.getUniquePath() + " (objID = " + str(file.getId()) + "):" + e.getMessage())
except TskCoreException as e: except TskCoreException as e:
if GPXParserDataSourceIngestModuleFactory.debuglevel: self.log(Level.SEVERE, "GPX: Error using geo artifact helper tskcoreexception" ) self.log(Level.SEVERE, "Error creating GPS track artifact for " + file.getUniquePath() + " (objID = " + str(file.getId()) + "):" + e.getMessage())
if GPXParserDataSourceIngestModuleFactory.debuglevel: self.log(Level.INFO, "GPX: WAYPOINTS") if self.writeDebugMsgs: self.log(Level.INFO, "Processing waypoints from " + file.getUniquePath() + " (objID = " + str(file.getId()) + ")")
for waypoint in gpx.waypoints: for waypoint in gpx.waypoints:
attributes = ArrayList()
art = file.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_GPS_BOOKMARK)
attributes.add(BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_GEO_LATITUDE.getTypeID(), moduleName, waypoint.latitude))
attributes.add(BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_GEO_LONGITUDE.getTypeID(), moduleName, waypoint.longitude))
attributes.add(BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_FLAG.getTypeID(), moduleName, "Waypoint"))
attributes.add(BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_NAME.getTypeID(), moduleName, waypoint.name))
attributes.add(BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PROG_NAME.getTypeID(), moduleName, "GPXParser"))
art.addAttributes(attributes)
try: try:
# Post the artifact to blackboard art = file.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_GPS_BOOKMARK)
skCase.getBlackboard().postArtifact(art, moduleName)
attributes = ArrayList()
attributes.add(BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_GEO_LATITUDE.getTypeID(), moduleName, waypoint.latitude))
attributes.add(BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_GEO_LONGITUDE.getTypeID(), moduleName, waypoint.longitude))
attributes.add(BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_FLAG.getTypeID(), moduleName, "Waypoint"))
attributes.add(BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_NAME.getTypeID(), moduleName, waypoint.name))
attributes.add(BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PROG_NAME.getTypeID(), moduleName, "GPXParser"))
art.addAttributes(attributes)
blackboard.postArtifact(art, moduleName)
except Blackboard.BlackboardException as e: except Blackboard.BlackboardException as e:
if GPXParserDataSourceIngestModuleFactory.debuglevel: self.log(Level.SEVERE, "GPX: Error using geo artifact helper with blackboard for waypoints" ) self.log(Level.SEVERE, "Error posting GPS bookmark artifact for " + file.getUniquePath() + " (objID = " + str(file.getId()) + "):" + e.getMessage())
except TskCoreException as e:
self.log(Level.SEVERE, "Error creating GPS bookmark artifact for " + file.getUniquePath() + " (objID = " + str(file.getId()) + "):" + e.getMessage())
if GPXParserDataSourceIngestModuleFactory.debuglevel: self.log(Level.INFO, "GPX: ROUTES") if self.writeDebugMsgs: self.log(Level.INFO, "Processing routes from " + file.getUniquePath() + " (objID = " + str(file.getId()) + ")")
for route in gpx.routes: for route in gpx.routes:
firstTimeThru = 0
startingPoint = list() geoWaypointList = TskGeoWaypointsUtil.GeoWaypointList()
endingPoint = list()
for point in route.points: for point in route.points:
# If first time in loop only populate starting point geoWaypointList.addPoint(point.latitude, point.longitude, elevation, point.name)
if (firstTimeThru == 0):
startingPoint.append((point.latitude, point.longitude))
firstTimeThru = 1
else:
startingPoint.append((point.latitude, point.longitude))
endingPoint.append((point.latitude, point.longitude))
if (len(endingPoint) > 0): try:
# get length of ending point as this ensures that we have equal points to process. geoArtifactHelper.addRoute(None, None, geoWaypointList, None)
for i in range(0,len(endingPoint) -1): except Blackboard.BlackboardException as e:
attributes = ArrayList() self.log("Error posting GPS route artifact for " + file.getUniquePath() + " (objID = " + str(file.getId()) + "):" + e.getMessage())
art = file.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_GPS_ROUTE) except TskCoreException as e:
self.log(Level.SEVERE, "Error creating GPS route artifact for " + file.getUniquePath() + " (objID = " + str(file.getId()) + "):" + e.getMessage())
attributes.add(BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_GEO_LATITUDE_START.getTypeID(), moduleName, startingPoint[i][0]))
attributes.add(BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_GEO_LONGITUDE_START.getTypeID(), moduleName, startingPoint[i][1]))
attributes.add(BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_GEO_LATITUDE_END.getTypeID(), moduleName, endingPoint[i][0]))
attributes.add(BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_GEO_LONGITUDE_END.getTypeID(), moduleName, endingPoint[i][1]))
art.addAttributes(attributes)
try:
# Post the artifact to blackboard
skCase.getBlackboard().postArtifact(art, moduleName)
except Blackboard.BlackboardException as e:
if GPXParserDataSourceIngestModuleFactory.debuglevel: self.log(Level.SEVERE, "GPX: Error using geo artifact helper with blackboard for waypoints" )
else:
if (len(startingPoint) > 0):
attributes = ArrayList()
art = file.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_GPS_ROUTE)
attributes.add(BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_GEO_LATITUDE_START.getTypeID(), moduleName, startingPoint[0][0]))
attributes.add(BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_GEO_LONGITUDE_START.getTypeID(), moduleName, startingPoint[0][1]))
attributes.add(BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_GEO_LATITUDE_END.getTypeID(), moduleName, startingPoint[0][0]))
attributes.add(BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_GEO_LONGITUDE_END.getTypeID(), moduleName, startingPoint[0][1]))
art.addAttributes(attributes)
try:
# Post the artifact to blackboard
skCase.getBlackboard().postArtifact(art, moduleName)
except Blackboard.BlackboardException as e:
if GPXParserDataSourceIngestModuleFactory.debuglevel: self.log(Level.SEVERE, "GPX: Error using geo artifact helper with blackboard for waypoints" )
# Update the progress bar. # Update the progress bar.
progressBar.progress(fileCount) progressBar.progress(fileCount)
if os.path.exists(fileName):
try:
os.remove(fileName)
if GPXParserDataSourceIngestModuleFactory.debuglevel: self.log(Level.INFO, "GPX:\t" + "FILE DELETED")
except:
self.log(Level.SEVERE, "GPX:\t" + "FILE NOT DELETED")
# Post a message to the ingest messages inbox. # Post a message to the ingest messages inbox.
message = IngestMessage.createMessage(IngestMessage.MessageType.DATA, "GPX Parser Data Source Ingest Module", "Found %d files" % fileCount) message = IngestMessage.createMessage(IngestMessage.MessageType.DATA, moduleName, "Processed %d files" % fileCount)
IngestServices.getInstance().postMessage(message) IngestServices.getInstance().postMessage(message)
return IngestModule.ProcessResult.OK; return IngestModule.ProcessResult.OK;