mirror of
https://github.com/overcuriousity/autopsy-flatpak.git
synced 2025-07-06 21:00:22 +00:00
cleanup of tskdbdiff and srcupdater.
This commit is contained in:
Jeff Wallace
parent
c902bead63
commit
c697f0458f
@ -17,171 +17,198 @@ import Emailer
|
||||
from regression_utils import *
|
||||
|
||||
def compile(errore, attachli, parsedin):
|
||||
global redo
|
||||
global tryredo
|
||||
global failedbool
|
||||
global errorem
|
||||
errorem = errore
|
||||
global attachl
|
||||
attachl = attachli
|
||||
global passed
|
||||
global parsed
|
||||
parsed = parsedin
|
||||
passed = True
|
||||
tryredo = False
|
||||
redo = True
|
||||
while(redo):
|
||||
passed = True
|
||||
if(passed):
|
||||
gitPull("sleuthkit")
|
||||
if(passed):
|
||||
vsBuild()
|
||||
if(passed):
|
||||
gitPull("autopsy")
|
||||
if(passed):
|
||||
antBuild("datamodel", False)
|
||||
if(passed):
|
||||
antBuild("autopsy", True)
|
||||
if(passed):
|
||||
redo = False
|
||||
else:
|
||||
print("Compile Failed")
|
||||
time.sleep(3600)
|
||||
attachl = []
|
||||
errorem = "The test standard didn't match the gold standard.\n"
|
||||
failedbool = False
|
||||
if(tryredo):
|
||||
errorem = ""
|
||||
errorem += "Rebuilt properly.\n"
|
||||
Emailer.send_email(parsed, errorem, attachl, True)
|
||||
attachl = []
|
||||
passed = True
|
||||
global to
|
||||
global server
|
||||
global subj
|
||||
global email_enabled
|
||||
global redo
|
||||
global tryredo
|
||||
global failedbool
|
||||
global errorem
|
||||
errorem = errore
|
||||
global attachl
|
||||
attachl = attachli
|
||||
global passed
|
||||
global parsed
|
||||
parsed = parsedin
|
||||
passed = True
|
||||
tryredo = False
|
||||
redo = True
|
||||
while(redo):
|
||||
passed = True
|
||||
if(passed):
|
||||
gitPull("sleuthkit")
|
||||
if(passed):
|
||||
vsBuild()
|
||||
print("TSK")
|
||||
if(passed):
|
||||
gitPull("autopsy")
|
||||
if(passed):
|
||||
antBuild("datamodel", False)
|
||||
print("DataModel")
|
||||
if(passed):
|
||||
antBuild("autopsy", True)
|
||||
print("Aut")
|
||||
if(passed):
|
||||
redo = False
|
||||
else:
|
||||
print("Compile Failed")
|
||||
time.sleep(3600)
|
||||
attachl = []
|
||||
errorem = "The test standard didn't match the gold standard.\n"
|
||||
failedbool = False
|
||||
if(tryredo):
|
||||
errorem = ""
|
||||
errorem += "Rebuilt properly.\n"
|
||||
if email_enabled:
|
||||
Emailer.send_email(to, server, subj, errorem, attachl)
|
||||
attachl = []
|
||||
passed = True
|
||||
|
||||
#Pulls from git
|
||||
def gitPull(TskOrAutopsy):
|
||||
global SYS
|
||||
global errorem
|
||||
global attachl
|
||||
ccwd = ""
|
||||
gppth = make_local_path("..", "GitPullOutput" + TskOrAutopsy + ".txt")
|
||||
attachl.append(gppth)
|
||||
gpout = open(gppth, 'a')
|
||||
toPull = "https://www.github.com/sleuthkit/" + TskOrAutopsy
|
||||
call = ["git", "pull", toPull]
|
||||
if TskOrAutopsy == "sleuthkit":
|
||||
ccwd = os.path.join("..", "..", "..", "sleuthkit")
|
||||
else:
|
||||
ccwd = os.path.join("..", "..")
|
||||
subprocess.call(call, stdout=sys.stdout, cwd=ccwd)
|
||||
gpout.close()
|
||||
global SYS
|
||||
global errorem
|
||||
global attachl
|
||||
ccwd = ""
|
||||
gppth = make_local_path("..", "GitPullOutput" + TskOrAutopsy + ".txt")
|
||||
attachl.append(gppth)
|
||||
gpout = open(gppth, 'a')
|
||||
toPull = "https://www.github.com/sleuthkit/" + TskOrAutopsy
|
||||
call = ["git", "pull", toPull]
|
||||
if TskOrAutopsy == "sleuthkit":
|
||||
ccwd = os.path.join("..", "..", "..", "sleuthkit")
|
||||
else:
|
||||
ccwd = os.path.join("..", "..")
|
||||
subprocess.call(call, stdout=sys.stdout, cwd=ccwd)
|
||||
gpout.close()
|
||||
|
||||
|
||||
#Builds TSK as a win32 applicatiion
|
||||
def vsBuild():
|
||||
global redo
|
||||
global tryredo
|
||||
global passed
|
||||
global parsed
|
||||
#Please ensure that the current working directory is $autopsy/testing/script
|
||||
oldpath = os.getcwd()
|
||||
os.chdir(os.path.join("..", "..", "..","sleuthkit", "win32"))
|
||||
vs = []
|
||||
vs.append("/cygdrive/c/windows/microsoft.NET/framework/v4.0.30319/MSBuild.exe")
|
||||
vs.append(os.path.join("Tsk-win.sln"))
|
||||
vs.append("/p:configuration=release")
|
||||
vs.append("/p:platform=win32")
|
||||
vs.append("/t:clean")
|
||||
vs.append("/t:rebuild")
|
||||
print(vs)
|
||||
VSpth = make_local_path("..", "VSOutput.txt")
|
||||
VSout = open(VSpth, 'a')
|
||||
subprocess.call(vs, stdout=VSout)
|
||||
VSout.close()
|
||||
os.chdir(oldpath)
|
||||
chk = os.path.join("..", "..", "..","sleuthkit", "win32", "Release", "libtsk_jni.dll")
|
||||
try:
|
||||
open(chk)
|
||||
except IOError as e:
|
||||
global errorem
|
||||
global attachl
|
||||
if(not tryredo):
|
||||
errorem += "LIBTSK C++ failed to build.\n"
|
||||
attachl.append(VSpth)
|
||||
send_email(parsed, errorem, attachl, False)
|
||||
tryredo = True
|
||||
passed = False
|
||||
redo = True
|
||||
global redo
|
||||
global tryredo
|
||||
global passed
|
||||
global parsed
|
||||
#Please ensure that the current working directory is $autopsy/testing/script
|
||||
oldpath = os.getcwd()
|
||||
os.chdir(os.path.join("..", "..", "..","sleuthkit", "win32"))
|
||||
vs = []
|
||||
vs.append("/cygdrive/c/windows/microsoft.NET/framework/v4.0.30319/MSBuild.exe")
|
||||
vs.append(os.path.join("Tsk-win.sln"))
|
||||
vs.append("/p:configuration=release")
|
||||
vs.append("/p:platform=x64")
|
||||
vs.append("/t:clean")
|
||||
vs.append("/t:rebuild")
|
||||
print(vs)
|
||||
VSpth = make_local_path("..", "VSOutput.txt")
|
||||
VSout = open(VSpth, 'a')
|
||||
subprocess.call(vs, stdout=VSout)
|
||||
VSout.close()
|
||||
os.chdir(oldpath)
|
||||
chk = os.path.join("..", "..", "..","sleuthkit", "win32", "x64", "Release", "libtsk_jni.dll")
|
||||
if not os.path.exists(chk):
|
||||
print("path doesn't exist")
|
||||
global errorem
|
||||
global attachl
|
||||
global email_enabled
|
||||
if(not tryredo):
|
||||
errorem += "LIBTSK C++ failed to build.\n"
|
||||
attachl.append(VSpth)
|
||||
if email_enabled:
|
||||
Emailer.send_email(parsed, errorem, attachl, False)
|
||||
tryredo = True
|
||||
passed = False
|
||||
redo = True
|
||||
|
||||
|
||||
|
||||
#Builds Autopsy or the Datamodel
|
||||
def antBuild(which, Build):
|
||||
global redo
|
||||
global passed
|
||||
global tryredo
|
||||
global parsed
|
||||
directory = os.path.join("..", "..")
|
||||
ant = []
|
||||
if which == "datamodel":
|
||||
directory = os.path.join("..", "..", "..", "sleuthkit", "bindings", "java")
|
||||
ant.append("ant")
|
||||
ant.append("-f")
|
||||
ant.append(directory)
|
||||
ant.append("clean")
|
||||
if(Build):
|
||||
ant.append("build")
|
||||
else:
|
||||
ant.append("dist")
|
||||
antpth = make_local_path("..", "ant" + which + "Output.txt")
|
||||
antout = open(antpth, 'a')
|
||||
succd = subprocess.call(ant, stdout=antout)
|
||||
antout.close()
|
||||
global errorem
|
||||
global attachl
|
||||
if which == "datamodel":
|
||||
chk = os.path.join("..", "..", "..","sleuthkit", "bindings", "java", "dist", "TSK_DataModel.jar")
|
||||
try:
|
||||
open(chk)
|
||||
except IOError as e:
|
||||
if(not tryredo):
|
||||
errorem += "DataModel Java build failed.\n"
|
||||
attachl.append(antpth)
|
||||
Emailer.send_email(parsed, errorem, attachl, False)
|
||||
passed = False
|
||||
tryredo = True
|
||||
elif (succd != 0 and (not tryredo)):
|
||||
errorem += "Autopsy build failed.\n"
|
||||
attachl.append(antpth)
|
||||
Emailer.send_email(parsed, errorem, attachl, False)
|
||||
tryredo = True
|
||||
elif (succd != 0):
|
||||
passed = False
|
||||
print("building: ", which)
|
||||
global redo
|
||||
global passed
|
||||
global tryredo
|
||||
global parsed
|
||||
directory = os.path.join("..", "..")
|
||||
ant = []
|
||||
if which == "datamodel":
|
||||
directory = os.path.join("..", "..", "..", "sleuthkit", "bindings", "java")
|
||||
ant.append("ant")
|
||||
ant.append("-f")
|
||||
ant.append(directory)
|
||||
ant.append("clean")
|
||||
if(Build):
|
||||
ant.append("build")
|
||||
else:
|
||||
ant.append("dist")
|
||||
antpth = make_local_path("..", "ant" + which + "Output.txt")
|
||||
antout = open(antpth, 'a')
|
||||
succd = subprocess.call(ant, stdout=antout)
|
||||
antout.close()
|
||||
global errorem
|
||||
global attachl
|
||||
global email_enabled
|
||||
global to
|
||||
global subj
|
||||
global server
|
||||
if which == "datamodel":
|
||||
chk = os.path.join("..", "..", "..","sleuthkit", "bindings", "java", "dist", "TSK_DataModel.jar")
|
||||
try:
|
||||
open(chk)
|
||||
except IOError as e:
|
||||
if(not tryredo):
|
||||
errorem += "DataModel Java build failed.\n"
|
||||
attachl.append(antpth)
|
||||
if email_enabled:
|
||||
Emailer.send_email(to, server, subj, errorem, attachl)
|
||||
passed = False
|
||||
tryredo = True
|
||||
elif (succd != 0 and (not tryredo)):
|
||||
errorem += "Autopsy build failed.\n"
|
||||
attachl.append(antpth)
|
||||
Emailer.send_email(to, server, subj, errorem, attachl)
|
||||
tryredo = True
|
||||
elif (succd != 0):
|
||||
passed = False
|
||||
|
||||
|
||||
def main():
|
||||
errore = ""
|
||||
attachli = []
|
||||
config_file = ""
|
||||
arg = sys.argv.pop(0)
|
||||
arg = sys.argv.pop(0)
|
||||
config_file = arg
|
||||
parsedin = parse(config_file)
|
||||
compile(errore, attachli, parsedin)
|
||||
global email_enabled
|
||||
global to
|
||||
global server
|
||||
global subj
|
||||
errore = ""
|
||||
attachli = []
|
||||
config_file = ""
|
||||
arg = sys.argv.pop(0)
|
||||
arg = sys.argv.pop(0)
|
||||
config_file = arg
|
||||
parsedin = parse(config_file)
|
||||
try:
|
||||
to = parsedin.getElementsByTagName("email")[0].getAttribute("value").encode().decode("utf_8")
|
||||
server = parsedin.getElementsByTagName("mail_server")[0].getAttribute("value").encode().decode("utf_8")
|
||||
subj = parsedin.getElementsByTagName("subject")[0].getAttribute("value").encode().decode("utf_8")
|
||||
except Exception:
|
||||
email_enabled = False
|
||||
# email_enabled = (to is not None) and (server is not None) and (subj is not None)
|
||||
email_enabled = False
|
||||
compile(errore, attachli, parsedin)
|
||||
|
||||
class OS:
|
||||
LINUX, MAC, WIN, CYGWIN = range(4)
|
||||
if __name__ == "__main__":
|
||||
global SYS
|
||||
if _platform == "linux" or _platform == "linux2":
|
||||
SYS = OS.LINUX
|
||||
elif _platform == "darwin":
|
||||
SYS = OS.MAC
|
||||
elif _platform == "win32":
|
||||
SYS = OS.WIN
|
||||
elif _platform == "cygwin":
|
||||
SYS = OS.CYGWIN
|
||||
global SYS
|
||||
if _platform == "linux" or _platform == "linux2":
|
||||
SYS = OS.LINUX
|
||||
elif _platform == "darwin":
|
||||
SYS = OS.MAC
|
||||
elif _platform == "win32":
|
||||
SYS = OS.WIN
|
||||
elif _platform == "cygwin":
|
||||
SYS = OS.CYGWIN
|
||||
|
||||
if SYS is OS.WIN or SYS is OS.CYGWIN:
|
||||
main()
|
||||
else:
|
||||
print("We only support Windows and Cygwin at this time.")
|
||||
if SYS is OS.WIN or SYS is OS.CYGWIN:
|
||||
main()
|
||||
else:
|
||||
print("We only support Windows and Cygwin at this time.")
|
||||
|
||||
@ -137,37 +137,39 @@ class TskDbDiff(object):
|
||||
unsorted_dump = TskDbDiff._get_tmp_file("dump_data", ".txt")
|
||||
conn = sqlite3.connect(db_file)
|
||||
conn.text_factory = lambda x: x.decode("utf-8", "ignore")
|
||||
autopsy_cur2 = conn.cursor()
|
||||
conn.row_factory = sqlite3.Row
|
||||
artifact_cursor = conn.cursor()
|
||||
# Get the list of all artifacts
|
||||
# @@@ Could add a SORT by parent_path in here since that is how we are going to later sort it.
|
||||
autopsy_cur2.execute("SELECT tsk_files.parent_path, tsk_files.name, blackboard_artifact_types.display_name, blackboard_artifacts.artifact_id FROM blackboard_artifact_types INNER JOIN blackboard_artifacts ON blackboard_artifact_types.artifact_type_id = blackboard_artifacts.artifact_type_id INNER JOIN tsk_files ON tsk_files.obj_id = blackboard_artifacts.obj_id")
|
||||
artifact_cursor.execute("SELECT tsk_files.parent_path, tsk_files.name, blackboard_artifact_types.display_name, blackboard_artifacts.artifact_id FROM blackboard_artifact_types INNER JOIN blackboard_artifacts ON blackboard_artifact_types.artifact_type_id = blackboard_artifacts.artifact_type_id INNER JOIN tsk_files ON tsk_files.obj_id = blackboard_artifacts.obj_id")
|
||||
database_log = codecs.open(unsorted_dump, "wb", "utf_8")
|
||||
rw = autopsy_cur2.fetchone()
|
||||
row = artifact_cursor.fetchone()
|
||||
appnd = False
|
||||
counter = 0
|
||||
artifact_count = 0
|
||||
artifact_fail = 0
|
||||
# Cycle through artifacts
|
||||
try:
|
||||
while (rw != None):
|
||||
while (row != None):
|
||||
# File Name and artifact type
|
||||
if(rw[0] != None):
|
||||
database_log.write(rw[0] + rw[1] + ' <artifact type="' + rw[2] + '" > ')
|
||||
if(row["parent_path"] != None):
|
||||
database_log.write(row["parent_path"] + row["name"] + ' <artifact type="' + row["display_name"] + '" > ')
|
||||
else:
|
||||
database_log.write(rw[1] + ' <artifact type="' + rw[2] + '" > ')
|
||||
database_log.write(row["name"] + ' <artifact type="' + row["display_name"] + '" > ')
|
||||
|
||||
# Get attributes for this artifact
|
||||
autopsy_cur1 = conn.cursor()
|
||||
attribute_cursor = conn.cursor()
|
||||
looptry = True
|
||||
artifact_count += 1
|
||||
try:
|
||||
key = ""
|
||||
key = str(rw[3])
|
||||
key = key,
|
||||
autopsy_cur1.execute("SELECT blackboard_attributes.source, blackboard_attribute_types.display_name, blackboard_attributes.value_type, blackboard_attributes.value_text, blackboard_attributes.value_int32, blackboard_attributes.value_int64, blackboard_attributes.value_double FROM blackboard_attributes INNER JOIN blackboard_attribute_types ON blackboard_attributes.attribute_type_id = blackboard_attribute_types.attribute_type_id WHERE artifact_id =? ORDER BY blackboard_attributes.source, blackboard_attribute_types.display_name, blackboard_attributes.value_type, blackboard_attributes.value_text, blackboard_attributes.value_int32, blackboard_attributes.value_int64, blackboard_attributes.value_double", key)
|
||||
attributes = autopsy_cur1.fetchall()
|
||||
art_id = ""
|
||||
art_id = str(row["artifact_id"])
|
||||
attribute_cursor.execute("SELECT blackboard_attributes.source, blackboard_attribute_types.display_name, blackboard_attributes.value_type, blackboard_attributes.value_text, blackboard_attributes.value_int32, blackboard_attributes.value_int64, blackboard_attributes.value_double FROM blackboard_attributes INNER JOIN blackboard_attribute_types ON blackboard_attributes.attribute_type_id = blackboard_attribute_types.attribute_type_id WHERE artifact_id =? ORDER BY blackboard_attributes.source, blackboard_attribute_types.display_name, blackboard_attributes.value_type, blackboard_attributes.value_text, blackboard_attributes.value_int32, blackboard_attributes.value_int64, blackboard_attributes.value_double", [art_id])
|
||||
attributes = attribute_cursor.fetchall()
|
||||
except sqlite3.Error as e:
|
||||
msg ="Attributes in artifact id (in output DB)# " + str(rw[3]) + " encountered an error: " + str(e) +" .\n"
|
||||
msg = "Attributes in artifact id (in output DB)# " + str(row["artifact_id"]) + " encountered an error: " + str(e) +" .\n"
|
||||
print("Attributes in artifact id (in output DB)# ", str(row["artifact_id"]), " encountered an error: ", str(e))
|
||||
print()
|
||||
looptry = False
|
||||
artifact_fail += 1
|
||||
database_log.write('Error Extracting Attributes')
|
||||
@ -178,33 +180,29 @@ class TskDbDiff(object):
|
||||
if(looptry == True):
|
||||
src = attributes[0][0]
|
||||
for attr in attributes:
|
||||
val = 3 + attr[2]
|
||||
attr_value_index = 3 + attr["value_type"]
|
||||
numvals = 0
|
||||
for x in range(3, 6):
|
||||
if(attr[x] != None):
|
||||
numvals += 1
|
||||
if(numvals > 1):
|
||||
msg = "There were too many values for attribute type: " + attr[1] + " for artifact with id #" + str(rw[3]) + ".\n"
|
||||
if(not attr[0] == src):
|
||||
msg ="There were inconsistent sources for artifact with id #" + str(rw[3]) + ".\n"
|
||||
msg = "There were too many values for attribute type: " + attr["display_name"] + " for artifact with id #" + str(row["artifact_id"]) + ".\n"
|
||||
if(not attr["source"] == src):
|
||||
msg = "There were inconsistent sources for artifact with id #" + str(row["artifact_id"]) + ".\n"
|
||||
try:
|
||||
database_log.write('<attribute source="' + attr[0] + '" type="' + attr[1] + '" value="')
|
||||
inpval = attr[val]
|
||||
if((type(inpval) != 'unicode') or (type(inpval) != 'str')):
|
||||
inpval = str(inpval)
|
||||
patrn = re.compile("[\n\0\a\b\r\f\e]")
|
||||
inpval = re.sub(patrn, ' ', inpval)
|
||||
database_log.write(inpval)
|
||||
attr_value_as_string = str(attr[attr_value_index])
|
||||
#if((type(attr_value_as_string) != 'unicode') or (type(attr_value_as_string) != 'str')):
|
||||
# attr_value_as_string = str(attr_value_as_string)
|
||||
patrn = re.compile("[\n\0\a\b\r\f]")
|
||||
attr_value_as_string = re.sub(patrn, ' ', attr_value_as_string)
|
||||
database_log.write('<attribute source="' + attr["source"] + '" type="' + attr["display_name"] + '" value="' + attr_value_as_string + '" />')
|
||||
except IOError as e:
|
||||
print("IO error")
|
||||
raise TskDbDiffException("Unexpected IO error while writing to database log." + str(e))
|
||||
|
||||
database_log.write('" />')
|
||||
database_log.write(' <artifact/>\n')
|
||||
rw = autopsy_cur2.fetchone()
|
||||
row = artifact_cursor.fetchone()
|
||||
|
||||
# Now sort the file
|
||||
srtcmdlst = ["sort", unsorted_dump, "-o", bb_dump_file]
|
||||
subprocess.call(srtcmdlst)
|
||||
print(artifact_fail)
|
||||
if(artifact_fail > 0):
|
||||
msg ="There were " + str(artifact_count) + " artifacts and " + str(artifact_fail) + " threw an exception while loading.\n"
|
||||
@ -213,6 +211,10 @@ class TskDbDiff(object):
|
||||
finally:
|
||||
database_log.close()
|
||||
|
||||
# Now sort the file
|
||||
srtcmdlst = ["sort", unsorted_dump, "-o", bb_dump_file]
|
||||
subprocess.call(srtcmdlst)
|
||||
|
||||
def _dump_output_db_nonbb(db_file, dump_file):
|
||||
"""Dumps a database to a text file.
|
||||
|
||||
@ -267,14 +269,14 @@ def main():
|
||||
print("usage: tskdbdiff [OUPUT DB PATH] [GOLD DB PATH]")
|
||||
sys.exit()
|
||||
|
||||
db_diff = TskDbDiff(output_db, gold_db)
|
||||
db_diff = TskDbDiff(output_db, gold_db, output_dir=".")
|
||||
dump_passed, bb_dump_passed = db_diff.run_diff()
|
||||
|
||||
if dump_passed and bb_dump_passed:
|
||||
print("Database comparison passed.")
|
||||
elif not dump_passed:
|
||||
if not dump_passed:
|
||||
print("Non blackboard database comparison failed.")
|
||||
elif not bb_dump_passed:
|
||||
if not bb_dump_passed:
|
||||
print("Blackboard database comparison failed.")
|
||||
|
||||
return 0
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user