From c697f0458f0a6cfef6c888454a749460b510141c Mon Sep 17 00:00:00 2001 From: Jeff Wallace Date: Wed, 30 Oct 2013 10:33:59 -0400 Subject: [PATCH] cleanup of tskdbdiff and srcupdater. --- test/script/srcupdater.py | 323 +++++++++++++++++++++----------------- test/script/tskdbdiff.py | 70 +++++---- 2 files changed, 211 insertions(+), 182 deletions(-) diff --git a/test/script/srcupdater.py b/test/script/srcupdater.py index 99a393d9eb..eab46604b6 100644 --- a/test/script/srcupdater.py +++ b/test/script/srcupdater.py @@ -17,171 +17,198 @@ import Emailer from regression_utils import * def compile(errore, attachli, parsedin): - global redo - global tryredo - global failedbool - global errorem - errorem = errore - global attachl - attachl = attachli - global passed - global parsed - parsed = parsedin - passed = True - tryredo = False - redo = True - while(redo): - passed = True - if(passed): - gitPull("sleuthkit") - if(passed): - vsBuild() - if(passed): - gitPull("autopsy") - if(passed): - antBuild("datamodel", False) - if(passed): - antBuild("autopsy", True) - if(passed): - redo = False - else: - print("Compile Failed") - time.sleep(3600) - attachl = [] - errorem = "The test standard didn't match the gold standard.\n" - failedbool = False - if(tryredo): - errorem = "" - errorem += "Rebuilt properly.\n" - Emailer.send_email(parsed, errorem, attachl, True) - attachl = [] - passed = True + global to + global server + global subj + global email_enabled + global redo + global tryredo + global failedbool + global errorem + errorem = errore + global attachl + attachl = attachli + global passed + global parsed + parsed = parsedin + passed = True + tryredo = False + redo = True + while(redo): + passed = True + if(passed): + gitPull("sleuthkit") + if(passed): + vsBuild() + print("TSK") + if(passed): + gitPull("autopsy") + if(passed): + antBuild("datamodel", False) + print("DataModel") + if(passed): + antBuild("autopsy", True) + print("Aut") + if(passed): + redo = False + else: + print("Compile Failed") + time.sleep(3600) + attachl = [] + errorem = "The test standard didn't match the gold standard.\n" + failedbool = False + if(tryredo): + errorem = "" + errorem += "Rebuilt properly.\n" + if email_enabled: + Emailer.send_email(to, server, subj, errorem, attachl) + attachl = [] + passed = True #Pulls from git def gitPull(TskOrAutopsy): - global SYS - global errorem - global attachl - ccwd = "" - gppth = make_local_path("..", "GitPullOutput" + TskOrAutopsy + ".txt") - attachl.append(gppth) - gpout = open(gppth, 'a') - toPull = "https://www.github.com/sleuthkit/" + TskOrAutopsy - call = ["git", "pull", toPull] - if TskOrAutopsy == "sleuthkit": - ccwd = os.path.join("..", "..", "..", "sleuthkit") - else: - ccwd = os.path.join("..", "..") - subprocess.call(call, stdout=sys.stdout, cwd=ccwd) - gpout.close() + global SYS + global errorem + global attachl + ccwd = "" + gppth = make_local_path("..", "GitPullOutput" + TskOrAutopsy + ".txt") + attachl.append(gppth) + gpout = open(gppth, 'a') + toPull = "https://www.github.com/sleuthkit/" + TskOrAutopsy + call = ["git", "pull", toPull] + if TskOrAutopsy == "sleuthkit": + ccwd = os.path.join("..", "..", "..", "sleuthkit") + else: + ccwd = os.path.join("..", "..") + subprocess.call(call, stdout=sys.stdout, cwd=ccwd) + gpout.close() #Builds TSK as a win32 applicatiion def vsBuild(): - global redo - global tryredo - global passed - global parsed - #Please ensure that the current working directory is $autopsy/testing/script - oldpath = os.getcwd() - os.chdir(os.path.join("..", "..", "..","sleuthkit", "win32")) - vs = [] - vs.append("/cygdrive/c/windows/microsoft.NET/framework/v4.0.30319/MSBuild.exe") - vs.append(os.path.join("Tsk-win.sln")) - vs.append("/p:configuration=release") - vs.append("/p:platform=win32") - vs.append("/t:clean") - vs.append("/t:rebuild") - print(vs) - VSpth = make_local_path("..", "VSOutput.txt") - VSout = open(VSpth, 'a') - subprocess.call(vs, stdout=VSout) - VSout.close() - os.chdir(oldpath) - chk = os.path.join("..", "..", "..","sleuthkit", "win32", "Release", "libtsk_jni.dll") - try: - open(chk) - except IOError as e: - global errorem - global attachl - if(not tryredo): - errorem += "LIBTSK C++ failed to build.\n" - attachl.append(VSpth) - send_email(parsed, errorem, attachl, False) - tryredo = True - passed = False - redo = True + global redo + global tryredo + global passed + global parsed + #Please ensure that the current working directory is $autopsy/testing/script + oldpath = os.getcwd() + os.chdir(os.path.join("..", "..", "..","sleuthkit", "win32")) + vs = [] + vs.append("/cygdrive/c/windows/microsoft.NET/framework/v4.0.30319/MSBuild.exe") + vs.append(os.path.join("Tsk-win.sln")) + vs.append("/p:configuration=release") + vs.append("/p:platform=x64") + vs.append("/t:clean") + vs.append("/t:rebuild") + print(vs) + VSpth = make_local_path("..", "VSOutput.txt") + VSout = open(VSpth, 'a') + subprocess.call(vs, stdout=VSout) + VSout.close() + os.chdir(oldpath) + chk = os.path.join("..", "..", "..","sleuthkit", "win32", "x64", "Release", "libtsk_jni.dll") + if not os.path.exists(chk): + print("path doesn't exist") + global errorem + global attachl + global email_enabled + if(not tryredo): + errorem += "LIBTSK C++ failed to build.\n" + attachl.append(VSpth) + if email_enabled: + Emailer.send_email(parsed, errorem, attachl, False) + tryredo = True + passed = False + redo = True #Builds Autopsy or the Datamodel def antBuild(which, Build): - global redo - global passed - global tryredo - global parsed - directory = os.path.join("..", "..") - ant = [] - if which == "datamodel": - directory = os.path.join("..", "..", "..", "sleuthkit", "bindings", "java") - ant.append("ant") - ant.append("-f") - ant.append(directory) - ant.append("clean") - if(Build): - ant.append("build") - else: - ant.append("dist") - antpth = make_local_path("..", "ant" + which + "Output.txt") - antout = open(antpth, 'a') - succd = subprocess.call(ant, stdout=antout) - antout.close() - global errorem - global attachl - if which == "datamodel": - chk = os.path.join("..", "..", "..","sleuthkit", "bindings", "java", "dist", "TSK_DataModel.jar") - try: - open(chk) - except IOError as e: - if(not tryredo): - errorem += "DataModel Java build failed.\n" - attachl.append(antpth) - Emailer.send_email(parsed, errorem, attachl, False) - passed = False - tryredo = True - elif (succd != 0 and (not tryredo)): - errorem += "Autopsy build failed.\n" - attachl.append(antpth) - Emailer.send_email(parsed, errorem, attachl, False) - tryredo = True - elif (succd != 0): - passed = False + print("building: ", which) + global redo + global passed + global tryredo + global parsed + directory = os.path.join("..", "..") + ant = [] + if which == "datamodel": + directory = os.path.join("..", "..", "..", "sleuthkit", "bindings", "java") + ant.append("ant") + ant.append("-f") + ant.append(directory) + ant.append("clean") + if(Build): + ant.append("build") + else: + ant.append("dist") + antpth = make_local_path("..", "ant" + which + "Output.txt") + antout = open(antpth, 'a') + succd = subprocess.call(ant, stdout=antout) + antout.close() + global errorem + global attachl + global email_enabled + global to + global subj + global server + if which == "datamodel": + chk = os.path.join("..", "..", "..","sleuthkit", "bindings", "java", "dist", "TSK_DataModel.jar") + try: + open(chk) + except IOError as e: + if(not tryredo): + errorem += "DataModel Java build failed.\n" + attachl.append(antpth) + if email_enabled: + Emailer.send_email(to, server, subj, errorem, attachl) + passed = False + tryredo = True + elif (succd != 0 and (not tryredo)): + errorem += "Autopsy build failed.\n" + attachl.append(antpth) + Emailer.send_email(to, server, subj, errorem, attachl) + tryredo = True + elif (succd != 0): + passed = False def main(): - errore = "" - attachli = [] - config_file = "" - arg = sys.argv.pop(0) - arg = sys.argv.pop(0) - config_file = arg - parsedin = parse(config_file) - compile(errore, attachli, parsedin) + global email_enabled + global to + global server + global subj + errore = "" + attachli = [] + config_file = "" + arg = sys.argv.pop(0) + arg = sys.argv.pop(0) + config_file = arg + parsedin = parse(config_file) + try: + to = parsedin.getElementsByTagName("email")[0].getAttribute("value").encode().decode("utf_8") + server = parsedin.getElementsByTagName("mail_server")[0].getAttribute("value").encode().decode("utf_8") + subj = parsedin.getElementsByTagName("subject")[0].getAttribute("value").encode().decode("utf_8") + except Exception: + email_enabled = False + # email_enabled = (to is not None) and (server is not None) and (subj is not None) + email_enabled = False + compile(errore, attachli, parsedin) class OS: LINUX, MAC, WIN, CYGWIN = range(4) if __name__ == "__main__": - global SYS - if _platform == "linux" or _platform == "linux2": - SYS = OS.LINUX - elif _platform == "darwin": - SYS = OS.MAC - elif _platform == "win32": - SYS = OS.WIN - elif _platform == "cygwin": - SYS = OS.CYGWIN + global SYS + if _platform == "linux" or _platform == "linux2": + SYS = OS.LINUX + elif _platform == "darwin": + SYS = OS.MAC + elif _platform == "win32": + SYS = OS.WIN + elif _platform == "cygwin": + SYS = OS.CYGWIN - if SYS is OS.WIN or SYS is OS.CYGWIN: - main() - else: - print("We only support Windows and Cygwin at this time.") + if SYS is OS.WIN or SYS is OS.CYGWIN: + main() + else: + print("We only support Windows and Cygwin at this time.") diff --git a/test/script/tskdbdiff.py b/test/script/tskdbdiff.py index 35617c48f3..95f609e447 100644 --- a/test/script/tskdbdiff.py +++ b/test/script/tskdbdiff.py @@ -137,37 +137,39 @@ class TskDbDiff(object): unsorted_dump = TskDbDiff._get_tmp_file("dump_data", ".txt") conn = sqlite3.connect(db_file) conn.text_factory = lambda x: x.decode("utf-8", "ignore") - autopsy_cur2 = conn.cursor() + conn.row_factory = sqlite3.Row + artifact_cursor = conn.cursor() # Get the list of all artifacts # @@@ Could add a SORT by parent_path in here since that is how we are going to later sort it. - autopsy_cur2.execute("SELECT tsk_files.parent_path, tsk_files.name, blackboard_artifact_types.display_name, blackboard_artifacts.artifact_id FROM blackboard_artifact_types INNER JOIN blackboard_artifacts ON blackboard_artifact_types.artifact_type_id = blackboard_artifacts.artifact_type_id INNER JOIN tsk_files ON tsk_files.obj_id = blackboard_artifacts.obj_id") + artifact_cursor.execute("SELECT tsk_files.parent_path, tsk_files.name, blackboard_artifact_types.display_name, blackboard_artifacts.artifact_id FROM blackboard_artifact_types INNER JOIN blackboard_artifacts ON blackboard_artifact_types.artifact_type_id = blackboard_artifacts.artifact_type_id INNER JOIN tsk_files ON tsk_files.obj_id = blackboard_artifacts.obj_id") database_log = codecs.open(unsorted_dump, "wb", "utf_8") - rw = autopsy_cur2.fetchone() + row = artifact_cursor.fetchone() appnd = False counter = 0 artifact_count = 0 artifact_fail = 0 # Cycle through artifacts try: - while (rw != None): + while (row != None): # File Name and artifact type - if(rw[0] != None): - database_log.write(rw[0] + rw[1] + ' ') + if(row["parent_path"] != None): + database_log.write(row["parent_path"] + row["name"] + ' ') else: - database_log.write(rw[1] + ' ') + database_log.write(row["name"] + ' ') # Get attributes for this artifact - autopsy_cur1 = conn.cursor() + attribute_cursor = conn.cursor() looptry = True artifact_count += 1 try: - key = "" - key = str(rw[3]) - key = key, - autopsy_cur1.execute("SELECT blackboard_attributes.source, blackboard_attribute_types.display_name, blackboard_attributes.value_type, blackboard_attributes.value_text, blackboard_attributes.value_int32, blackboard_attributes.value_int64, blackboard_attributes.value_double FROM blackboard_attributes INNER JOIN blackboard_attribute_types ON blackboard_attributes.attribute_type_id = blackboard_attribute_types.attribute_type_id WHERE artifact_id =? ORDER BY blackboard_attributes.source, blackboard_attribute_types.display_name, blackboard_attributes.value_type, blackboard_attributes.value_text, blackboard_attributes.value_int32, blackboard_attributes.value_int64, blackboard_attributes.value_double", key) - attributes = autopsy_cur1.fetchall() + art_id = "" + art_id = str(row["artifact_id"]) + attribute_cursor.execute("SELECT blackboard_attributes.source, blackboard_attribute_types.display_name, blackboard_attributes.value_type, blackboard_attributes.value_text, blackboard_attributes.value_int32, blackboard_attributes.value_int64, blackboard_attributes.value_double FROM blackboard_attributes INNER JOIN blackboard_attribute_types ON blackboard_attributes.attribute_type_id = blackboard_attribute_types.attribute_type_id WHERE artifact_id =? ORDER BY blackboard_attributes.source, blackboard_attribute_types.display_name, blackboard_attributes.value_type, blackboard_attributes.value_text, blackboard_attributes.value_int32, blackboard_attributes.value_int64, blackboard_attributes.value_double", [art_id]) + attributes = attribute_cursor.fetchall() except sqlite3.Error as e: - msg ="Attributes in artifact id (in output DB)# " + str(rw[3]) + " encountered an error: " + str(e) +" .\n" + msg = "Attributes in artifact id (in output DB)# " + str(row["artifact_id"]) + " encountered an error: " + str(e) +" .\n" + print("Attributes in artifact id (in output DB)# ", str(row["artifact_id"]), " encountered an error: ", str(e)) + print() looptry = False artifact_fail += 1 database_log.write('Error Extracting Attributes') @@ -178,33 +180,29 @@ class TskDbDiff(object): if(looptry == True): src = attributes[0][0] for attr in attributes: - val = 3 + attr[2] + attr_value_index = 3 + attr["value_type"] numvals = 0 for x in range(3, 6): if(attr[x] != None): numvals += 1 if(numvals > 1): - msg = "There were too many values for attribute type: " + attr[1] + " for artifact with id #" + str(rw[3]) + ".\n" - if(not attr[0] == src): - msg ="There were inconsistent sources for artifact with id #" + str(rw[3]) + ".\n" + msg = "There were too many values for attribute type: " + attr["display_name"] + " for artifact with id #" + str(row["artifact_id"]) + ".\n" + if(not attr["source"] == src): + msg = "There were inconsistent sources for artifact with id #" + str(row["artifact_id"]) + ".\n" try: - database_log.write('') except IOError as e: + print("IO error") raise TskDbDiffException("Unexpected IO error while writing to database log." + str(e)) - - database_log.write('" />') + database_log.write(' \n') - rw = autopsy_cur2.fetchone() + row = artifact_cursor.fetchone() - # Now sort the file - srtcmdlst = ["sort", unsorted_dump, "-o", bb_dump_file] - subprocess.call(srtcmdlst) print(artifact_fail) if(artifact_fail > 0): msg ="There were " + str(artifact_count) + " artifacts and " + str(artifact_fail) + " threw an exception while loading.\n" @@ -212,6 +210,10 @@ class TskDbDiff(object): raise TskDbDiffException("Unexpected error while dumping blackboard database: " + str(e)) finally: database_log.close() + + # Now sort the file + srtcmdlst = ["sort", unsorted_dump, "-o", bb_dump_file] + subprocess.call(srtcmdlst) def _dump_output_db_nonbb(db_file, dump_file): """Dumps a database to a text file. @@ -267,14 +269,14 @@ def main(): print("usage: tskdbdiff [OUPUT DB PATH] [GOLD DB PATH]") sys.exit() - db_diff = TskDbDiff(output_db, gold_db) + db_diff = TskDbDiff(output_db, gold_db, output_dir=".") dump_passed, bb_dump_passed = db_diff.run_diff() if dump_passed and bb_dump_passed: print("Database comparison passed.") - elif not dump_passed: + if not dump_passed: print("Non blackboard database comparison failed.") - elif not bb_dump_passed: + if not bb_dump_passed: print("Blackboard database comparison failed.") return 0