mstoeck3/autopsy-flatpak
1
0
Fork 0
mirror of https://github.com/overcuriousity/autopsy-flatpak.git synced 2025-07-06 21:00:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Regression test changes

Browse Source 
Added email support to the regression test, documentation coming soon
This commit is contained in:
Sean-M 2013-02-20 17:48:56 -05:00
parent 90b1a49d29
commit 0a84f38902
4 changed files with 1451 additions and 1340 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
.gitignore vendored
View File

@ -6,14 +6,34 @@
/Core/release/modules/lib/
/Core/release/modules/ext/
/Core/src/org/sleuthkit/autopsy/coreutils/Version.properties
/Core/build/
/Core/dist/
/Core/nbproject/*
!/Core/nbproject/project.xml
!/Core/nbproject/project.properties
/CoreLibs/release/modules/lib/
/CoreLibs/release/modules/ext/
/CoreLibs/build/
/CoreLibs/dist/
/CoreLibs/nbproject/*
!/CoreLibs/nbproject/project.xml
!/CoreLibs/nbproject/project.properties
/KeywordSearch/release/modules/ext/
/KeywordSearch/release/solr/lib/
/KeywordSearch/release/solr/solr/lib/
/KeywordSearch/release/solr/start.jar
/KeywordSearch/release/solr/webapps/solr.war
/KeywordSearch/build/
/KeywordSearch/dist/
/KeywordSearch/nbproject/*
!/KeywordSearch/nbproject/project.xml
!/KeywordSearch/nbproject/project.properties
/Ingest/release/modules/ext/
/Ingest/build/
/Ingest/dist/
/Ingest/nbproject/*
!/Ingest/nbproject/project.xml
!/Ingest/nbproject/project.properties
/branding_spear
/installer_spear
Bundle_*.properties
@ -21,9 +41,19 @@ Bundle_*.properties
genfiles.properties
/branding/core/core.jar/org/netbeans/core/startup/Bundle.properties
/branding/modules/org-netbeans-core-windows.jar/org/netbeans/core/windows/view/ui/Bundle.properties
/branding/build/
/branding/dist/
/branding/nbproject/*
!/branding/nbproject/project.xml
!/branding/nbproject/project.properties
/Testing/script/input/
/Testing/script/output/
/Testing/script/gold/
/Testing/build/
/Testing/dist/
/Testing/nbproject/*
!/Testing/nbproject/project.xml
!/Testing/nbproject/project.properties
*~
/netbeans-plat
/docs/doxygen/doxygen_docs
@ -31,4 +61,6 @@ genfiles.properties
/jdiff-logs/*
/gen_version.txt
hs_err_pid*.log
Core/src/org/sleuthkit/autopsy/casemodule/docs/QuickStart.html
Core/src/org/sleuthkit/autopsy/casemodule/docs/screenshot.png
/Testing/script/myconfig.xml

8
HashDatabase/nbproject/genfiles.properties
View File

@ -1,8 +0,0 @@
build.xml.data.CRC32=dee5be43
build.xml.script.CRC32=1308cb72
build.xml.stylesheet.CRC32=a56c6a5b@2.50.1
# This file is used by a NetBeans-based IDE to track changes in generated files such as build-impl.xml.
# Do not edit this file. You may delete it but then the IDE will never regenerate such files for you.
nbproject/build-impl.xml.data.CRC32=dee5be43
nbproject/build-impl.xml.script.CRC32=a7a0d07a
nbproject/build-impl.xml.stylesheet.CRC32=238281d1@2.50.1

8
RecentActivity/nbproject/genfiles.properties
View File

@ -1,8 +0,0 @@
build.xml.data.CRC32=11199bf7
build.xml.script.CRC32=d323407a
build.xml.stylesheet.CRC32=a56c6a5b@2.50.1
# This file is used by a NetBeans-based IDE to track changes in generated files such as build-impl.xml.
# Do not edit this file. You may delete it but then the IDE will never regenerate such files for you.
nbproject/build-impl.xml.data.CRC32=11199bf7
nbproject/build-impl.xml.script.CRC32=aef16a21
nbproject/build-impl.xml.stylesheet.CRC32=238281d1@2.50.1

323
Testing/script/regression.py
View File

@ -16,6 +16,14 @@ import traceback
import xml
from time import localtime, strftime
from xml.dom.minidom import parse, parseString
import smtplib
from email.mime.image import MIMEImage
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from email.MIMEBase import MIMEBase
from email import Encoders
import urllib2
#
# Please read me...
@ -57,13 +65,16 @@ class Args:
self.verbose = False
self.exception = False
self.exception_string = ""
self.mugen = False
self.contin = False
def parse(self):
sys.argv.pop(0)
global nxtproc
nxtproc = []
nxtproc.append("python")
nxtproc.append(sys.argv.pop(0))
while sys.argv:
arg = sys.argv.pop(0)
nxtproc.append(arg)
if(arg == "-f"):
try:
arg = sys.argv.pop(0)
@ -80,6 +91,7 @@ class Args:
elif(arg == "-l" or arg == "--list"):
try:
arg = sys.argv.pop(0)
nxtproc.append(arg)
printout("Running from configuration file:")
printout(arg + "\n")
self.list = True
@ -103,15 +115,13 @@ class Args:
elif(arg == "-e" or arg == "--exception"):
try:
arg = sys.argv.pop(0)
nxtproc.append(arg)
printout("Running in exception mode: ")
printout("Printing all exceptions with the string '" + arg + "'\n")
self.exception = True
self.exception_string = arg
except:
printerror("Error: No exception string given.")
elif(arg == "-m" or arg == "--mugen"):
printout("Running in mugen mode. Will loop through config file until interrupted.")
self.mugen = True
elif arg == "-h" or arg == "--help":
printout(usage())
return False
@ -267,11 +277,19 @@ class Database:
if not self.artifact_comparison:
return "All counts matched"
else:
global failedbool
global errorem
failedbool = True
errorem += "There was a difference in the number of artifacts.\n"
return "; ".join(self.artifact_comparison)
def get_attribute_comparison(self):
if not self.attribute_comparison:
return "All counts matched"
global failedbool
global errorem
failedbool = True
errorem += "There was a difference in the number of attributes.\n"
list = []
for error in self.attribute_comparison:
list.append(error)
@ -347,6 +365,7 @@ class Database:
# Iterates through an XML configuration file to find all given elements
def run_config_test(config_file):
try:
global parsed
parsed = parse(config_file)
counts = {}
if parsed.getElementsByTagName("indir"):
@ -355,36 +374,37 @@ def run_config_test(config_file):
case.global_csv = parsed.getElementsByTagName("global_csv")[0].getAttribute("value").encode().decode("utf-8")
# Generate the top navbar of the HTML for easy access to all images
case.global_csv = make_local_path(case.global_csv)
values = []
for element in parsed.getElementsByTagName("image"):
value = element.getAttribute("value").encode().decode("utf-8")
if file_exists(value):
values.append(value)
html_add_images(values)
# Run the test for each file in the configuration
global args
if(args.mugen):
if(args.contin):
#set all times an image has been processed to 0
for element in parsed.getElementsByTagName("image"):
value = element.getAttribute("value").encode().decode("utf-8")
counts[value] = 0
#Begin infiniloop
while(True):
for elem in counts.keys():
if(newDay()):
if(True):
global daycount
global nxtproc
if (daycount==1):
print("starting process")
pid = subprocess.Popen(nxtproc,
stdout=subprocess.PIPE).pid
sys.exit()
else:
daycount = 1
setDay()
gitPull("sleuthkit")
vsBuild()
gitPull("autopsy")
antBuild("datamodel", False)
antBuild("autopsy", True)
if file_exists(elem):
counts[elem]+=1
run_test(elem, counts[elem])
else:
printerror("Warning: Image file listed in the configuration does not exist:")
printerror(elem + "\n")
else:
for img in values:
if file_exists(img):
@ -397,7 +417,6 @@ def run_config_test(config_file):
printerror("Error: There was an error running with the configuration file.")
printerror(str(e) + "\n")
logging.critical(traceback.format_exc())
# Runs the test on the single given file.
# The path must be guarenteed to be a correct path.
def run_test(image_file, count):
@ -429,10 +448,6 @@ def run_test(image_file, count):
printerror("Error: Unknown fatal error when filling case data.")
printerror(str(e) + "\n")
logging.critical(traceback.format_exc())
# If running in rebuild mode (-r)
if args.rebuild:
rebuild()
# If NOT keeping Solr index (-k)
if not args.keep:
solr_index = make_local_path(case.output_dir, case.image_name, "AutopsyTestCase", "KeywordSearch")
@ -460,7 +475,9 @@ def run_test(image_file, count):
if case.global_csv:
generate_csv(case.global_csv)
generate_html()
# If running in rebuild mode (-r)
if args.rebuild:
rebuild()
# Reset the case and return the tests sucessfully finished
case.reset()
return True
@ -489,14 +506,16 @@ def run_ant():
case.ant.append("-Dgold_path=" + make_local_path(case.gold))
case.ant.append("-Dout_path=" + make_local_path(case.output_dir, case.image_name))
case.ant.append("-Dignore_unalloc=" + "%s" % args.unallocated)
case.ant.append("-Dmugen_mode" + str(args.mugen))
case.ant.append("-Dcontin_mode=" + str(args.contin))
case.ant.append("-Dtest.timeout=" + str(case.timeout))
printout("Ingesting Image:\n" + case.image_file + "\n")
printout("CMD: " + " ".join(case.ant))
printout("Starting test...\n")
antoutpth = make_local_path(case.output_dir, "antRunOutput.txt")
antout = open(antoutpth, "a")
if SYS is OS.CYGWIN:
subprocess.call(case.ant)
subprocess.call(case.ant, stdout=antout)
elif SYS is OS.WIN:
theproc = subprocess.Popen(case.ant, shell = True)
theproc.communicate()
@ -542,10 +561,10 @@ def rebuild():
copy_file(gold_from, gold_to)
except FileNotFoundException as e:
errors.append(e.error)
print(str(e))
except Exception as e:
errors.append("Error: Unknown fatal error when rebuilding the gold database.")
errors.append(str(e) + "\n")
# Rebuild the HTML report
htmlfolder = ""
for fs in os.listdir(os.path.join(os.getcwd(),case.output_dir, case.image_name, "AutopsyTestCase", "Reports")):
@ -553,20 +572,18 @@ def rebuild():
htmlfolder = fs
autopsy_html_path = make_local_path(case.output_dir, case.image_name, "AutopsyTestCase", "Reports", htmlfolder)
#html_path = make_local_path(case.output_dir, case.image_name,
# "AutopsyTestCase", "Reports")
html_path = make_local_path(case.output_dir, case.image_name,
"AutopsyTestCase", "Reports")
try:
os.makedirs(os.path.join(os.getcwd(), case.gold, case.image_name, htmlfolder))
for file in os.listdir(autopsy_html_path):
html_to = make_local_path(case.gold, case.image_name, file)
copy_file(get_file_in_dir(autopsy_html_path, file), html_to)
html_to = make_local_path(case.gold, case.image_name, file.replace("HTML Report", "Report"))
copy_dir(get_file_in_dir(autopsy_html_path, file), html_to)
except FileNotFoundException as e:
errors.append(e.error)
except Exception as e:
errors.append("Error: Unknown fatal error when rebuilding the gold html report.")
errors.append(str(e) + "\n")
okay = "Sucessfully rebuilt all gold standards."
print_report(errors, "REBUILDING", okay)
@ -700,6 +717,10 @@ def compare_bb_artifacts():
(database.gold_artifacts[type_id],
database.autopsy_artifacts[type_id]))
exceptions.append(error)
global failedbool
global errorem
failedbool = True
errorem += "There was a difference in the number of artifacts.\n"
return exceptions
except Exception as e:
exceptions.append("Error: Unable to compare blackboard_artifacts.\n")
@ -714,6 +735,10 @@ def compare_bb_attributes():
error = "Attribute counts do not match. "
error += str("Gold: %d, Test: %d" % (database.gold_attributes, database.autopsy_attributes))
exceptions.append(error)
global failedbool
global errorem
failedbool = True
errorem += "There was a difference in the number of attributes.\n"
return exceptions
except Exception as e:
exceptions.append("Error: Unable to compare blackboard_attributes.\n")
@ -728,6 +753,10 @@ def compare_tsk_objects():
error = "TSK Object counts do not match. "
error += str("Gold: %d, Test: %d" % (database.gold_objects, database.autopsy_objects))
exceptions.append(error)
global failedbool
global errorem
failedbool = True
errorem += "There was a difference between the tsk object counts.\n"
return exceptions
except Exception as e:
exceptions.append("Error: Unable to compare tsk_objects.\n")
@ -762,7 +791,7 @@ def generate_common_log():
common_log.close()
except Exception as e:
printerror("Error: Unable to generate the common log.")
printerror(str(e))
printerror(str(e) + "\n")
logging.critical(traceback.format_exc())
# Fill in the global case's variables that require the log files
@ -782,7 +811,6 @@ def fill_case_data():
printerror("Error: Unable to open autopsy.log.0.")
printerror(str(e) + "\n")
logging.warning(traceback.format_exc())
# Set the case total test time
# Start date must look like: "Jul 16, 2012 12:57:53 PM"
# End date must look like: "Mon Jul 16 13:02:42 2012"
@ -1044,7 +1072,7 @@ def print_report(errors, name, okay):
if errors:
printerror("--------< " + name + " >----------")
for error in errors:
printerror(error)
printerror(str(error))
printerror("--------< / " + name + " >--------\n")
else:
printout("-----------------------------------------------------------------")
@ -1068,6 +1096,7 @@ def generate_html():
if not file_exists(case.html_log):
write_html_head()
try:
global html
html = open(case.html_log, "a")
# The image title
title = "<h1><a name='" + case.image_name + "'>" + case.image_name + " \
@ -1248,25 +1277,40 @@ def newDay():
def gitPull(TskOrAutopsy):
global SYS
ccwd = ""
gppth = make_local_path(case.output_dir, "GitPullOutput" + TskOrAutopsy + ".txt")
gpout = open(gppth, 'a')
toPull = "http://www.github.com/sleuthkit/" + TskOrAutopsy
call = ["git", "pull", toPull]
if TskOrAutopsy == "sleuthkit":
ccwd = os.path.join("..", "..", "..", "TSK")
ccwd = os.path.join("..", "..", "..", "sleuthkit")
else:
ccwd = os.path.join("..", "..")
subprocess.call(call, stdout=subprocess.PIPE, cwd=ccwd)
subprocess.call(call, stdout=gpout, cwd=ccwd)
#Builds TSK as a win32 applicatiion
def vsBuild():
#Please ensure that the current working directory is $autopsy/testing/script
vs = []
vs.append("/cygdrive/c/windows/microsoft.NET/framework/v4.0.30319/MSBuild.exe")
vs.append(os.path.join("..", "..", "..","TSK", "win32", "Tsk-win.sln"))
vs.append(os.path.join("..", "..", "..","sleuthkit", "win32", "Tsk-win.sln"))
vs.append("/p:configuration=release")
vs.append("/p:platform=win32")
vs.append("/t:rebuild")
print(vs)
subprocess.call(vs)
VSpth = make_local_path(case.output_dir, "VSOutput.txt")
VSout = open(VSpth, 'a')
subprocess.call(vs, stdout=VSout)
chk = os.path.join("..", "..", "..","sleuthkit", "win32", "Release", "libtsk_jni.dll")
try:
open(chk)
except IOError as e:
global errorem
global attachl
errorem += "LIBTSK C++ failed to build.\n"
attachl.append(VSpth)
send_email()
sys.exit()
#Builds Autopsy or the Datamodel
@ -1274,7 +1318,7 @@ def antBuild(which, Build):
directory = os.path.join("..", "..")
ant = []
if which == "datamodel":
directory = os.path.join("..", "TSK", "bindings", "java")
directory = os.path.join("..", "..", "..", "sleuthkit", "bindings", "java")
ant.append("ant")
ant.append("-f")
ant.append(directory)
@ -1283,7 +1327,21 @@ def antBuild(which, Build):
ant.append("build")
else:
ant.append("dist")
subprocess.call(ant)
antpth = make_local_path(case.output_dir, "ant" + which + "Output.txt")
antout = open(antpth, 'a')
subprocess.call(ant, stdout=antout)
if which == "datamodel":
chk = os.path.join("..", "..", "..","sleuthkit", "bindings", "java", "dist", "TSK_DataModel.jar")
try:
open(chk)
except IOError as e:
global errorem
global attachl
errorem += "DataModel Java build failed.\n"
attachl.append(antpth)
send_email()
sys.exit()
#Watches clock and waits for current ingest to be done
@ -1340,7 +1398,6 @@ def copy_logs():
printerror("Error: Failed to copy the logs.")
printerror(str(e) + "\n")
logging.warning(traceback.format_exc())
# Clears all the files from a directory and remakes it
def clear_dir(dir):
try:
@ -1362,6 +1419,14 @@ def copy_file(ffrom, to):
except:
raise FileNotFoundException(to)
# Copies a directory file from "ffrom" to "to"
def copy_dir(ffrom, to):
try :
if not os.path.isdir(ffrom):
raise FileNotFoundException(ffrom)
shutil.copytree(ffrom, to)
except:
raise FileNotFoundException(to)
# Returns the first file in the given directory with the given extension
def get_file_in_dir(dir, ext):
try:
@ -1472,7 +1537,6 @@ class FileNotFoundException(Exception):
def print_error(self):
printerror("Error: File could not be found at:")
printerror(self.file + "\n")
def error(self):
error = "Error: File could not be found at:\n" + self.file + "\n"
return error
@ -1487,13 +1551,99 @@ class DirNotFoundException(Exception):
def print_error(self):
printerror("Error: Directory could not be found at:")
printerror(self.dir + "\n")
def error(self):
error = "Error: Directory could not be found at:\n" + self.dir + "\n"
return error
#Executes the tests, makes continuous testing easier
def execute_test():
global failedbool
global html
global attachl
case.output_dir = make_path("output", time.strftime("%Y.%m.%d-%H.%M.%S"))
os.makedirs(case.output_dir)
case.common_log = make_local_path(case.output_dir, "AutopsyErrors.txt")
case.csv = make_local_path(case.output_dir, "CSV.txt")
case.html_log = make_local_path(case.output_dir, "AutopsyTestCase.html")
log_name = case.output_dir + "\\regression.log"
logging.basicConfig(filename=log_name, level=logging.DEBUG)
# If user wants to do a single file and a list (contradictory?)
if args.single and args.list:
printerror("Error: Cannot run both from config file and on a single file.")
return
# If working from a configuration file
if args.list:
if not file_exists(args.config_file):
printerror("Error: Configuration file does not exist at:")
printerror(args.config_file)
return
run_config_test(args.config_file)
# Else if working on a single file
elif args.single:
if not file_exists(args.single_file):
printerror("Error: Image file does not exist at:")
printerror(args.single_file)
return
run_test(args.single_file, 0)
# If user has not selected a single file, and does not want to ignore
# the input directory, continue on to parsing ./input
if (not args.single) and (not args.ignore):
for file in os.listdir(case.input_dir):
# Make sure it's not a required hash/keyword file or dir
if (not required_input_file(file) and
not os.path.isdir(make_path(case.input_dir, file))):
run_test(make_path(case.input_dir, file), 0)
write_html_foot()
html.close()
if failedbool:
htmll = html.name.split('\\')
ainsert = html.name
attachl.insert(0, ainsert)
send_email()
html.close()
#email_send(HTML_email, None)
def send_email():
global parsed
global errorem
global attachl
global html
element = parsed.getElementsByTagName("email")[0]
toval = element.getAttribute("value").encode().decode("utf-8")
element = parsed.getElementsByTagName("mail_server")[0]
serverval = element.getAttribute("value").encode().decode("utf-8")
# Create the container (outer) email message.
msg = MIMEMultipart()
msg['Subject'] = 'Email Test'
# me == the sender's email address
# family = the list of all recipients' email addresses
msg['From'] = 'AutopsyContinuousTest'
msg['To'] = toval
msg.preamble = 'This is a test'
container = MIMEText(errorem, 'plain')
msg.attach(container)
Build_email(msg)
s = smtplib.SMTP(serverval)
s.sendmail(msg['From'], msg['To'], msg.as_string())
s.quit()
def Build_email(msg):
global attachl
for file in attachl:
part = MIMEBase('application', "octet-stream")
atach = open(file, "rb")
attch = atach.read()
noml = file.split("\\")
nom = noml[len(noml)-1]
part.set_payload(attch)
Encoders.encode_base64(part)
part.add_header('Content-Disposition', 'attachment; filename="' + nom + '"')
msg.attach(part)
#----------------------#
# Main #
#----------------------#
@ -1502,90 +1652,35 @@ def main():
global args
global case
global database
global failedbool
global inform
global fl
global errorem
global attachl
global daycount
setDay()
daycount = 0
failedbool = False
errorem = "The test standard didn't match the gold standard.\n"
case = TestAutopsy()
database = Database()
printout("")
args = Args()
attachl = []
# The arguments were given wrong:
if not args.parse():
case.reset()
pass
# Otherwise test away!
else:
case.output_dir = make_path("output", time.strftime("%Y.%m.%d-%H.%M.%S"))
os.makedirs(case.output_dir)
case.common_log = make_local_path(case.output_dir, "AutopsyErrors.txt")
case.csv = make_local_path(case.output_dir, "CSV.txt")
case.html_log = make_local_path(case.output_dir, "AutopsyTestCase.html")
log_name = case.output_dir + "\\regression.log"
logging.basicConfig(filename=log_name, level=logging.DEBUG)
# If user wants to do a single file and a list (contradictory?)
if args.single and args.list:
printerror("Error: Cannot run both from config file and on a single file.")
return
# If working from a configuration file
if args.list:
if not file_exists(args.config_file):
printerror("Error: Configuration file does not exist at:")
printerror(args.config_file)
return
run_config_test(args.config_file)
# Else if working on a single file
elif args.single:
if not file_exists(args.single_file):
printerror("Error: Image file does not exist at:")
printerror(args.single_file)
return
run_test(args.single_file, 0)
# If user has not selected a single file, and does not want to ignore
# the input directory, continue on to parsing ./input
if (not args.single) and (not args.ignore):
for file in os.listdir(case.input_dir):
# Make sure it's not a required hash/keyword file or dir
if (not required_input_file(file) and
not os.path.isdir(make_path(case.input_dir, file))):
run_test(make_path(case.input_dir, file), 0)
write_html_foot()
execute_test()
while args.contin:
case.output_dir = make_path("output", time.strftime("%Y.%m.%d-%H.%M.%S"))
os.makedirs(case.output_dir)
case.common_log = make_local_path(case.output_dir, "AutopsyErrors.txt")
case.csv = make_local_path(case.output_dir, "CSV.txt")
case.html_log = make_local_path(case.output_dir, "AutopsyTestCase.html")
log_name = case.output_dir + "\\regression.log"
logging.basicConfig(filename=log_name, level=logging.DEBUG)
# If user wants to do a single file and a list (contradictory?)
if args.single and args.list:
printerror("Error: Cannot run both from config file and on a single file.")
return
# If working from a configuration file
if args.list:
if not file_exists(args.config_file):
printerror("Error: Configuration file does not exist at:")
printerror(args.config_file)
return
run_config_test(args.config_file)
# Else if working on a single file
elif args.single:
if not file_exists(args.single_file):
printerror("Error: Image file does not exist at:")
printerror(args.single_file)
return
run_test(args.single_file, 0)
# If user has not selected a single file, and does not want to ignore
# the input directory, continue on to parsing ./input
if (not args.single) and (not args.ignore):
for file in os.listdir(case.input_dir):
# Make sure it's not a required hash/keyword file or dir
if (not required_input_file(file) and
not os.path.isdir(make_path(case.input_dir, file))):
run_test(make_path(case.input_dir, file), 0)
attachl = []
errorem = "The test standard didn't match the gold standard.\n"
failedbool = False
execute_test()
write_html_foot()
class OS:
LINUX, MAC, WIN, CYGWIN = range(4)