mstoeck3/autopsy-flatpak
1
0
Fork 0
mirror of https://github.com/overcuriousity/autopsy-flatpak.git synced 2025-07-06 21:00:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

regression.py : changed the single command to -s, and the ignore command to -i to reduce confusion

Browse Source 
regressionTesting.dox : updated the documentation to reflect yesterdays update

Other:
Uploaded an example XML config file
This commit is contained in:
0xNF 2012-07-18 09:32:28 -04:00
parent 819ec4eb95
commit 35bc7b1cb6
3 changed files with 62 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
Testing/script/config.xml Normal file
View File

@ -0,0 +1,16 @@
<?xml version="1.0" encoding="ASCII"?>
<!-- This file is an example config file for regression.py.
the following tags are mandatory:
1 "indir" tag that specifies where the program can find the hash databases
any number of "image" tags that specify where the images are stored. They do not have to be local.
invalid paths and images are ignored by the tester.
in this file, the first three tags are accepted by the tester, while the last two are disregarded as errors-->
<Properties>
<indir name="indir" value="C:\Users\0xNF\autopsyA\Testing\script\important_files" />
<image name="image1" value="P:\shared\Testing\script\input\64mb2.img"/>
<image name="image2" value="C:\Users\0xNF\Documents\backups \Testing\script\input\MC1001-0001.E01"/>
<image name="image3" value="should be error"/>
<image name="image4" value="C:\Invalid\Path\Does\Not\Exist.img"/>
</Properties>

52
Testing/script/regression.py
View File

@ -12,7 +12,7 @@ from xml.dom.minidom import parse, parseString
# Last modified 7/17/12 @5pm
# Usage: ./regression.py [-i FILE] OR [-l CONFIG] [OPTIONS]
# Usage: ./regression.py [-s FILE] OR [-l CONFIG] [OPTIONS]
# Run the RegressionTest.java file, and compare the result with a gold standard
# When the -i flag is set, this script only tests the image given by FILE.
# An indexed NSRL database is expected at ./input/nsrl.txt-md5.idx,
@ -23,7 +23,7 @@ from xml.dom.minidom import parse, parseString
# the /script folder.
# Options:
# -r, --rebuild Rebuild the gold standards from the test results for each image
# -u, --ignore Ignores unallocated space when ingesting. Faster, but less accurate results.
# -i, --ignore Ignores unallocated space when ingesting. Faster, but less accurate results.
hadErrors = False # If any of the tests failed
@ -41,8 +41,10 @@ def testAddImageIngest(inFile, ignoreUnalloc, list):
# Set up case directory path
testCaseName = imageName(inFile)
#check for flags to append to folder name
if ignoreUnalloc:
testCaseName+="-u"
testCaseName+="-i"
if list:
testCaseName+="-l"
if os.path.exists(os.path.join(outDir,testCaseName)):
@ -128,7 +130,7 @@ def testCompareToGold(inFile, ignore):
name = imageName(inFile)
if ignore:
name += ("-u")
name += ("-i")
cwd = wgetcwd()
goldFile = os.path.join("./",goldDir,name,"standard.db")
testFile = os.path.join("./",outDir,name,"AutopsyTestCase","autopsy.db")
@ -189,7 +191,7 @@ def clearGoldDir(inFile, ignore, list):
cwd = wgetcwd()
inFile = imageName(inFile)
if ignore:
inFile += "-u"
inFile += "-i"
if list:
inFile += "-l"
if os.path.exists(os.path.join(cwd,goldDir,inFile)):
@ -201,7 +203,7 @@ def copyTestToGold(inFile, ignore, list):
print "Recreating gold standard from results."
inFile = imageName(inFile)
if ignore:
inFile += "-u"
inFile += "-i"
if list:
inFile += "-l"
cwd = wgetcwd()
@ -214,7 +216,7 @@ def copyReportToGold(inFile, ignore, list):
print "Recreating gold report from results."
inFile = imageName(inFile)
if ignore:
inFile += "-u"
inFile += "-i"
if list:
inFile += "-l"
cwd = wgetcwd()
@ -238,7 +240,7 @@ def testCompareReports(inFile, ignore, list):
print "Comparing report to golden report."
name = imageName(inFile)
if ignore:
name += "-u"
name += "-i"
if list:
name += "-l"
goldReport = os.path.join("./",goldDir,name,"report.html")
@ -343,7 +345,7 @@ def wabspath(inFile):
def copyLogs(inFile, ignore, list):
name = imageName(inFile)
if ignore:
name +="-u"
name +="-i"
if list:
name+="-l"
logDir = os.path.join("..","build","test","qa-functional","work","userdir0","var","log")
@ -363,13 +365,16 @@ def testFile(image, rebuild, ignore, list):
def usage():
usage = "\
Usage: ./regression.py [-i FILE] [OPTIONS] \n\n\
Usage: ./regression.py [-s FILE] [OPTIONS] \n\n\
Run the RegressionTest.java file, and compare the result with a gold standard \n\n\
When the -i flag is set, this script only tests the image given by FILE.\n\
By default, it tests every image in ./input/\n\n\
An indexed NSRL database is expected at ./input/nsrl.txt-md5.idx,\n\
and an indexed notable hash database at ./input/notablehashes.txt-md5.idx\n\
In addition, any keywords to search for must be in ./input/notablekeywords.xml\n\n\
When the -l flag is set, the script looks for a config.xml file of the given name\n\
where images are stored. For usage notes please see the example config.xml in\n\
the /script folder.\
Options:\n\n\
-r, --rebuild\t\tRebuild the gold standards from the test results for each image\n\n\
-u, --ignore\t\tIgnore unallocated space while ingesting"
@ -382,32 +387,33 @@ def main():
list = False
test = True
argi = 1
Config = None #file pointed to by -l
imgListB = [] #list of legal images from imgListA generated by main()
Config = None #file pointed to by --list
imgListB = [] #list of legal images from config
cwd = wgetcwd()
while argi < len(sys.argv):
arg = sys.argv[argi]
if arg == "-i" and argi+1 < len(sys.argv):
if arg == "-s" and argi+1 < len(sys.argv): #check for single
single = True
argi+=1
image = sys.argv[argi]
print "Running on single image: " + image
if arg == "-l" or arg == "--list":
if arg == "-l" or arg == "--list": #check for config file
list = True
argi+=1
#check for file in ./
if(os.path.isfile(os.path.join("./", sys.argv[argi]))):
Config = parse(os.path.join("./", sys.argv[argi]))
#else check if it is a specified path
elif (os.path.exists(wabspath(sys.argv[argi]))):
Config = parse(sys.argv[argi])
else:
print sys.argv[argi]
print wabspath(sys.argv[argi])
markError("Ran with " + arg +" but no such file exists", arg)
elif (arg == "--rebuild") or (arg == "-r"):
elif (arg == "--rebuild") or (arg == "-r"): #check for rebuild flag
rebuild = True
print "Running in REBUILD mode"
elif (arg == "--ignore") or (arg == "-u"):
elif (arg == "--ignore") or (arg == "-i"): #check for ignore flag
ignore = True
print "Ignoring unallocated space"
else:
@ -419,18 +425,12 @@ def main():
if list:
listImages = []
errors = 0
global inDir
out = Config.getElementsByTagName("indir")[0].getAttribute("value").encode()
# proc = subprocess.Popen(("cygpath", "-u", Config.getElementsByTagName("indir")[0].getAttribute("value")), stdout=subprocess.PIPE)
# out,err = proc.communicate()
# out.rstrip()
# print "number of slashs is:" + str(out.count("/"))
# out.replace("/", "\\\\")
# print "out is" + out
global inDir
out = Config.getElementsByTagName("indir")[0].getAttribute("value").encode() #there should only be one indir element in the config
inDir = out
for element in Config.getElementsByTagName("image"):
elem = element.getAttribute("value").encode()
proc2 = subprocess.Popen(("cygpath", "-u", elem), stdout=subprocess.PIPE)
proc2 = subprocess.Popen(("cygpath", "-i", elem), stdout=subprocess.PIPE)
out2,err = proc2.communicate()
out2 = out2.rstrip()
if(os.path.exists(out2) and os.path.isfile(out2)):
@ -438,7 +438,7 @@ def main():
else:
print out2 + " is not a valid path or is not an image"
errors+=1
print "error total: " + str(errors)
print "Illegal files specified: " + str(errors)
for image in listImages:
testFile(image, rebuild, ignore, list)
elif test:

28
docs/doxygen/regressionTesting.dox
View File

@ -60,27 +60,39 @@ https://github.com/sleuthkit/autopsy/blob/master/BUILDING.txt
3) Run regression tests
3a) From the Cygwin terminal, navigate to the /script folder and run "./regression.py". The script will automatically begin Autopsy and run ingestion and analysis on all the images from the /input directory, and will close when finished. The Cygwin terminal will print out whether or not errors were encountered at the end of each image's test.
3a) From the Cygwin terminal, navigate to the /script folder and run "./regression.py". The script will automatically begin Autopsy and run ingestion and analysis on all the images from the ./input directory, and will close when finished. The Cygwin terminal will print out whether or not errors were encountered at the end of each image's test.
3b) It is possible to run testing on images that do not reside within ./input. To accomplish this, a configuration file is specified as a flag when run. Please see 4 below.
\section regression_test_update Options
Currently, the script supports three argument parameters:
Currently, the script supports four argument parameters:
-i {imgname} : runs the test with a single image as specified by {imgname}. Must include the path, it does not read from input.
-r or --rebuild : runs in REBUILD mode, see 4 below
-u or --ignore : runs ignoring unallocated space. Useful for determining framework is operational. Appends "-u" to the output and gold folders for distinguishing them from others. Will automatically compare the right reports.
-s {imgname} : runs the test with a single image as specified by {imgname}. Must include the path, it does not read from input.
-r or --rebuild : runs in REBUILD mode, see 5 below
-l {config} or --list {config} : runs with the specified {config} file. If you do not specify a path to the file, it searches for it under ./. It will attempt to search for the file by the specified path name if surrounded in quotes.
-u or --ignore : runs ignoring unallocated space. Useful for determining framework is operational. Appends "-u" to the output and gold folders for distinguishing them from others. Will automatically compare the right reports.
These can be run in any combination.
4) OPTIONAL: Update the standards databases
4a) From the Cygwin terminal, navigate to
\section regression_test_config Running regression tests with a config file
4) Running with a configuration file
The config file is any user defined XML file that has the following properties: 1 element with the tag "indir" to specify where the hash databases are, and any number of elements with the tag "image", where the program will look for external images. These images do not have to be held locally. They may be on network shares, or other attached drives. We have included a sample "config.xml" in the script folder as a guide. Note that the config file may be named anything.
5) OPTIONAL: Update the standards databases
5a) From the Cygwin terminal, navigate to
autopsy/Testing/script
4b) run "./regression.py -r", The script will automatically delete pre-existing standards.db files and generate the updated ones in the proper locations (/script/gold/{name of image}).
5b) run "./regression.py -r", The script will automatically delete pre-existing standards.db files and generate the updated ones in the proper locations (/script/gold/{name of image}).
Running in -r will also generate a golden report file built from the image. Normal runs of regression.py compare their generated report against the golden one, and report any differences in the file, ignoring the timestamp.