autopsy-flatpak/update_versions.py
2012-11-01 15:21:23 -04:00

857 lines
31 KiB
Python

# ============================================================
# update_versions.py
# ============================================================
#
# When run from the Autopsy build script, this script will:
# - Clone Autopsy and checkout to the previous release tag
# as found in the NEWS.txt file
# - Auto-discover all modules and packages
# - Run jdiff, comparing the current and previous modules
# - Use jdiff's output to determine if each module
# a) has no changes
# b) has backwards compatible changes
# c) has backwards incompatible changes
# - Based off it's compatibility, updates each module's
# a) Major version
# b) Specification version
# c) Implementation version
# - Updates the dependencies on each module depending on the
# updated version numbers
#
# Optionally, when run from the command line, one can provide the
# desired tag to compare the current version to, the directory for
# the current version of Autopsy, and whether to automatically
# update the version numbers and dependencies.
# ------------------------------------------------------------
import os
import shutil
import subprocess
import sys
from os import remove, close
from shutil import move
from tempfile import mkstemp
from xml.dom.minidom import parse, parseString
# An Autopsy module object
class Module:
# Initialize it with a name, return code, and version numbers
def __init__(self, name=None, ret=None, versions=None):
self.name = name
self.ret = ret
self.versions = versions
# As a string, the module should be it's name
def __str__(self):
return self.name
def __repr__(self):
return self.name
# When compared to another module, the two are equal if the names are the same
def __cmp__(self, other):
if isinstance(other, Module):
if self.name == other.name:
return 0
elif self.name < other.name:
return -1
else:
return 1
return 1
def __eq__(self, other):
if isinstance(other, Module):
if self.name == other.name:
return True
return False
def set_name(self, name):
self.name = name
def set_ret(self, ret):
self.ret = ret
def set_versions(self, versions):
self.versions = versions
def spec(self):
return self.versions[0]
def impl(self):
return self.versions[1]
def release(self):
return self.versions[2]
# Representation of the Specification version number
class Spec:
# Initialize specification number, where num is a string like x.y
def __init__(self, num):
l, r = num.split(".")
self.left = int(l)
self.right = int(r)
def __str__(self):
return self.get()
def __cmp__(self, other):
if isinstance(other, Spec):
if self.left == other.left:
if self.right == other.right:
return 0
if self.right < other.right:
return -1
return 1
if self.left < other.left:
return -1
return 1
elif isinstance(other, str):
l, r = other.split(".")
if self.left == int(l):
if self.right == int(r):
return 0
if self.right < int(r):
return -1
return 1
if self.left < int(l):
return -1
return 1
return -1
def overflow(self):
return str(self.left + 1) + ".0"
def increment(self):
return str(self.left) + "." + str(self.right + 1)
def get(self):
return str(self.left) + "." + str(self.right)
def set(self, num):
if isinstance(num, str):
l, r = num.split(".")
self.left = int(l)
self.right = int(r)
elif isinstance(num, Spec):
self.left = num.left
self.right = num.right
return self
# ================================ #
# Core Functions #
# ================================ #
# Given a list of modules and the names for each version, compare
# the generated jdiff XML for each module and output the jdiff
# JavaDocs.
#
# modules: the list of all modules both versions have in common
# apiname_tag: the api name of the previous version, most likely the tag
# apiname_cur: the api name of the current version, most likely "Current"
#
# returns the exit code from the modified jdiff.jar
# return code 1 = error in jdiff
# return code 100 = no changes
# return code 101 = compatible changes
# return code 102 = incompatible changes
def compare_xml(module, apiname_tag, apiname_cur):
global docdir
make_dir(docdir)
null_file = fix_path(os.path.abspath("./thirdparty/jdiff/v-custom/lib/Null.java"))
jdiff = fix_path(os.path.abspath("./thirdparty/jdiff/v-custom/jdiff.jar"))
oldapi = fix_path("jdiff-xml/" + apiname_tag + "-" + module.name)
newapi = fix_path("jdiff-xml/" + apiname_cur + "-" + module.name)
docs = fix_path(docdir + "/" + module.name)
comments = fix_path(docs + "/user_comments_for_jdiff-xml")
tag_comments = fix_path(comments + "/" + apiname_tag + "-" + module.name + "_to_jdiff-xml")
make_dir(docs)
make_dir(comments)
make_dir(tag_comments)
make_dir("jdiff-logs")
log = open("jdiff-logs/COMPARE-" + module.name + ".log", "w")
cmd = ["javadoc",
"-doclet", "jdiff.JDiff",
"-docletpath", jdiff,
"-d", docs,
"-oldapi", oldapi,
"-newapi", newapi,
"-script",
null_file]
jdiff = subprocess.Popen(cmd, stdout=log, stderr=log)
jdiff.wait()
log.close()
code = jdiff.returncode
print("Compared XML for " + module.name)
if code == 100:
print(" No changes")
elif code == 101:
print(" Changes are backwards compatible")
elif code == 102:
print(" Changes are not backwards compatible")
else:
print(" *Error in XML, most likely an empty module")
sys.stdout.flush()
return code
# Generate the jdiff xml for the given module
# path: path to the autopsy source
# module: Module object
# name: api name for jdiff
def gen_xml(path, modules, name):
for module in modules:
# If its the regression test, the source is in the "test" dir
if module.name == "Testing":
src = os.path.join(path, module.name, "test", "qa-functional", "src")
else:
src = os.path.join(path, module.name, "src")
# xerces = os.path.abspath("./lib/xerces.jar")
xml_out = fix_path(os.path.abspath("./jdiff-xml/" + name + "-" + module.name))
jdiff = fix_path(os.path.abspath("./thirdparty/jdiff/v-custom/jdiff.jar"))
make_dir("jdiff-xml")
make_dir("jdiff-logs")
log = open("jdiff-logs/GEN_XML-" + name + "-" + module.name + ".log", "w")
cmd = ["javadoc",
"-doclet", "jdiff.JDiff",
"-docletpath", jdiff, # ;" + xerces, <-- previous problems required this
"-apiname", xml_out, # leaving it in just in case it's needed once again
"-sourcepath", fix_path(src)]
cmd = cmd + get_packages(src)
jdiff = subprocess.Popen(cmd, stdout=log, stderr=log)
jdiff.wait()
log.close()
print("Generated XML for " + name + " " + module.name)
sys.stdout.flush()
# Find all the modules in the given path
def find_modules(path):
modules = []
# Step into each folder in the given path and
# see if it has manifest.mf - if so, it's a module
for dir in os.listdir(path):
directory = os.path.join(path, dir)
if os.path.isdir(directory):
for file in os.listdir(directory):
if file == "manifest.mf":
modules.append(Module(dir, None, None))
return modules
# Detects the differences between the source and tag modules
def module_diff(source_modules, tag_modules):
added_modules = [x for x in source_modules if x not in tag_modules]
removed_modules = [x for x in tag_modules if x not in source_modules]
similar_modules = [x for x in source_modules if x in tag_modules]
added_modules = (added_modules if added_modules else [])
removed_modules = (removed_modules if removed_modules else [])
similar_modules = (similar_modules if similar_modules else [])
return similar_modules, added_modules, removed_modules
# Reads the previous tag from NEWS.txt
def get_tag(sourcepath):
news = open(sourcepath + "/NEWS.txt", "r")
second_instance = False
for line in news:
if "----------------" in line:
if second_instance:
ver = line.split("VERSION ")[1]
ver = ver.split(" -")[0]
return "autopsy-" + ver
else:
second_instance = True
continue
news.close()
# ========================================== #
# Dependency Functions #
# ========================================== #
# Write a new XML file, copying all the lines from projectxml
# and replacing the specification version for the code-name-base base
# with the supplied specification version spec
def set_dep_spec(projectxml, base, spec):
print(" Updating Specification version..")
orig = open(projectxml, "r")
f, abs_path = mkstemp()
new_file = open(abs_path, "w")
found_base = False
spacing = " "
sopen = "<specification-version>"
sclose = "</specification-version>\n"
for line in orig:
if base in line:
found_base = True
if found_base and sopen in line:
update = spacing + sopen + str(spec) + sclose
new_file.write(update)
else:
new_file.write(line)
new_file.close()
close(f)
orig.close()
remove(projectxml)
move(abs_path, projectxml)
# Write a new XML file, copying all the lines from projectxml
# and replacing the release version for the code-name-base base
# with the supplied release version
def set_dep_release(projectxml, base, release):
print(" Updating Release version..")
orig = open(projectxml, "r")
f, abs_path = mkstemp()
new_file = open(abs_path, "w")
found_base = False
spacing = " "
ropen = "<release-version>"
rclose = "</release-version>\n"
for line in orig:
if base in line:
found_base = True
if found_base and ropen in line:
update = spacing + ropen + str(release) + rclose
new_file.write(update)
else:
new_file.write(line)
new_file.close()
close(f)
orig.close()
remove(projectxml)
move(abs_path, projectxml)
# Return the dependency versions in the XML dependency node
def get_dep_versions(dep):
run_dependency = dep.getElementsByTagName("run-dependency")[0]
release_version = run_dependency.getElementsByTagName("release-version")
if release_version:
release_version = getTagText(release_version[0].childNodes)
specification_version = run_dependency.getElementsByTagName("specification-version")
if specification_version:
specification_version = getTagText(specification_version[0].childNodes)
return int(release_version), Spec(specification_version)
# Given a code-name-base, see if it corresponds with any of our modules
def get_module_from_base(modules, code_name_base):
for module in modules:
if "org.sleuthkit.autopsy." + module.name.lower() == code_name_base:
return module
return None # If it didn't match one of our modules
# Check the text between two XML tags
def getTagText(nodelist):
for node in nodelist:
if node.nodeType == node.TEXT_NODE:
return node.data
# Check the projectxml for a dependency on any module in modules
def check_for_dependencies(projectxml, modules):
dom = parse(projectxml)
dep_list = dom.getElementsByTagName("dependency")
for dep in dep_list:
code_name_base = dep.getElementsByTagName("code-name-base")[0]
code_name_base = getTagText(code_name_base.childNodes)
module = get_module_from_base(modules, code_name_base)
if module:
print(" Found dependency on " + module.name)
release, spec = get_dep_versions(dep)
if release != module.release() and module.release() is not None:
set_dep_release(projectxml, code_name_base, module.release())
else: print(" Release version is correct")
if spec != module.spec() and module.spec() is not None:
set_dep_spec(projectxml, code_name_base, module.spec())
else: print(" Specification version is correct")
# Given the module and the source directory, return
# the paths to the manifest and project properties files
def get_dependency_file(module, source):
projectxml = os.path.join(source, module.name, "nbproject", "project.xml")
if os.path.isfile(projectxml):
return projectxml
# Verify/Update the dependencies for each module, basing the dependency
# version number off the versions in each module
def update_dependencies(modules, source):
for module in modules:
print("Checking the dependencies for " + module.name + "...")
projectxml = get_dependency_file(module, source)
if projectxml == None:
print(" Error finding project xml file")
else:
other = [x for x in modules]
check_for_dependencies(projectxml, other)
sys.stdout.flush()
# ======================================== #
# Versioning Functions #
# ======================================== #
# Return the specification version in the given project.properties/manifest.mf file
def get_specification(project, manifest):
try:
# Try to find it in the project file
# it will be there if impl version is set to append automatically
f = open(project, 'r')
for line in f:
if "spec.version.base" in line:
return Spec(line.split("=")[1].strip())
f.close()
# If not found there, try the manifest file
f = open(manifest, 'r')
for line in f:
if "OpenIDE-Module-Specification-Version:" in line:
return Spec(line.split(": ")[1].strip())
except:
print("Error parsing Specification version for")
print(project)
# Set the specification version in the given project properties file
# but if it can't be found there, set it in the manifest file
def set_specification(project, manifest, num):
try:
# First try the project file
f = open(project, 'r')
for line in f:
if "spec.version.base" in line:
f.close()
replace(project, line, "spec.version.base=" + str(num) + "\n")
return
f.close()
# If it's not there, try the manifest file
f = open(manifest, 'r')
for line in f:
if "OpenIDE-Module-Specification-Version:" in line:
f.close()
replace(manifest, line, "OpenIDE-Module-Specification-Version: " + str(num) + "\n")
return
# Otherwise we're out of luck
print(" Error finding the Specification version to update")
print(" " + manifest)
f.close()
except:
print(" Error incrementing Specification version for")
print(" " + project)
# Return the implementation version in the given manifest.mf file
def get_implementation(manifest):
try:
f = open(manifest, 'r')
for line in f:
if "OpenIDE-Module-Implementation-Version" in line:
return int(line.split(": ")[1].strip())
f.close()
except:
print("Error parsing Implementation version for")
print(manifest)
# Set the implementation version in the given manifest file
def set_implementation(manifest, num):
try:
f = open(manifest, 'r')
for line in f:
if "OpenIDE-Module-Implementation-Version" in line:
f.close()
replace(manifest, line, "OpenIDE-Module-Implementation-Version: " + str(num) + "\n")
return
# If it isn't there, add it
f.close()
write_implementation(manifest, num)
except:
print(" Error incrementing Implementation version for")
print(" " + manifest)
# Rewrite the manifest file to include the implementation version
def write_implementation(manifest, num):
f = open(manifest, "r")
contents = f.read()
contents = contents[:-2] + "OpenIDE-Module-Implementation-Version: " + str(num) + "\n\n"
f.close()
f = open(manifest, "w")
f.write(contents)
f.close()
# Return the release version in the given manifest.mf file
def get_release(manifest):
try:
f = open(manifest, 'r')
for line in f:
if "OpenIDE-Module:" in line:
return int(line.split("/")[1].strip())
f.close()
except:
print("Error parsing Release version for")
print(manifest)
# Set the release version in the given manifest file
def set_release(manifest, num):
try:
f = open(manifest, 'r')
for line in f:
if "OpenIDE-Module:" in line:
f.close()
index = line.index('/') - len(line) + 1
newline = line[:index] + str(num)
replace(manifest, line, newline + "\n")
return
print(" Error finding the release version to update")
print(" " + manifest)
f.close()
except:
print(" Error incrementing release version for")
print(" " + manifest)
# Given the module and the source directory, return
# the paths to the manifest and project properties files
def get_version_files(module, source):
manifest = os.path.join(source, module.name, "manifest.mf")
project = os.path.join(source, module.name, "nbproject", "project.properties")
if os.path.isfile(manifest) and os.path.isfile(project):
return manifest, project
# Returns a the current version numbers for the module in source
def get_versions(module, source):
manifest, project = get_version_files(module, source)
if manifest == None or project == None:
print(" Error finding manifeset and project properties files")
return
spec = get_specification(project, manifest)
impl = get_implementation(manifest)
release = get_release(manifest)
return [spec, impl, release]
# Update the version numbers for every module in modules
def update_versions(modules, source):
for module in modules:
versions = module.versions
manifest, project = get_version_files(module, source)
print("Updating " + module.name + "...")
if manifest == None or project == None:
print(" Error finding manifeset and project properties files")
return
if module.ret == 101:
versions = [versions[0].set(versions[0].increment()), versions[1] + 1, versions[2]]
set_specification(project, manifest, versions[0])
set_implementation(manifest, versions[1])
module.set_versions(versions)
elif module.ret == 102:
versions = [versions[0].set(versions[0].overflow()), versions[1] + 1, versions[2] + 1]
set_specification(project, manifest, versions[0])
set_implementation(manifest, versions[1])
set_release(manifest, versions[2])
module.set_versions(versions)
elif module.ret == None:
versions = [Spec("1.0"), 1, 1]
set_specification(project, manifest, versions[0])
set_implementation(manifest, versions[1])
set_release(manifest, versions[2])
module.set_versions(versions)
sys.stdout.flush()
# Given a list of the added modules, remove the modules
# which have the correct 'new module default' version number
def remove_correct_added(modules):
correct = [x for x in modules]
for module in modules:
if module.spec() == "1.0" or module.spec() == "0.0":
if module.impl() == 1:
if module.release() == 1 or module.release() == 0:
correct.remove(module)
return correct
# ==================================== #
# Helper Functions #
# ==================================== #
# Replace pattern with subst in given file
def replace(file, pattern, subst):
#Create temp file
fh, abs_path = mkstemp()
new_file = open(abs_path,'w')
old_file = open(file)
for line in old_file:
new_file.write(line.replace(pattern, subst))
#close temp file
new_file.close()
close(fh)
old_file.close()
#Remove original file
remove(file)
#Move new file
move(abs_path, file)
# Given a list of modules print the version numbers that need changing
def print_version_updates(modules):
f = open("gen_version.txt", "a")
for module in modules:
versions = module.versions
if module.ret == 101:
output = (module.name + ":\n")
output += (" Current Specification version:\t" + str(versions[0]) + "\n")
output += (" Updated Specification version:\t" + str(versions[0].increment()) + "\n")
output += ("\n")
output += (" Current Implementation version:\t" + str(versions[1]) + "\n")
output += (" Updated Implementation version:\t" + str(versions[1] + 1) + "\n")
output += ("\n")
print(output)
f.write(output)
elif module.ret == 102:
output = (module.name + ":\n")
output += (" Current Specification version:\t" + str(versions[0]) + "\n")
output += (" Updated Specification version:\t" + str(versions[0].overflow()) + "\n")
output += ("\n")
output += (" Current Implementation version:\t" + str(versions[1]) + "\n")
output += (" Updated Implementation version:\t" + str(versions[1] + 1) + "\n")
output += ("\n")
output += (" Current Release version:\t\t" + str(versions[2]) + "\n")
output += (" Updated Release version:\t\t" + str(versions[2] + 1) + "\n")
output += ("\n")
print(output)
f.write(output)
elif module.ret == 1:
output = (module.name + ":\n")
output += (" *Unable to detect necessary changes\n")
output += (" Current Specification version:\t" + str(versions[0]) + "\n")
output += (" Current Implementation version:\t" + str(versions[1]) + "\n")
output += (" Current Release version:\t\t" + str(versions[2]) + "\n")
output += ("\n")
print(output)
f.write(output)
elif module.ret is None:
output = ("Added " + module.name + ":\n")
if module.spec() != "1.0" and module.spec() != "0.0":
output += (" Current Specification version:\t" + str(module.spec()) + "\n")
output += (" Updated Specification version:\t1.0\n")
output += ("\n")
if module.impl() != 1:
output += (" Current Implementation version:\t" + str(module.impl()) + "\n")
output += (" Updated Implementation version:\t1\n")
output += ("\n")
if module.release() != 1 and module.release() != 0:
output += (" Current Release version:\t\t" + str(module.release()) + "\n")
output += (" Updated Release version:\t\t1\n")
output += ("\n")
print(output)
f.write(output)
sys.stdout.flush()
f.close()
# Changes cygwin paths to Windows
def fix_path(path):
if "cygdrive" in path:
new_path = path[11:]
return "C:/" + new_path
else:
return path
# Print a 'title'
def printt(title):
print("\n" + title)
lines = ""
for letter in title:
lines += "-"
print(lines)
sys.stdout.flush()
# Get a list of package names in the given path
# The path is expected to be of the form {base}/module/src
#
# NOTE: We currently only check for packages of the form
# org.sleuthkit.autopsy.x
# If we add other namespaces for commercial modules we will
# have to add a check here
def get_packages(path):
packages = []
package_path = os.path.join(path, "org", "sleuthkit", "autopsy")
for folder in os.listdir(package_path):
package_string = "org.sleuthkit.autopsy."
packages.append(package_string + folder)
return packages
# Create the given directory, if it doesn't already exist
def make_dir(dir):
try:
if not os.path.isdir(dir):
os.mkdir(dir)
if os.path.isdir(dir):
return True
return False
except:
print("Exception thrown when creating directory")
return False
# Delete the given directory, and make sure it is deleted
def del_dir(dir):
try:
if os.path.isdir(dir):
shutil.rmtree(dir)
if os.path.isdir(dir):
return False
else:
return True
return True
except:
print("Exception thrown when deleting directory")
return False
# Run git clone and git checkout for the tag
def do_git(tag, tag_dir):
try:
printt("Cloning Autopsy (this could take a while)...")
subprocess.call(["git", "clone", "https://github.com/sleuthkit/autopsy.git", tag_dir],
stdout=subprocess.PIPE)
printt("Checking out tag " + tag + "...")
subprocess.call(["git", "checkout", tag],
stdout=subprocess.PIPE,
cwd=tag_dir)
return True
except:
print("Error cloning and checking out Autopsy.")
print("The terminal you are using most likely does not recognize git commands.")
return False
# Get the flags from argv
def args():
try:
sys.argv.pop(0)
while sys.argv:
arg = sys.argv.pop(0)
if arg == "-h" or arg == "--help":
return 1
elif arg == "-t" or arg == "--tag":
global tag
tag = sys.argv.pop(0)
elif arg == "-s" or arg == "--source":
global source
source = sys.argv.pop(0)
elif arg == "-d" or arg == "--dir":
global docdir
docdir = sys.argv.pop(0)
elif arg == "-a" or arg == "--auto":
global dry
dry = False
else:
raise Exception()
except:
pass
# Print script run info
def printinfo():
global tag
global source
global docdir
global dry
printt("Release script information:")
if source is None:
source = fix_path(os.path.abspath("."))
print("Using source directory:\n " + source)
if tag is None:
tag = get_tag(source)
print("Checking out to tag:\n " + tag)
if docdir is None:
docdir = fix_path(os.path.abspath("./jdiff-javadocs"))
print("Generating jdiff JavaDocs in:\n " + docdir)
if dry is True:
print("Dry run: will not auto-update version numbers")
sys.stdout.flush()
# Print the script's usage/help
def usage():
return \
"""
USAGE:
Run this script to generate a jdiff XML summary for every module
in the current Autopsy source and in a previous source specified
by the given tag. Then, compare the XML files to see which modules
need updated version numbers. If the dry run tag is not given, the
module numbers will be automatically updated.
OPTIONAL FLAGS:
-t --tag The tag name in git. Otherwise the NEWS file in source
will be used to determine the previous tag.
-d --dir The output directory for the jdiff JavaDocs. If no
directory is given, the default is /javadocs/{module}.
-s --source The directory containing Autopsy's source code.
-a --auto Automatically update version numbers (not dry).
-h --help Prints this usage.
"""
# ==================================== #
# Main Functionality #
# ==================================== #
# Where the magic happens
def main():
global tag; global source; global docdir; global dry
tag = None; source = None; docdir = None; dry = True
ret = args()
if ret:
print(usage())
return 0
printinfo()
# -----------------------------------------------
# 1) Clone Autopsy, checkout to given tag/commit
# 2) Get the modules in the clone and the source
# 3) Generate the xml comparison
# -----------------------------------------------
del_dir("autopsy-update_versions")
tag_dir = os.path.abspath("./autopsy-update_versions")
if not do_git(tag, tag_dir):
return 1
sys.stdout.flush()
tag_modules = find_modules(tag_dir)
source_modules = find_modules(source)
printt("Generating jdiff XML reports...")
apiname_tag = tag
apiname_cur = "current"
gen_xml(tag_dir, tag_modules, apiname_tag)
gen_xml(source, source_modules, apiname_cur)
printt("Deleting cloned Autopsy directory...")
print("Clone successfully deleted" if del_dir(tag_dir) else "Failed to delete clone")
sys.stdout.flush()
# -----------------------------------------------------
# 1) Seperate modules into added, similar, and removed
# 2) Compare XML for each module
# -----------------------------------------------------
printt("Comparing modules found...")
similar_modules, added_modules, removed_modules = module_diff(source_modules, tag_modules)
if added_modules or removed_modules:
for m in added_modules:
print("+ Added " + m.name)
sys.stdout.flush()
for m in removed_modules:
print("- Removed " + m.name)
sys.stdout.flush()
else:
print("No added or removed modules")
sys.stdout.flush()
printt("Comparing jdiff outputs...")
for module in similar_modules:
module.set_ret(compare_xml(module, apiname_tag, apiname_cur))
# ------------------------------------------------------------
# 1) Do versioning
# 2) Auto-update version numbers in files and the_modules list
# 3) Auto-update dependencies
# ------------------------------------------------------------
printt("Auto-detecting version numbers and changes...")
for module in added_modules:
module.set_versions(get_versions(module, source))
for module in similar_modules:
module.set_versions(get_versions(module, source))
added_modules = remove_correct_added(added_modules)
the_modules = similar_modules + added_modules
print_version_updates(the_modules)
if not dry:
printt("Auto-updating version numbers...")
update_versions(the_modules, source)
print("All auto-updates complete")
printt("Detecting and auto-updating dependencies...")
update_dependencies(the_modules, source)
printt("Deleting jdiff XML...")
xml_dir = os.path.abspath("./jdiff-xml")
print("XML successfully deleted" if del_dir(xml_dir) else "Failed to delete XML")
print("\n--- Script completed successfully ---")
return 0
# Start off the script
if __name__ == "__main__":
sys.exit(main())