From de063b995782065e92cb023aed7bc4a8ab04a6b8 Mon Sep 17 00:00:00 2001 From: alexjacks92 Date: Mon, 2 Jun 2014 16:38:32 -0400 Subject: [PATCH 1/5] Working towards moving both sets of logs for testing. --- .../sleuthkit/autopsy/coreutils/Logger.java | 4 +- test/script/regression.py | 41 ++++++++++++++----- 2 files changed, 32 insertions(+), 13 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/coreutils/Logger.java b/Core/src/org/sleuthkit/autopsy/coreutils/Logger.java index da29925272..6b2f5c474b 100644 --- a/Core/src/org/sleuthkit/autopsy/coreutils/Logger.java +++ b/Core/src/org/sleuthkit/autopsy/coreutils/Logger.java @@ -33,8 +33,8 @@ public final class Logger extends java.util.logging.Logger { private static final String LOG_ENCODING = PlatformUtil.getLogFileEncoding(); private static final int LOG_SIZE = 0; // In bytes, zero is unlimited private static final int LOG_FILE_COUNT = 10; - private static final String LOG_WITHOUT_STACK_TRACES = "autopsy.log"; //NON-NLS - private static final String LOG_WITH_STACK_TRACES = "autopsy_traces.log"; //NON-NLS + private static final String LOG_WITHOUT_STACK_TRACES = "autopsy_case.log"; //NON-NLS + private static final String LOG_WITH_STACK_TRACES = "autopsy_case_traces.log"; //NON-NLS private static final CaseChangeListener caseChangeListener = new CaseChangeListener(); private static final Handler console = new java.util.logging.ConsoleHandler(); private static FileHandler userFriendlyLogFile = createFileHandler(PlatformUtil.getLogDirectory(), LOG_WITHOUT_STACK_TRACES); diff --git a/test/script/regression.py b/test/script/regression.py index 6cc3dd9917..ce289edbb4 100755 --- a/test/script/regression.py +++ b/test/script/regression.py @@ -142,7 +142,7 @@ class TestRunner(object): test_data.printerror = Errors.printerror # give solr process time to die. time.sleep(10) - + print("Total ingest time was " + test_data.total_ingest_time) Reports.write_html_foot(test_config.html_log) if test_config.jenkins: @@ -1200,17 +1200,18 @@ class Logs(object): test_data: the TestData to modify """ try: - # Open autopsy.log.0 - log_path = make_path(test_data.logs_dir, "autopsy.log.0") + # Open autopsy_case.log.0 + log_path = make_path(test_data.logs_dir, "autopsy_case.log.0") log = open(log_path) - # Set the TestData start time based off the first line of autopsy.log.0 + # Set the TestData start time based off the first line of autopsy_case.log.0 # *** If logging time format ever changes this will break *** test_data.start_date = log.readline().split(" org.")[0] + print("1210") # Set the test_data ending time based off the "create" time (when the file was copied) test_data.end_date = time.ctime(os.path.getmtime(log_path)) except IOError as e: - Errors.print_error("Error: Unable to open autopsy.log.0.") + Errors.print_error("Error: Unable to open autopsy_case.log.0.") Errors.print_error(str(e) + "\n") logging.warning(traceback.format_exc()) # Start date must look like: "Jul 16, 2012 12:57:53 PM" @@ -1224,9 +1225,10 @@ class Logs(object): # Set Autopsy version, heap space, ingest time, and service times version_line = search_logs("INFO: Application name: Autopsy, version:", test_data)[0] + print("1228") test_data.autopsy_version = get_word_at(version_line, 5).rstrip(",") test_data.heap_space = search_logs("Heap memory usage:", test_data)[0].rstrip().split(": ")[1] - + print("1231") ingest_line = search_logs("Ingest (including enqueue)", test_data)[0] test_data.total_ingest_time = get_word_at(ingest_line, 6).rstrip() @@ -1245,7 +1247,7 @@ class Logs(object): logging.critical(traceback.format_exc()) print(traceback.format_exc()) try: - service_lines = find_msg_in_log("autopsy.log.0", "to process()", test_data) + service_lines = find_msg_in_log("autopsy_case.log.0", "to process()", test_data) service_list = [] for line in service_lines: words = line.split(" ") @@ -1253,11 +1255,16 @@ class Logs(object): # If this format changes, the tester will break i = words.index("secs.") times = words[i-4] + " " + print("1") times += words[i-3] + " " + print("2") times += words[i-2] + " " + print("3") times += words[i-1] + " " + print("4") times += words[i] service_list.append(times) + print("5") test_data.service_times = "; ".join(service_list) except (OSError, IOError) as e: Errors.print_error("Error: Unknown fatal error when finding service times.") @@ -1361,11 +1368,22 @@ def copy_logs(test_data): test_data: the TestData whose logs will be copied """ try: - log_dir = os.path.join("..", "..", "Testing","build","test","qa-functional","work","userdir0","var","log") + # copy logs from autopsy case's Log folder + log_dir = os.path.join(test_data.output_path, AUTOPSY_TEST_CASE, "Log") shutil.copytree(log_dir, test_data.logs_dir) + + # copy logs from userdir0 + log_dir = os.path.join("..", "..", "Testing","build","test","qa-functional","work","userdir0","var","log/") + print("log_dir is " + log_dir) + for log in log_dir: + new_name = "userdir0." + log + os.rename(log, new_name) + print("a log name is " + log) + print("log_dir is " + log_dir) + shutil.copytree(log_dir, test_data.logs_dir) except OSError as e: - printerror(test_data,"Error: Failed to copy the logs.") - printerror(test_data,str(e) + "\n") + print(test_data,"Error: Failed to copy the logs.") + print(test_data,str(e) + "\n") logging.warning(traceback.format_exc()) def setDay(): @@ -1655,6 +1673,7 @@ def search_logs(string, test_data): logs_path = test_data.logs_dir results = [] for file in os.listdir(logs_path): + print(file) log = codecs.open(make_path(logs_path, file), "r", "utf_8") for line in log: if string in line: @@ -1739,7 +1758,7 @@ def find_msg_in_log(log, string, test_data): """ lines = [] try: - lines = search_log("autopsy.log.0", string, test_data)[0] + lines = search_log("autopsy_case.log.0", string, test_data)[0] except (Exception) as e: # there weren't any matching messages found pass From 2382a3c023e284086ae22775833523863c502bd1 Mon Sep 17 00:00:00 2001 From: alexjacks92 Date: Tue, 3 Jun 2014 12:51:10 -0400 Subject: [PATCH 2/5] Tests now pull in both sets of logs for diffing. --- test/script/regression.py | 34 ++++++++++------------------------ 1 file changed, 10 insertions(+), 24 deletions(-) diff --git a/test/script/regression.py b/test/script/regression.py index ce289edbb4..628ba10538 100755 --- a/test/script/regression.py +++ b/test/script/regression.py @@ -1207,7 +1207,6 @@ class Logs(object): # Set the TestData start time based off the first line of autopsy_case.log.0 # *** If logging time format ever changes this will break *** test_data.start_date = log.readline().split(" org.")[0] - print("1210") # Set the test_data ending time based off the "create" time (when the file was copied) test_data.end_date = time.ctime(os.path.getmtime(log_path)) except IOError as e: @@ -1225,10 +1224,8 @@ class Logs(object): # Set Autopsy version, heap space, ingest time, and service times version_line = search_logs("INFO: Application name: Autopsy, version:", test_data)[0] - print("1228") test_data.autopsy_version = get_word_at(version_line, 5).rstrip(",") test_data.heap_space = search_logs("Heap memory usage:", test_data)[0].rstrip().split(": ")[1] - print("1231") ingest_line = search_logs("Ingest (including enqueue)", test_data)[0] test_data.total_ingest_time = get_word_at(ingest_line, 6).rstrip() @@ -1255,15 +1252,10 @@ class Logs(object): # If this format changes, the tester will break i = words.index("secs.") times = words[i-4] + " " - print("1") times += words[i-3] + " " - print("2") times += words[i-2] + " " - print("3") times += words[i-1] + " " - print("4") times += words[i] - service_list.append(times) print("5") test_data.service_times = "; ".join(service_list) except (OSError, IOError) as e: @@ -1374,13 +1366,12 @@ def copy_logs(test_data): # copy logs from userdir0 log_dir = os.path.join("..", "..", "Testing","build","test","qa-functional","work","userdir0","var","log/") - print("log_dir is " + log_dir) - for log in log_dir: - new_name = "userdir0." + log - os.rename(log, new_name) - print("a log name is " + log) - print("log_dir is " + log_dir) - shutil.copytree(log_dir, test_data.logs_dir) + for log in os.listdir(log_dir): + if log.find("log"): + new_name = log_dir + "userdir0." + log + log = log_dir + log + shutil.move(log, new_name) + shutil.copy(new_name, test_data.logs_dir) except OSError as e: print(test_data,"Error: Failed to copy the logs.") print(test_data,str(e) + "\n") @@ -1574,15 +1565,11 @@ class Args(object): arg = sys.argv.pop(0) nxtproc.append(arg) if(arg == "-f"): - #try: @@@ Commented out until a more specific except statement is added arg = sys.argv.pop(0) print("Running on a single file:") print(path_fix(arg) + "\n") self.single = True self.single_file = path_fix(arg) - #except: - # print("Error: No single file given.\n") - # return False elif(arg == "-r" or arg == "--rebuild"): print("Running in rebuild mode.\n") self.rebuild = True @@ -1673,7 +1660,6 @@ def search_logs(string, test_data): logs_path = test_data.logs_dir results = [] for file in os.listdir(logs_path): - print(file) log = codecs.open(make_path(logs_path, file), "r", "utf_8") for line in log: if string in line: @@ -1776,8 +1762,8 @@ def clear_dir(dir): os.makedirs(dir) return True; except OSError as e: - printerror(test_data,"Error: Cannot clear the given directory:") - printerror(test_data,dir + "\n") + print_error(test_data,"Error: Cannot clear the given directory:") + print_error(test_data,dir + "\n") print(str(e)) return False; @@ -1792,8 +1778,8 @@ def del_dir(dir): shutil.rmtree(dir) return True; except: - printerror(test_data,"Error: Cannot delete the given directory:") - printerror(test_data,dir + "\n") + print_error(test_data,"Error: Cannot delete the given directory:") + print_error(test_data,dir + "\n") return False; def get_file_in_dir(dir, ext): From 977d34b07bc9c33472402856e373501e6643e9ff Mon Sep 17 00:00:00 2001 From: alexjacks92 Date: Tue, 3 Jun 2014 13:30:50 -0400 Subject: [PATCH 3/5] Documentation and code clean up. --- test/script/regression.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/script/regression.py b/test/script/regression.py index 628ba10538..a4276bc9a6 100755 --- a/test/script/regression.py +++ b/test/script/regression.py @@ -1256,7 +1256,7 @@ class Logs(object): times += words[i-2] + " " times += words[i-1] + " " times += words[i] - print("5") + service_list.append(times) test_data.service_times = "; ".join(service_list) except (OSError, IOError) as e: Errors.print_error("Error: Unknown fatal error when finding service times.") @@ -1364,7 +1364,7 @@ def copy_logs(test_data): log_dir = os.path.join(test_data.output_path, AUTOPSY_TEST_CASE, "Log") shutil.copytree(log_dir, test_data.logs_dir) - # copy logs from userdir0 + # copy logs from userdir0/var/log log_dir = os.path.join("..", "..", "Testing","build","test","qa-functional","work","userdir0","var","log/") for log in os.listdir(log_dir): if log.find("log"): @@ -1373,8 +1373,8 @@ def copy_logs(test_data): shutil.move(log, new_name) shutil.copy(new_name, test_data.logs_dir) except OSError as e: - print(test_data,"Error: Failed to copy the logs.") - print(test_data,str(e) + "\n") + print_error(test_data,"Error: Failed to copy the logs.") + print_error(test_data,str(e) + "\n") logging.warning(traceback.format_exc()) def setDay(): From a3dfd9c4b5bec827bf247c554140b4af362968c8 Mon Sep 17 00:00:00 2001 From: alexjacks92 Date: Thu, 5 Jun 2014 10:04:59 -0400 Subject: [PATCH 4/5] Reverting to old log naming conventions. --- Core/src/org/sleuthkit/autopsy/coreutils/Logger.java | 4 ++-- test/script/regression.py | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/coreutils/Logger.java b/Core/src/org/sleuthkit/autopsy/coreutils/Logger.java index 6b2f5c474b..da29925272 100644 --- a/Core/src/org/sleuthkit/autopsy/coreutils/Logger.java +++ b/Core/src/org/sleuthkit/autopsy/coreutils/Logger.java @@ -33,8 +33,8 @@ public final class Logger extends java.util.logging.Logger { private static final String LOG_ENCODING = PlatformUtil.getLogFileEncoding(); private static final int LOG_SIZE = 0; // In bytes, zero is unlimited private static final int LOG_FILE_COUNT = 10; - private static final String LOG_WITHOUT_STACK_TRACES = "autopsy_case.log"; //NON-NLS - private static final String LOG_WITH_STACK_TRACES = "autopsy_case_traces.log"; //NON-NLS + private static final String LOG_WITHOUT_STACK_TRACES = "autopsy.log"; //NON-NLS + private static final String LOG_WITH_STACK_TRACES = "autopsy_traces.log"; //NON-NLS private static final CaseChangeListener caseChangeListener = new CaseChangeListener(); private static final Handler console = new java.util.logging.ConsoleHandler(); private static FileHandler userFriendlyLogFile = createFileHandler(PlatformUtil.getLogDirectory(), LOG_WITHOUT_STACK_TRACES); diff --git a/test/script/regression.py b/test/script/regression.py index a4276bc9a6..dafc239a1c 100755 --- a/test/script/regression.py +++ b/test/script/regression.py @@ -1200,17 +1200,17 @@ class Logs(object): test_data: the TestData to modify """ try: - # Open autopsy_case.log.0 - log_path = make_path(test_data.logs_dir, "autopsy_case.log.0") + # Open autopsy.log.0 + log_path = make_path(test_data.logs_dir, "autopsy.log.0") log = open(log_path) - # Set the TestData start time based off the first line of autopsy_case.log.0 + # Set the TestData start time based off the first line of autopsy.log.0 # *** If logging time format ever changes this will break *** test_data.start_date = log.readline().split(" org.")[0] # Set the test_data ending time based off the "create" time (when the file was copied) test_data.end_date = time.ctime(os.path.getmtime(log_path)) except IOError as e: - Errors.print_error("Error: Unable to open autopsy_case.log.0.") + Errors.print_error("Error: Unable to open autopsy.log.0.") Errors.print_error(str(e) + "\n") logging.warning(traceback.format_exc()) # Start date must look like: "Jul 16, 2012 12:57:53 PM" @@ -1244,7 +1244,7 @@ class Logs(object): logging.critical(traceback.format_exc()) print(traceback.format_exc()) try: - service_lines = find_msg_in_log("autopsy_case.log.0", "to process()", test_data) + service_lines = find_msg_in_log("autopsy.log.0", "to process()", test_data) service_list = [] for line in service_lines: words = line.split(" ") @@ -1744,7 +1744,7 @@ def find_msg_in_log(log, string, test_data): """ lines = [] try: - lines = search_log("autopsy_case.log.0", string, test_data)[0] + lines = search_log("autopsy.log.0", string, test_data)[0] except (Exception) as e: # there weren't any matching messages found pass From 7f4d2ac75f35661901471be25b8f7585f3c6f4ae Mon Sep 17 00:00:00 2001 From: alexjacks92 Date: Thu, 5 Jun 2014 10:23:06 -0400 Subject: [PATCH 5/5] Bug with moving userdir logs fixed. --- test/script/regression.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/script/regression.py b/test/script/regression.py index dafc239a1c..c36dcefc00 100755 --- a/test/script/regression.py +++ b/test/script/regression.py @@ -1371,7 +1371,8 @@ def copy_logs(test_data): new_name = log_dir + "userdir0." + log log = log_dir + log shutil.move(log, new_name) - shutil.copy(new_name, test_data.logs_dir) + shutil.copy(new_name, test_data.logs_dir) + shutil.move(new_name, log) except OSError as e: print_error(test_data,"Error: Failed to copy the logs.") print_error(test_data,str(e) + "\n")