From 04a51d6234c0998b74c01a5705017eeb422ef2e5 Mon Sep 17 00:00:00 2001 From: sidheshenator Date: Mon, 23 Mar 2015 14:29:19 -0400 Subject: [PATCH 1/4] time diff inside gold xip --- test/script/config.xml | 3 ++- test/script/regression.py | 32 ++++++++++++++++++++++++++++---- 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/test/script/config.xml b/test/script/config.xml index eece486d3f..55894b1651 100644 --- a/test/script/config.xml +++ b/test/script/config.xml @@ -31,7 +31,8 @@ It is up to the user to distinguish between the paths when adding to this file. - + + diff --git a/test/script/regression.py b/test/script/regression.py index 78a21ed5ff..3f6192d6ab 100755 --- a/test/script/regression.py +++ b/test/script/regression.py @@ -157,6 +157,9 @@ class TestRunner(object): # Analyze the given image TestRunner._run_autopsy_ingest(test_data) + # Generate HTML report + Reports.write_html_foot(test_config.html_log) + # Either copy the data or compare the data if test_config.args.rebuild: TestRunner.rebuild(test_data) @@ -167,8 +170,7 @@ class TestRunner(object): test_data.printerror = Errors.printerror # give solr process time to die. time.sleep(10) - - Reports.write_html_foot(test_config.html_log) + # This code was causing errors with paths, so its disabled #if test_config.jenkins: @@ -364,6 +366,20 @@ class TestRunner(object): errors.append("Error: Unknown fatal error when rebuilding the gold html report.") errors.append(str(e) + "\n") print(traceback.format_exc()) + # Rebuild the Run time report + if(test_data.main_config.timing): + current_run_time_path = os.path.join(test_data.get_run_time_path(), test_data.image_name + "_time.txt") + file = open(current_run_time_path, "w") + file.writelines(test_data.total_ingest_time) + file.close() + if(os.path.exists(current_run_time_path)): + gold_run_time_path = make_path(tmpdir, test_data.image_name + "_time.txt") + try: + shutil.copy(current_run_time_path, gold_run_time_path) + except IOError as e: + Errors.print_error(str(e)) + print(str(e)) + print(traceback.format_exc()) oldcwd = os.getcwd() zpdir = gold_dir os.chdir(zpdir) @@ -594,6 +610,7 @@ class TestData(object): def get_run_time_path(self): """Get the path to the run time storage file." """ + return _get_path_to_file() return os.path.join("..", "input") def _get_path_to_file(self, file_type, file_name): @@ -801,7 +818,8 @@ class TestResultsDiffer(object): output_report_path = test_data.get_html_report_path(DBType.OUTPUT) passed = TestResultsDiffer._html_report_diff(test_data) test_data.html_report_passed = passed - + + # Compare time outputs # Clean up tmp folder del_dir(test_data.gold_data_dir) @@ -901,7 +919,12 @@ class TestResultsDiffer(object): line = file.readline() oldtime = int(line[:line.find("ms")].replace(',', '')) file.close() - + + # should be the old time from gold standard. + # 1. rebuild gold must generate this _time.txt file + # NOTE: Refer to how old files end up in that zip. Also note how their location is determined. Check rebuild() + # 2. _run_time_diff() must use time from gold standard. + # NOTE: Refer to how HTML diff gets thing from the zipped gold standard. newtime = test_data.total_ingest_time newtime = int(newtime[:newtime.find("ms")].replace(',', '')) @@ -1192,6 +1215,7 @@ class Reports(object): def _write_time(test_data): """Write out the time ingest took. For jenkins purposes. + Copies the _time.txt file the the input dir. Args: test_data: the TestData From 72505b40d670db931a2cfd3a402590f498c47de4 Mon Sep 17 00:00:00 2001 From: sidheshenator Date: Mon, 23 Mar 2015 16:26:19 -0400 Subject: [PATCH 2/4] regression.py compares run time diff between current and gold --- test/script/regression.py | 25 ++++++++----------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/test/script/regression.py b/test/script/regression.py index 3f6192d6ab..3e7c8730e8 100755 --- a/test/script/regression.py +++ b/test/script/regression.py @@ -245,9 +245,6 @@ class TestRunner(object): # run time test only for the specific jenkins test if test_data.main_config.timing: - old_time_path = test_data.get_run_time_path() - passed = TestResultsDiffer._run_time_diff(test_data, old_time_path) - test_data.run_time_passed = passed print("Run time test passed: ", test_data.run_time_passed) test_data.overall_passed = (test_data.html_report_passed and test_data.errors_diff_passed and test_data.db_diff_passed and @@ -368,18 +365,9 @@ class TestRunner(object): print(traceback.format_exc()) # Rebuild the Run time report if(test_data.main_config.timing): - current_run_time_path = os.path.join(test_data.get_run_time_path(), test_data.image_name + "_time.txt") - file = open(current_run_time_path, "w") + file = open(test_data.get_run_time_path(DBType.GOLD), "w") file.writelines(test_data.total_ingest_time) file.close() - if(os.path.exists(current_run_time_path)): - gold_run_time_path = make_path(tmpdir, test_data.image_name + "_time.txt") - try: - shutil.copy(current_run_time_path, gold_run_time_path) - except IOError as e: - Errors.print_error(str(e)) - print(str(e)) - print(traceback.format_exc()) oldcwd = os.getcwd() zpdir = gold_dir os.chdir(zpdir) @@ -607,11 +595,10 @@ class TestData(object): """ return self._get_path_to_file(file_type, "DBDump.txt") - def get_run_time_path(self): + def get_run_time_path(self, file_type): """Get the path to the run time storage file." """ - return _get_path_to_file() - return os.path.join("..", "input") + return self._get_path_to_file(file_type, "_time.txt") def _get_path_to_file(self, file_type, file_name): """Get the path to the specified file with the specified type. @@ -820,6 +807,10 @@ class TestResultsDiffer(object): test_data.html_report_passed = passed # Compare time outputs + if test_data.main_config.timing: + old_time_path = test_data.get_run_time_path(DBType.GOLD) + passed = TestResultsDiffer._run_time_diff(test_data, old_time_path) + test_data.run_time_passed = passed # Clean up tmp folder del_dir(test_data.gold_data_dir) @@ -915,7 +906,7 @@ class TestResultsDiffer(object): old_time_path: path to the log containing the run time from a previous test """ # read in time - file = open(old_time_path + '/' + test_data.image + "_time.txt", "r") + file = open(old_time_path, "r") line = file.readline() oldtime = int(line[:line.find("ms")].replace(',', '')) file.close() From 81a4549b4d67cb64095a167d6494ebbd9f211b3a Mon Sep 17 00:00:00 2001 From: Sidhesh Mhatre Date: Mon, 23 Mar 2015 16:28:31 -0400 Subject: [PATCH 3/4] Reverted to default config.xml --- test/script/config.xml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/script/config.xml b/test/script/config.xml index 55894b1651..eece486d3f 100644 --- a/test/script/config.xml +++ b/test/script/config.xml @@ -31,8 +31,7 @@ It is up to the user to distinguish between the paths when adding to this file. - - + From 1412365980f0cbcc7d83998c50e3a303696d327a Mon Sep 17 00:00:00 2001 From: Sidhesh Mhatre Date: Mon, 23 Mar 2015 17:19:01 -0400 Subject: [PATCH 4/4] txt file showing current run time created --- test/script/regression.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/test/script/regression.py b/test/script/regression.py index 3e7c8730e8..95742d63fe 100755 --- a/test/script/regression.py +++ b/test/script/regression.py @@ -911,12 +911,13 @@ class TestResultsDiffer(object): oldtime = int(line[:line.find("ms")].replace(',', '')) file.close() - # should be the old time from gold standard. - # 1. rebuild gold must generate this _time.txt file - # NOTE: Refer to how old files end up in that zip. Also note how their location is determined. Check rebuild() - # 2. _run_time_diff() must use time from gold standard. - # NOTE: Refer to how HTML diff gets thing from the zipped gold standard. newtime = test_data.total_ingest_time + + # write newtime to the file inside the report dir. + file = open(test_data.get_run_time_path(DBType.OUTPUT), "w") + file.writelines(newtime) + file.close() + newtime = int(newtime[:newtime.find("ms")].replace(',', '')) # run the test, 5% tolerance