From 50f49c529a906e333c3acef8a3e3049728a83627 Mon Sep 17 00:00:00 2001 From: alexjacks92 Date: Mon, 7 Apr 2014 15:56:47 -0400 Subject: [PATCH 1/3] Removing existing diff files before writing them out to the jenkins attachment location. --- test/script/regression.py | 134 +++++++++++++------------------------- 1 file changed, 46 insertions(+), 88 deletions(-) diff --git a/test/script/regression.py b/test/script/regression.py index 5e8025fb5d..1cf5c6e46c 100755 --- a/test/script/regression.py +++ b/test/script/regression.py @@ -34,16 +34,13 @@ import xml from time import localtime, strftime from xml.dom.minidom import parse, parseString import smtplib -from email.mime.image import MIMEImage -from email.mime.multipart import MIMEMultipart -from email.mime.text import MIMEText import re import zipfile import zlib -import Emailer import srcupdater from regression_utils import * - +import shutil +import ntpath # # Please read me... # @@ -130,7 +127,6 @@ class TestRunner(object): logres =[] for test_data in test_data_list: Errors.clear_print_logs() - Errors.set_testing_phase(test_data.image) if not (test_config.args.rebuild or os.path.exists(test_data.gold_archive)): msg = "Gold standard doesn't exist, skipping image:" Errors.print_error(msg) @@ -148,28 +144,16 @@ class TestRunner(object): time.sleep(10) Reports.write_html_foot(test_config.html_log) - # TODO: move this elsewhere - if (len(logres)>0): - for lm in logres: - for ln in lm: - Errors.add_email_msg(ln) - # TODO: possibly worth putting this in a sub method if all([ test_data.overall_passed for test_data in test_data_list ]): - Errors.add_email_msg("All images passed.\n") + pass else: - msg = "The following images failed:\n" - for test_data in test_data_list: - if not test_data.overall_passed: - msg += "\t" + test_data.image + "\n" - Errors.add_email_msg(msg) html = open(test_config.html_log) - Errors.add_email_attachment(html.name) + Errors.add_errors_out(html.name) html.close() - - if test_config.email_enabled: - Emailer.send_email(test_config.mail_to, test_config.mail_server, - test_config.mail_subject, Errors.email_body, Errors.email_attachs) + + if test_config.jenkins: + setupAttachments(Errors.errors_out, test_config) def _run_autopsy_ingest(test_data): """Run Autopsy ingest for the image in the given TestData. @@ -234,8 +218,8 @@ class TestRunner(object): diffFiles = [ f for f in os.listdir(test_data.output_path) if os.path.isfile(os.path.join(test_data.output_path,f)) ] for f in diffFiles: if f.endswith("Diff.txt"): - Errors.add_email_attachment(os.path.join(test_data.output_path, f)) - Errors.add_email_attachment(test_data.common_log_path) + Errors.add_errors_out(os.path.join(test_data.output_path, f)) + Errors.add_errors_out(test_data.common_log_path) return logres def _extract_gold(test_data): @@ -303,7 +287,6 @@ class TestRunner(object): shutil.copy(test_data.sorted_log, error_pth) except IOError as e: Errors.print_error(str(e)) - Errors.add_email_message("Not rebuilt properly") print(str(e)) print(traceback.format_exc()) # Rebuild the HTML report @@ -365,6 +348,7 @@ class TestRunner(object): test_data.ant.append("-Dgold_path=" + test_config.gold) test_data.ant.append("-Dout_path=" + make_local_path(test_data.output_path)) + test_data.ant.append("-Ddiff_dir="+ test_config.diff_dir) test_data.ant.append("-Dignore_unalloc=" + "%s" % test_config.args.unallocated) test_data.ant.append("-Dtest.timeout=" + str(test_config.timeout)) @@ -582,6 +566,7 @@ class TestConfiguration(object): images: a listof_Image, the images to be tested timeout: a Nat, the amount of time before killing the test ant: a listof_String, the ant command to run the tests + jenkins: a boolean, is this test running through a Jenkins job? """ def __init__(self, args): @@ -608,11 +593,7 @@ class TestConfiguration(object): # Infinite Testing info timer = 0 self.images = [] - # Email info - self.email_enabled = args.email_enabled - self.mail_server = "" - self.mail_to = "" - self.mail_subject = "" + self.jenkins = False # Set the timeout to something huge # The entire tester should not timeout before this number in ms # However it only seems to take about half this time @@ -650,15 +631,18 @@ class TestConfiguration(object): if parsed_config.getElementsByTagName("golddir"): self.gold = parsed_config.getElementsByTagName("golddir")[0].getAttribute("value").encode().decode("utf_8") self.img_gold = make_path(self.gold, 'tmp') - + if parsed_config.getElementsByTagName("jenkins"): + self.jenkins = True + if parsed_config.getElementsByTagName("diffdir"): + self.diff_dir = parsed_config.getElementsByTagName("diffdir")[0].getAttribute("value").encode().decode("utf_8") + else: + self.jenkins = False self._init_imgs(parsed_config) self._init_build_info(parsed_config) - self._init_email_info(parsed_config) except IOError as e: msg = "There was an error loading the configuration file.\n" msg += "\t" + str(e) - Errors.add_email_msg(msg) logging.critical(traceback.format_exc()) print(traceback.format_exc()) @@ -691,7 +675,6 @@ class TestConfiguration(object): else: msg = "File: " + value + " doesn't exist" Errors.print_error(msg) - Errors.add_email_msg(msg) image_count = len(self.images) # Sanity check to see if there are obvious gold images that we are not testing @@ -705,27 +688,6 @@ class TestConfiguration(object): elif (image_count < gold_count): print("******Alert: There are more gold standards than input images, this will not check all gold Standards.\n") - def _init_email_info(self, parsed_config): - """Initializes email information dictionary""" - email_elements = parsed_config.getElementsByTagName("email") - if email_elements: - mail_to = email_elements[0] - self.mail_to = mail_to.getAttribute("value").encode().decode("utf_8") - mail_server_elements = parsed_config.getElementsByTagName("mail_server") - if mail_server_elements: - mail_from = mail_server_elements[0] - self.mail_server = mail_from.getAttribute("value").encode().decode("utf_8") - subject_elements = parsed_config.getElementsByTagName("subject") - if subject_elements: - subject = subject_elements[0] - self.mail_subject = subject.getAttribute("value").encode().decode("utf_8") - if self.mail_server and self.mail_to and self.args.email_enabled: - self.email_enabled = True - print("Email will be sent to ", self.mail_to) - else: - self.email_enabled = False - print("No email will be sent.") - #-------------------------------------------------# # Functions relating to comparing outputs # @@ -803,11 +765,7 @@ class TestResultsDiffer(object): diff_file = codecs.open(diff_path, "wb", "utf_8") dffcmdlst = ["diff", output_file, gold_file] subprocess.call(dffcmdlst, stdout = diff_file) - Errors.add_email_attachment(diff_path) - msg = "There was a difference in " - msg += os.path.basename(output_file) + ".\n" - Errors.add_email_msg(msg) - Errors.print_error(msg) + Errors.add_errors_out(diff_path) return False else: return True @@ -1467,24 +1425,12 @@ class Errors: Attributes: printout: a listof_String, the non-error messages that were printed printerror: a listof_String, the error messages that were printed - email_body: a String, the body of the report email - email_msg_prefix: a String, the prefix for lines added to the email email_attchs: a listof_pathto_File, the files to be attached to the report email """ printout = [] printerror = [] - email_body = "" - email_msg_prefix = "Configuration" - email_attachs = [] - - def set_testing_phase(image_name): - """Change the email message prefix to be the given testing phase. - - Args: - image_name: a String, representing the current image being tested - """ - Errors.email_msg_prefix = image_name + errors_out = [] def print_out(msg): """Print out an informational message. @@ -1509,21 +1455,13 @@ class Errors: Errors.printout = [] Errors.printerror = [] - def add_email_msg(msg): - """Add the given message to the body of the report email. - - Args: - msg: a String, the message to be added to the email - """ - Errors.email_body += Errors.email_msg_prefix + ":" + msg - - def add_email_attachment(path): + def add_errors_out(path): """Add the given file to be an attachment for the report email Args: file: a pathto_File, the file to add """ - Errors.email_attachs.append(path) + Errors.errors_out.append(path) class DiffResults(object): @@ -1605,7 +1543,6 @@ class Args(object): self.exception = False self.exception_string = "" self.fr = False - self.email_enabled = False def parse(self): """Get the command line arguments and parse them.""" @@ -1665,8 +1602,6 @@ class Args(object): elif arg == "-fr" or arg == "--forcerun": print("Not downloading new images") self.fr = True - elif arg == "--email": - self.email_enabled = True else: print(usage()) return False @@ -1875,11 +1810,34 @@ def find_file_in_dir(dir, name, ext): except: raise DirNotFoundException(dir) +def setupAttachments(attachments, test_config): + """Move email attachments to the location specified in the config file. + Used for Jenkins build. + + Args: + attachments: a listof_String, the files to be moved + test_config: TestConfiguration, used to determine where to move the files to + """ + call = ['pwd'] + subprocess.call(call) + + # remove old diff files + filelist = [f for f in os.listdir(test_config.diff_dir) if (f.endswith(".txt") or f.endswith(".html"))] + for f in filelist: + if os.path.isfile(f): + os.remove(f) + + # move in the new diff files + for file in attachments: + filename = ntpath.basename(file) + destination = os.path.join(test_config.diff_dir, filename) + call = ['cp', file, destination] + subprocess.call(call) + class OS: LINUX, MAC, WIN, CYGWIN = range(4) - if __name__ == "__main__": global SYS if _platform == "linux" or _platform == "linux2": From 402d3651cc1338c96054fee20b9df8f17d599698 Mon Sep 17 00:00:00 2001 From: alexjacks92 Date: Tue, 8 Apr 2014 11:56:14 -0400 Subject: [PATCH 2/3] Cleaning up code to work when you aren't running a jenkins job. --- test/script/regression.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/script/regression.py b/test/script/regression.py index 1cf5c6e46c..7c33e62d33 100755 --- a/test/script/regression.py +++ b/test/script/regression.py @@ -348,7 +348,8 @@ class TestRunner(object): test_data.ant.append("-Dgold_path=" + test_config.gold) test_data.ant.append("-Dout_path=" + make_local_path(test_data.output_path)) - test_data.ant.append("-Ddiff_dir="+ test_config.diff_dir) + if test_config.jenkins: + test_data.ant.append("-Ddiff_dir="+ test_config.diff_dir) test_data.ant.append("-Dignore_unalloc=" + "%s" % test_config.args.unallocated) test_data.ant.append("-Dtest.timeout=" + str(test_config.timeout)) @@ -1824,8 +1825,8 @@ def setupAttachments(attachments, test_config): # remove old diff files filelist = [f for f in os.listdir(test_config.diff_dir) if (f.endswith(".txt") or f.endswith(".html"))] for f in filelist: - if os.path.isfile(f): - os.remove(f) + if os.path.isfile(test_config.diff_dir + "/" + f): + os.remove(test_config.diff_dir + "/" + f) # move in the new diff files for file in attachments: From 3bd0b6a1a1d0f98043083c021dc4e008d6c41ef2 Mon Sep 17 00:00:00 2001 From: alexjacks92 Date: Wed, 9 Apr 2014 15:16:05 -0400 Subject: [PATCH 3/3] Work to incorporate module descriptions into the UI. --- .../autopsy/ingest/Bundle.properties | 4 +- .../ingest/IngestJobConfigurationPanel.form | 83 +++++++++++++++---- .../ingest/IngestJobConfigurationPanel.java | 67 +++++++++++---- 3 files changed, 120 insertions(+), 34 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties b/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties index 837612fa53..0a088da4dd 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/ingest/Bundle.properties @@ -21,7 +21,6 @@ IngestJob.progress.fileIngest.displayName=File Ingest of {0} IngestJob.progress.cancelling={0} (Cancelling...) IngestJobConfigurationPanel.processUnallocCheckbox.toolTipText=Processes unallocated space, such as deleted files. Produces more complete results, but it may take longer to process on large images. IngestJobConfigurationPanel.processUnallocCheckbox.text=Process Unallocated Space -IngestJobConfigurationPanel.advancedButton.text=Advanced IngestJob.toString.text=ScheduledTask'{'input\={0}, modules\={1}'}' IngestJobLauncher.modName.tbirdParser.text=Thunderbird Parser IngestJobLauncher.modName.mboxParser.text=MBox Parser @@ -98,3 +97,6 @@ IngestScheduler.remove.exception.notSupported.msg=Not supported. IngestScheduler.DataSourceScheduler.exception.next.msg=There is no data source tasks in the queue, check hasNext() IngestScheduler.DataSourceScheduler.exception.remove.msg=Removing of scheduled data source ingest tasks is not supported. IngestScheduler.DataSourceScheduler.toString.size=DataSourceQueue, size\: {0} +Label1 +IngestJobConfigurationPanel.advancedButton.text=Advanced +IngestJobConfigurationPanel.advancedButton.actionCommand=Advanced diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobConfigurationPanel.form b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobConfigurationPanel.form index ae6e4c7e93..a0533766ff 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobConfigurationPanel.form +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobConfigurationPanel.form @@ -13,7 +13,7 @@ - + @@ -48,9 +48,9 @@ - + - + @@ -108,24 +108,40 @@ - - - - - - + + + + + + + + + + + + - + + + + + + + + + + + + + + + - - - - @@ -136,6 +152,9 @@ + + + @@ -164,6 +183,40 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -183,7 +236,7 @@ - + diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobConfigurationPanel.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobConfigurationPanel.java index 0e0ce8d74d..a0c617a4bd 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobConfigurationPanel.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobConfigurationPanel.java @@ -107,6 +107,7 @@ class IngestJobConfigurationPanel extends javax.swing.JPanel { simplePanel.revalidate(); simplePanel.repaint(); advancedButton.setEnabled(null != selectedModule.getGlobalSettingsPanel()); + descriptionLabel.setText(selectedModule.getDescription()); } } }); @@ -131,12 +132,14 @@ class IngestJobConfigurationPanel extends javax.swing.JPanel { jSeparator2 = new javax.swing.JSeparator(); jScrollPane1 = new javax.swing.JScrollPane(); simplePanel = new javax.swing.JPanel(); + scrollpane = new javax.swing.JScrollPane(); + descriptionLabel = new javax.swing.JTextArea(); processUnallocPanel = new javax.swing.JPanel(); processUnallocCheckbox = new javax.swing.JCheckBox(); setMaximumSize(new java.awt.Dimension(5750, 3000)); setMinimumSize(new java.awt.Dimension(522, 257)); - setPreferredSize(new java.awt.Dimension(575, 300)); + setPreferredSize(new java.awt.Dimension(575, 400)); modulesScrollPane.setBorder(javax.swing.BorderFactory.createLineBorder(new java.awt.Color(160, 160, 160))); modulesScrollPane.setPreferredSize(new java.awt.Dimension(160, 160)); @@ -158,6 +161,7 @@ class IngestJobConfigurationPanel extends javax.swing.JPanel { jPanel1.setPreferredSize(new java.awt.Dimension(338, 257)); advancedButton.setText(org.openide.util.NbBundle.getMessage(IngestJobConfigurationPanel.class, "IngestJobConfigurationPanel.advancedButton.text")); // NOI18N + advancedButton.setActionCommand(org.openide.util.NbBundle.getMessage(IngestJobConfigurationPanel.class, "IngestJobConfigurationPanel.advancedButton.actionCommand")); // NOI18N advancedButton.setEnabled(false); advancedButton.addActionListener(new java.awt.event.ActionListener() { public void actionPerformed(java.awt.event.ActionEvent evt) { @@ -171,25 +175,49 @@ class IngestJobConfigurationPanel extends javax.swing.JPanel { simplePanel.setLayout(new javax.swing.BoxLayout(simplePanel, javax.swing.BoxLayout.PAGE_AXIS)); jScrollPane1.setViewportView(simplePanel); + scrollpane.setBorder(null); + scrollpane.setHorizontalScrollBarPolicy(javax.swing.ScrollPaneConstants.HORIZONTAL_SCROLLBAR_NEVER); + scrollpane.setVerticalScrollBarPolicy(javax.swing.ScrollPaneConstants.VERTICAL_SCROLLBAR_NEVER); + + descriptionLabel.setEditable(false); + descriptionLabel.setBackground(new java.awt.Color(240, 240, 240)); + descriptionLabel.setColumns(20); + descriptionLabel.setFont(new java.awt.Font("Tahoma", 0, 11)); // NOI18N + descriptionLabel.setLineWrap(true); + descriptionLabel.setRows(5); + descriptionLabel.setWrapStyleWord(true); + descriptionLabel.setBorder(null); + scrollpane.setViewportView(descriptionLabel); + javax.swing.GroupLayout jPanel1Layout = new javax.swing.GroupLayout(jPanel1); jPanel1.setLayout(jPanel1Layout); jPanel1Layout.setHorizontalGroup( jPanel1Layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addComponent(jScrollPane1, javax.swing.GroupLayout.DEFAULT_SIZE, 326, Short.MAX_VALUE) - .addComponent(jSeparator2, javax.swing.GroupLayout.Alignment.TRAILING) - .addGroup(javax.swing.GroupLayout.Alignment.TRAILING, jPanel1Layout.createSequentialGroup() - .addContainerGap(javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) - .addComponent(advancedButton) - .addContainerGap()) + .addGroup(jPanel1Layout.createSequentialGroup() + .addContainerGap() + .addGroup(jPanel1Layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addComponent(jScrollPane1, javax.swing.GroupLayout.DEFAULT_SIZE, 316, Short.MAX_VALUE) + .addGroup(jPanel1Layout.createSequentialGroup() + .addComponent(scrollpane) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addComponent(advancedButton) + .addGap(14, 14, 14)))) + .addComponent(jSeparator2) ); jPanel1Layout.setVerticalGroup( jPanel1Layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addGroup(javax.swing.GroupLayout.Alignment.TRAILING, jPanel1Layout.createSequentialGroup() - .addComponent(jScrollPane1, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) - .addComponent(jSeparator2, javax.swing.GroupLayout.PREFERRED_SIZE, 10, javax.swing.GroupLayout.PREFERRED_SIZE) - .addGap(0, 0, 0) - .addComponent(advancedButton) + .addComponent(jScrollPane1, javax.swing.GroupLayout.DEFAULT_SIZE, 242, Short.MAX_VALUE) + .addGroup(jPanel1Layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addGroup(jPanel1Layout.createSequentialGroup() + .addGap(22, 22, 22) + .addComponent(scrollpane, javax.swing.GroupLayout.PREFERRED_SIZE, 65, javax.swing.GroupLayout.PREFERRED_SIZE)) + .addGroup(jPanel1Layout.createSequentialGroup() + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) + .addComponent(jSeparator2, javax.swing.GroupLayout.PREFERRED_SIZE, 10, javax.swing.GroupLayout.PREFERRED_SIZE)) + .addGroup(jPanel1Layout.createSequentialGroup() + .addGap(18, 18, 18) + .addComponent(advancedButton))) .addContainerGap()) ); @@ -210,7 +238,7 @@ class IngestJobConfigurationPanel extends javax.swing.JPanel { .addGroup(processUnallocPanelLayout.createSequentialGroup() .addContainerGap() .addComponent(processUnallocCheckbox) - .addContainerGap(60, Short.MAX_VALUE)) + .addContainerGap(108, Short.MAX_VALUE)) ); processUnallocPanelLayout.setVerticalGroup( processUnallocPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) @@ -238,15 +266,19 @@ class IngestJobConfigurationPanel extends javax.swing.JPanel { .addGroup(layout.createSequentialGroup() .addContainerGap() .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addComponent(jPanel1, javax.swing.GroupLayout.DEFAULT_SIZE, 278, Short.MAX_VALUE) + .addComponent(jPanel1, javax.swing.GroupLayout.DEFAULT_SIZE, 342, Short.MAX_VALUE) .addGroup(layout.createSequentialGroup() - .addComponent(modulesScrollPane, javax.swing.GroupLayout.DEFAULT_SIZE, 233, Short.MAX_VALUE) + .addComponent(modulesScrollPane, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) .addComponent(processUnallocPanel, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE))) .addContainerGap()) ); }// //GEN-END:initComponents + private void processUnallocCheckboxActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_processUnallocCheckboxActionPerformed + processUnallocatedSpace = processUnallocCheckbox.isSelected(); + }//GEN-LAST:event_processUnallocCheckboxActionPerformed + private void advancedButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_advancedButtonActionPerformed final AdvancedConfigurationDialog dialog = new AdvancedConfigurationDialog(); @@ -270,11 +302,9 @@ class IngestJobConfigurationPanel extends javax.swing.JPanel { dialog.display(selectedModule.getGlobalSettingsPanel()); }//GEN-LAST:event_advancedButtonActionPerformed - private void processUnallocCheckboxActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_processUnallocCheckboxActionPerformed - processUnallocatedSpace = processUnallocCheckbox.isSelected(); - }//GEN-LAST:event_processUnallocCheckboxActionPerformed // Variables declaration - do not modify//GEN-BEGIN:variables private javax.swing.JButton advancedButton; + private javax.swing.JTextArea descriptionLabel; private javax.swing.JPanel jPanel1; private javax.swing.JScrollPane jScrollPane1; private javax.swing.JSeparator jSeparator2; @@ -282,6 +312,7 @@ class IngestJobConfigurationPanel extends javax.swing.JPanel { private javax.swing.JTable modulesTable; private javax.swing.JCheckBox processUnallocCheckbox; private javax.swing.JPanel processUnallocPanel; + private javax.swing.JScrollPane scrollpane; private javax.swing.JPanel simplePanel; private javax.swing.ButtonGroup timeGroup; // End of variables declaration//GEN-END:variables