Merge remote-tracking branch 'upstream/develop' into keywordDocs

Core/src/org/sleuthkit/autopsy/ingest/IngestProfiles.java

@@ -25,6 +25,7 @@ import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.List;
 import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.FilenameUtils;
 import org.openide.util.Exceptions;
 import org.sleuthkit.autopsy.coreutils.ModuleSettings;
 import org.sleuthkit.autopsy.coreutils.PlatformUtil;
@@ -38,6 +39,7 @@ public final class IngestProfiles {
     private static final String PROFILE_NAME_KEY = "Profile_Name";
     private static final String PROFILE_DESC_KEY = "Profile_Description";
     private static final String PROFILE_FILTER_KEY = "Profile_Filter";
+    private static final String PROFILE_FILE_EXT = ".properties";
 
     /**
      * Gets the collection of profiles which currently exist.
@@ -50,7 +52,7 @@ public final class IngestProfiles {
         List<IngestProfile> profileList = new ArrayList<>();
         if (directoryListing != null) {
             for (File child : directoryListing) {
-                String name = child.getName().split("\\.")[0];
+                String name = FilenameUtils.removeExtension(child.getName()); 
                 String context = PROFILE_FOLDER + File.separator + name;
                 String desc = ModuleSettings.getConfigSetting(context, PROFILE_DESC_KEY);
                 String fileIngestFilter = ModuleSettings.getConfigSetting(context, PROFILE_FILTER_KEY);
@@ -137,8 +139,8 @@ public final class IngestProfiles {
          */
         synchronized static void deleteProfile(IngestProfile selectedProfile) {
             try {
-                Files.deleteIfExists(Paths.get(PlatformUtil.getUserConfigDirectory(), PROFILE_FOLDER, selectedProfile.getName() + ".properties"));
+                Files.deleteIfExists(Paths.get(PlatformUtil.getUserConfigDirectory(), PROFILE_FOLDER, selectedProfile.getName() + PROFILE_FILE_EXT));
-                Files.deleteIfExists(Paths.get(PlatformUtil.getUserConfigDirectory(), selectedProfile.getName() + ".properties"));
+                Files.deleteIfExists(Paths.get(PlatformUtil.getUserConfigDirectory(), selectedProfile.getName() + PROFILE_FILE_EXT));
                 FileUtils.deleteDirectory(IngestJobSettings.getSavedModuleSettingsFolder(selectedProfile.getName() + File.separator).toFile());
             } catch (IOException ex) {
                 Exceptions.printStackTrace(ex);
@@ -153,11 +155,11 @@ public final class IngestProfiles {
          */
         synchronized static void renameProfile(String oldName, String newName) {
             if (!oldName.equals(newName)) { //if renameProfile was called with the new name being the same as the old name, it is complete already
-                File oldFile = Paths.get(PlatformUtil.getUserConfigDirectory(), PROFILE_FOLDER, oldName + ".properties").toFile();
+                File oldFile = Paths.get(PlatformUtil.getUserConfigDirectory(), PROFILE_FOLDER, oldName + PROFILE_FILE_EXT).toFile();
-                File newFile = Paths.get(PlatformUtil.getUserConfigDirectory(), PROFILE_FOLDER, newName + ".properties").toFile();
+                File newFile = Paths.get(PlatformUtil.getUserConfigDirectory(), PROFILE_FOLDER, newName + PROFILE_FILE_EXT).toFile();
                 oldFile.renameTo(newFile);
-                oldFile = Paths.get(PlatformUtil.getUserConfigDirectory(), oldName + ".properties").toFile();
+                oldFile = Paths.get(PlatformUtil.getUserConfigDirectory(), oldName + PROFILE_FILE_EXT).toFile();
-                newFile = Paths.get(PlatformUtil.getUserConfigDirectory(), newName + ".properties").toFile();
+                newFile = Paths.get(PlatformUtil.getUserConfigDirectory(), newName + PROFILE_FILE_EXT).toFile();
                 oldFile.renameTo(newFile);
                 oldFile = IngestJobSettings.getSavedModuleSettingsFolder(oldName + File.separator).toFile();
                 newFile = IngestJobSettings.getSavedModuleSettingsFolder(newName + File.separator).toFile();

Core/src/org/sleuthkit/autopsy/ingest/ProfileSettingsPanel.form

@@ -105,7 +105,7 @@
                   <Group type="102" attributes="0">
                       <Group type="103" groupAlignment="0" attributes="0">
                           <Group type="102" attributes="0">
-                              <Component id="profileDescLabel" pref="15" max="32767" attributes="0"/>
+                              <Component id="profileDescLabel" min="-2" pref="15" max="-2" attributes="0"/>
                               <EmptySpace min="-2" max="-2" attributes="0"/>
                               <Component id="profileDescPane" min="-2" max="-2" attributes="0"/>
                               <EmptySpace min="-2" max="-2" attributes="0"/>

Core/src/org/sleuthkit/autopsy/ingest/ProfileSettingsPanel.java

@@ -265,7 +265,7 @@ class ProfileSettingsPanel extends IngestModuleGlobalSettingsPanel implements Op
                     .addGroup(layout.createSequentialGroup()
                         .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
                             .addGroup(layout.createSequentialGroup()
-                                .addComponent(profileDescLabel, javax.swing.GroupLayout.DEFAULT_SIZE, 15, Short.MAX_VALUE)
+                                .addComponent(profileDescLabel, javax.swing.GroupLayout.PREFERRED_SIZE, 15, javax.swing.GroupLayout.PREFERRED_SIZE)
                                 .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
                                 .addComponent(profileDescPane, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)
                                 .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)

Core/src/org/sleuthkit/autopsy/modules/interestingitems/Bundle.properties

@@ -44,8 +44,8 @@ FilesSetRulePanel.mimeCheck.text=MIME Type:
 FilesSetRulePanel.fileSizeCheck.text=File Size:
 FilesSetRulePanel.filesRadioButton.text=Files
 FilesSetRulePanel.dirsRadioButton.text=Directories
-FilesSetDefsPanel.interesting.setsListLabel.text=Rule Sets
+FilesSetDefsPanel.interesting.setsListLabel.text=Rule Sets:
-FilesSetDefsPanel.ingest.setsListLabel.text=File Ingest Filters
+FilesSetDefsPanel.ingest.setsListLabel.text=File Ingest Filters:
 FilesSetDefsPanel.interesting.jTextArea1.text=This module allows you to find files that match specified criteria. Each set has a list of rules, which will match on their chosen file characteristics. A file need only match one rule to be found.
 FilesSetDefsPanel.ingest.jTextArea1.text=Add rules so that only a subset of the files in a data source are analyzed. Rules are organized into sets and only one set can be used at a time. A file need only match one rule to be analyzed.
 FilesSetDefsPanel.interesting.editSetButton.text=Edit Set

KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties

@@ -20,9 +20,9 @@ ExtractedContentPanel.copyMenuItem.text=Copy
 ExtractedContentPanel.selectAllMenuItem.text=Select All
 KeywordSearchEditListPanel.saveListButton.text=Copy List
 KeywordSearchEditListPanel.addWordField.text=
-KeywordSearchEditListPanel.addWordButton.text=New keyword
+KeywordSearchEditListPanel.addWordButton.text=New Keyword
 KeywordSearchEditListPanel.chRegex.text=Regular Expression
-KeywordSearchEditListPanel.deleteWordButton.text=Delete keywords
+KeywordSearchEditListPanel.deleteWordButton.text=Delete Keywords
 KeywordSearchEditListPanel.cutMenuItem.text=Cut
 KeywordSearchEditListPanel.selectAllMenuItem.text=Select All
 KeywordSearchEditListPanel.pasteMenuItem.text=Paste
@@ -46,7 +46,7 @@ ExtractedContentPanel.pageCurLabel.text=-
 ExtractedContentPanel.pageTotalLabel.text=-
 ExtractedContentPanel.hitLabel.toolTipText=
 KeywordSearchEditListPanel.ingestMessagesCheckbox.text=Send ingest inbox messages for each hit
-KeywordSearchEditListPanel.ingestMessagesCheckbox.toolTipText=Send messages during ingest when hits on keyword from this list occur
+KeywordSearchEditListPanel.ingestMessagesCheckbox.toolTipText=Send messages during ingest when hits on keywords from this list occur
 KeywordSearchEditListPanel.keywordOptionsLabel.text=Keyword Options
 KeywordSearchEditListPanel.listOptionsLabel.text=List Options
 KeywordSearchListsManagementPanel.keywordListsLabel.text=Keyword Lists:
@@ -79,7 +79,7 @@ KeywordSearchConfigurationPanel.customizeComponents.genTabTitle=General
 KeywordSearchConfigurationPanel.customizeComponents.listLabToolTip=List configuration
 KeywordSearchConfigurationPanel.customizeComponents.stringExtToolTip=String extraction configuration for Keyword Search Ingest
 KeywordSearchConfigurationPanel.customizeComponents.genTabToolTip=General configuration
-KeywordSearchConfigurationPanel1.customizeComponents.title=Delete a keyword list
+KeywordSearchConfigurationPanel1.customizeComponents.title=Delete a Keyword List
 KeywordSearchConfigurationPanel1.customizeComponents.body=This will delete the keyword list globally (for all Cases). Do you want to proceed with the deletion?
 KeywordSearchConfigurationPanel1.customizeComponents.keywordListEmptyErr=Keyword List is empty and cannot be saved
 KeywordSearch.newKwListTitle=New keyword list name\:
@@ -99,12 +99,12 @@ KeywordSearchEditListPanel.addWordButtonAction.kwAlreadyExistsMsg=Keyword alread
 KeywordSearchEditListPanel.invalidKwMsg=Invalid keyword pattern.  Use words or a correct regex pattern.
 KeywordSearchEditListPanel.removeKwMsg=Removing a keyword
 KeywordSearchEditListPanel.deleteWordButtonActionPerformed.delConfirmMsg=This will remove a keyword from the list globally (for all Cases). Do you want to proceed?
-KeywordSearchEditListPanel.exportButtonActionPerformed.fileFilterLabel=Keyword List XML file
+KeywordSearchEditListPanel.exportButtonActionPerformed.fileFilterLabel=Keyword List XML Files
 KeywordSearchEditListPanel.exportButtonActionPerformed.fileExistPrompt=File {0} exists, overwrite?
 KeywordSearchEditListPanel.exportButtonActionPerformed.kwListExportedMsg=Keyword lists exported
 KeywordSearchEditListPanel.kwColName=Keyword
 KeywordSearchEditListPanel.addKeyword.message=Add a new word to the keyword search list\:
-KeywordSearchEditListPanel.addKeyword.title=New keyword
+KeywordSearchEditListPanel.addKeyword.title=New Keyword
 KeywordSearchFilterNode.getFileActions.openExternViewActLbl=Open in External Viewer
 KeywordSearchFilterNode.getFileActions.searchSameMd5=Search for files with the same MD5 hash
 KeywordSearchFilterNode.getFileActions.viewInNewWinActionLbl=View in New Window
@@ -273,8 +273,8 @@ KeywordSearchListsAbstract.saveList.errMsg2.msg=A module caused an error listeni
 KeywordSearchListsAbstract.writeLists.errMsg1.msg=A module caused an error listening to KeywordSearchListsAbstract updates. See log to determine which module. Some data could be incomplete.
 KeywordSearchListsAbstract.writeLists.errMsg2.msg=A module caused an error listening to KeywordSearchListsAbstract updates. See log to determine which module. Some data could be incomplete.
 KeywordSearchListsAbstract.deleteList.errMsg1.msg=A module caused an error listening to KeywordSearchListsAbstract updates. See log to determine which module. Some data could be incomplete.
-KeywordSearchListsManagementPanel.newKeywordListDescription=Keyword List <{0}> already exists as a read-only list. Do you want to replace it for the duration of the program (the change will not be persistent).
+KeywordSearchListsManagementPanel.newKeywordListDescription=Keyword list <{0}> already exists as a read-only list. Do you want to replace it for the duration of the program (the change will not be persistent).
-KeywordSearchListsManagementPanel.newKeywordListDescription2=Keyword List <{0}> already exists, do you want to replace it?
+KeywordSearchListsManagementPanel.newKeywordListDescription2=Keyword list <{0}> already exists, do you want to replace it?
 KeywordSearchModuleFactory.getIngestJobSettingsPanel.exception.msg=Expected settings argument to be instanceof KeywordSearchJobSettings
 KeywordSearchModuleFactory.createFileIngestModule.exception.msg=Expected settings argument to be instanceof KeywordSearchJobSettings
 SearchRunner.Searcher.done.err.msg=Error performing keyword search
@@ -300,9 +300,9 @@ AddKeywordsDialog.enterKeywordsLabel.text=Enter keywords (one per line) below:
 AddKeywordsDialog.pasteButton.text=Paste From Clipboard
 AddKeywordsDialog.addButton.text=OK
 AddKeywordsDialog.cancelButton.text=Cancel
-AddKeywordsDialog.addKeywordsTitle.text=New keywords
+AddKeywordsDialog.addKeywordsTitle.text=New Keywords
-GlobalEditListPanel.newKeywordsButton.text=New keywords
+GlobalEditListPanel.newKeywordsButton.text=New Keywords
-GlobalEditListPanel.addKeywordResults.text=Add keyword results
+GlobalEditListPanel.addKeywordResults.text=Add Keyword Results
 GlobalEditListPanel.keywordsAdded.text={0} keyword was successfully added.
 GlobalEditListPanel.keywordsAddedPlural.text={0} keywords were successfully added.
 GlobalEditListPanel.keywordDupesSkipped.text={0} keyword was already in the list.
@@ -312,8 +312,8 @@ GlobalEditListPanel.keywordErrorsPlural.text={0} keywords could not be parsed. P
 GlobalListsManagementPanel.exportButton.text=Export List
 GlobalListsManagementPanel.deleteListButton.text=Delete List
 GlobalListsManagementPanel.copyListButton.text=Copy List
-GlobalListsManagementPanel.renameListButton.text=Rename List
+GlobalListsManagementPanel.renameListButton.text=Edit List
-GlobalEditListPanel.editWordButton.text=Edit keyword
+GlobalEditListPanel.editWordButton.text=Edit Keyword
 SolrSearchService.ServiceName=Solr Keyword Search Service
 SolrSearchService.IndexUpgradeDialog.title=Text Index Upgrade Required In Order To Open Case
 SolrSearchService.IndexUpgradeDialog.msg=<html>The text index upgrade can take some time. <br />When completed, you will be able to see existing keyword search results and perform literal keyword searches,<br />but you will not be able to add new text to the index or perform regex searches. You may instead open the case<br /> with your previous version of this application. Do you wish to proceed with the index upgrade?</html>

KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Index.java

@@ -87,20 +87,15 @@ final class Index {
 
         String result;
         
-        // Remove all non-ASCII characters
+        // Allow these characters: '-', '.', '0'-'9', 'A'-'Z', '_', and 'a'-'z'.
-        result = coreName.replaceAll("[^\\p{ASCII}]", "_"); //NON-NLS
+        // Replace all else with '_'.
-
+        result = coreName.replaceAll("[^-.0-9A-Z_a-z]", "_"); // NON-NLS
-        // Remove all control characters
-        result = result.replaceAll("[\\p{Cntrl}]", "_"); //NON-NLS
-
-        // Remove spaces / \ : ? ' "
-        result = result.replaceAll("[ /?:'\"\\\\]", "_"); //NON-NLS
 
         // Make it all lowercase
         result = result.toLowerCase();
 
         // Must not start with hyphen
-        if (result.length() > 0 && !(Character.isLetter(result.codePointAt(0))) && !(result.codePointAt(0) == '-')) {
+        if (result.length() > 0 && (result.codePointAt(0) == '-')) {
             result = "_" + result;
         }
 

KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java

@@ -370,7 +370,7 @@ public final class SearchRunner {
         private List<Keyword> keywords; //keywords to search
         private List<String> keywordListNames; // lists currently being searched
         private List<KeywordList> keywordLists;
-        private Map<String, KeywordList> keywordToList; //keyword to list name mapping
+        private Map<Keyword, KeywordList> keywordToList; //keyword to list name mapping
         private AggregateProgressHandle progressGroup;
         private final Logger logger = Logger.getLogger(SearchRunner.Searcher.class.getName());
         private boolean finalRun = false;
@@ -431,8 +431,7 @@ public final class SearchRunner {
                         return null;
                     }
 
-                    final String queryStr = keyword.getSearchTerm();
+                    final KeywordList keywordList = keywordToList.get(keyword);
-                    final KeywordList keywordList = keywordToList.get(queryStr);
 
                     //new subProgress will be active after the initial query
                     //when we know number of hits to start() with
@@ -545,7 +544,7 @@ public final class SearchRunner {
                 keywordLists.add(list);
                 for (Keyword k : list.getKeywords()) {
                     keywords.add(k);
-                    keywordToList.put(k.getSearchTerm(), list);
+                    keywordToList.put(k, list);
                 }
             }
         }

KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java

@@ -565,7 +565,7 @@ public class Server {
         try {
             // Close any open core before stopping server
             closeCore();
-        } catch (KeywordSearchModuleException | NoOpenCoreException e) {
+        } catch (KeywordSearchModuleException e) {
             logger.log(Level.WARNING, "Failed to close core: ", e); //NON-NLS
         }
 
@@ -704,15 +704,14 @@ public class Server {
         }
     }
     
-    void closeCore() throws KeywordSearchModuleException, NoOpenCoreException {
+    void closeCore() throws KeywordSearchModuleException {
         currentCoreLock.writeLock().lock();
         try {
-            if (null == currentCore) {
+            if (null != currentCore) {
-                throw new NoOpenCoreException();
-            }
                 currentCore.close();
                 currentCore = null;
                 serverAction.putValue(CORE_EVT, CORE_EVT_STATES.STOPPED);
+            }
         } finally {
             currentCoreLock.writeLock().unlock();
         }
@@ -760,7 +759,7 @@ public class Server {
                     closeCore();
                 }
             }
-        } catch (KeywordSearchModuleException | NoOpenCoreException ex) {
+        } catch (KeywordSearchModuleException ex) {
             throw new KeywordSearchServiceException(NbBundle.getMessage(Server.class, "Server.close.exception.msg"), ex);
         } finally {
             currentCoreLock.readLock().unlock();

KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java

@@ -375,7 +375,7 @@ public class SolrSearchService implements KeywordSearchService, AutopsyService {
 
         try {
             KeywordSearch.getServer().closeCore();
-        } catch (KeywordSearchModuleException | NoOpenCoreException ex) {
+        } catch (KeywordSearchModuleException ex) {
             throw new AutopsyServiceException(String.format("Failed to close core for %s", context.getCase().getCaseDirectory()), ex);
         }
     }

Testing/test/qa-functional/src/org/sleuthkit/autopsy/testing/RegressionTest.java

@@ -25,6 +25,7 @@ import junit.framework.Test;
 import junit.framework.TestCase;
 import org.netbeans.jemmy.Timeouts;
 import org.netbeans.junit.NbModuleSuite;
+import org.sleuthkit.autopsy.core.UserPreferences;
 
 /**
  * This test expects the following system properties to be set: img_path: The
@@ -98,6 +99,7 @@ public class RegressionTest extends TestCase {
     public void setUp() {
         logger.info("########  " + AutopsyTestCases.getEscapedPath(System.getProperty("img_path")) + "  #######");
         Timeouts.setDefault("ComponentOperator.WaitComponentTimeout", 1000000);
+        UserPreferences.setNumberOfFileIngestThreads(1);  //Let nightly test using 1 ingest thread to avoid ordering results in report of insertion to tsk tables
     }
 
     /**