Merge pull request #4819 from markmckinnon/2410-delete-data-sources-from-the-case-text-index

2410 delete data sources from the case text index

Core/src/org/sleuthkit/autopsy/actions/DeleteDataSourceAction.java

@@ -21,10 +21,13 @@ package org.sleuthkit.autopsy.actions;
 import java.awt.event.ActionEvent;
 import java.util.logging.Level;
 import javax.swing.AbstractAction;
+import org.openide.util.Lookup;
 import org.openide.util.NbBundle;
 import org.sleuthkit.autopsy.casemodule.Case;
 import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException;
 import org.sleuthkit.autopsy.coreutils.Logger;
+import org.sleuthkit.autopsy.keywordsearchservice.KeywordSearchService;
+import org.sleuthkit.autopsy.keywordsearchservice.KeywordSearchServiceException;
 import org.sleuthkit.datamodel.TskCoreException;
 
 /**
@@ -32,21 +35,30 @@ import org.sleuthkit.datamodel.TskCoreException;
  */
 public final class DeleteDataSourceAction extends AbstractAction {
     private static final Logger logger = Logger.getLogger(DeleteDataSourceAction.class.getName());
-    private final Long selectedDataSource;
+    private final Long dataSourceId;
     
     @NbBundle.Messages({"DeleteDataSourceAction.name.text=Delete Data Source"})
-    public DeleteDataSourceAction(Long selectedDataSource) {
+    public DeleteDataSourceAction(Long dataSourceId) {
         super(Bundle.DeleteDataSourceAction_name_text());
-        this.selectedDataSource = selectedDataSource;
+        this.dataSourceId = dataSourceId;
     }
 
     @Override
     public void actionPerformed(ActionEvent event) {
         try {
-            Case.getCurrentCaseThrows().getSleuthkitCase().deleteDataSource(selectedDataSource);
+            Case.getCurrentCaseThrows().getSleuthkitCase().deleteDataSource(dataSourceId);
-        } catch (NoCurrentCaseException | TskCoreException e) {
+            deleteDataSource(dataSourceId);
-            logger.log(Level.WARNING, "Error Deleting Data source " + selectedDataSource, e);
+        } catch (NoCurrentCaseException | TskCoreException | KeywordSearchServiceException e) {
+            logger.log(Level.WARNING, "Error Deleting Data source " + dataSourceId, e);
         }
     }
-
+    private static void deleteDataSource(Long dataSourceId) throws KeywordSearchServiceException {
+        try {
+            KeywordSearchService kwsService = Lookup.getDefault().lookup(KeywordSearchService.class);
+            kwsService.deleteDataSource(dataSourceId);
+        } catch (KeywordSearchServiceException e) {
+            logger.log(Level.WARNING, "KWS Error", e);
+        }
+        
+    }
 }

Core/src/org/sleuthkit/autopsy/keywordsearchservice/KeywordSearchService.java

@@ -28,11 +28,11 @@ import org.sleuthkit.datamodel.TskCoreException;
 /**
  * An interface for implementations of a keyword search service. You can find
  * the implementations by using Lookup, such as:
- * 
+ *
  * Lookup.getDefault().lookup(KeywordSearchService.class)
- * 
+ *
  * although most clients should obtain a keyword search service by calling:
- * 
+ *
  * Case.getCurrentCase().getServices().getKeywordSearchService()
  *
  * TODO (AUT-2158): This interface should not extend Closeable.
@@ -82,7 +82,7 @@ public interface KeywordSearchService extends Closeable {
      * @throws KeywordSearchServiceException if unable to delete.
      */
     public void deleteTextIndex(CaseMetadata metadata) throws KeywordSearchServiceException;
-    
+
     /**
      * Closes the keyword search service.
      *
@@ -95,6 +95,15 @@ public interface KeywordSearchService extends Closeable {
          * No-op maintained for backwards compatibility. Clients should not
          * attempt to close case services.
          */
-    }    
+    }
+
+    /**
+     * Deletes the keyword search text for a specific data source.
+     *
+     * @param dataSourceId The data source id to be deleted.
+     *
+     * @throws KeywordSearchServiceException if unable to delete.
+     */
+    void deleteDataSource(Long dataSourceId) throws KeywordSearchServiceException;
 
 }

KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties

@@ -298,6 +298,7 @@ GlobalEditListPanel.editWordButton.text=Edit Keyword
 SolrSearchService.ServiceName=Solr Keyword Search Service
 SolrSearchService.IndexReadOnlyDialog.title=Text Index Is Read-Only
 SolrSearchService.IndexReadOnlyDialog.msg=<html>The text index for this case is read-only. <br />You will be able to see existing keyword search results and perform exact match and substring match keyword searches,<br />but you will not be able to add new text to the index or perform regex searches. You may instead open the case<br /> with your previous version of this application.</html>
+SolrSearchService.DeleteDataSource.msg=Error Deleting Solr data for data source id {0}
 ExtractedContentPanel.jLabel1.text=Text Source:
 ExtractedContentPanel.pagePreviousButton.actionCommand=pagePreviousButton
 ExtractedContentPanel.pagePreviousButton.text=

KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED

@@ -348,6 +348,7 @@ SolrSearch.openCore.msg=Opening text index
 SolrSearch.openGiantCore.msg=Opening text index. Text index for this case is very large and may take long time to load.
 SolrSearch.openLargeCore.msg=Opening text index. This may take several minutes.
 SolrSearch.readingIndexes.msg=Reading text index metadata file
+SolrSearchService.deleteDataSource.exceptionMessage.noCurrentSolrCore=DeleteDataSource did not contain a current Solr core so could not delete the Data Source
 # {0} - index folder path
 SolrSearchService.exceptionMessage.failedToDeleteIndexFiles=Failed to delete text index files at {0}
 SolrSearchService.exceptionMessage.noCurrentSolrCore=IndexMetadata did not contain a current Solr core so could not delete the case
@@ -357,6 +358,7 @@ SolrSearchService.indexingError=Unable to index blackboard artifact.
 SolrSearchService.ServiceName=Solr Keyword Search Service
 SolrSearchService.IndexReadOnlyDialog.title=Text Index Is Read-Only
 SolrSearchService.IndexReadOnlyDialog.msg=<html>The text index for this case is read-only. <br />You will be able to see existing keyword search results and perform exact match and substring match keyword searches,<br />but you will not be able to add new text to the index or perform regex searches. You may instead open the case<br /> with your previous version of this application.</html>
+SolrSearchService.DeleteDataSource.msg=Error Deleting Solr data for data source id {0}
 ExtractedContentPanel.jLabel1.text=Text Source:
 ExtractedContentPanel.pagePreviousButton.actionCommand=pagePreviousButton
 ExtractedContentPanel.pagePreviousButton.text=

KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java

@@ -189,16 +189,16 @@ public class Server {
             }
         },
         /**
-         * termfreq is a function which returns the number of times the term appears.
+         * termfreq is a function which returns the number of times the term
-         * This is not an actual field defined in schema.xml, but can be gotten from returned documents
+         * appears. This is not an actual field defined in schema.xml, but can
-         * in the same way as fields.
+         * be gotten from returned documents in the same way as fields.
          */
         TERMFREQ {
             @Override
             public String toString() {
                 return "termfreq"; //NON-NLS
             }
-        }        
+        }
     };
 
     public static final String HL_ANALYZE_CHARS_UNLIMITED = "500000"; //max 1MB in a chunk. use -1 for unlimited, but -1 option may not be supported (not documented)
@@ -525,7 +525,7 @@ public class Server {
                     @Override
                     public void run() {
                         MessageNotifyUtil.Notify.error(
-                                NbBundle.getMessage(this.getClass(), "Installer.errorInitKsmMsg"), 
+                                NbBundle.getMessage(this.getClass(), "Installer.errorInitKsmMsg"),
                                 Bundle.Server_status_failed_msg());
                     }
                 });
@@ -890,28 +890,30 @@ public class Server {
             throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.cantOpen.msg"), ex);
         }
     }
-    
+
     /**
-     * Get the host and port for a multiuser case.
+     * Get the host and port for a multiuser case. If the file solrserver.txt
-     * If the file solrserver.txt exists, then use the values from that file.
+     * exists, then use the values from that file. Otherwise use the settings
-     * Otherwise use the settings from the properties file.
+     * from the properties file.
-     * 
+     *
      * @param caseDirectory Current case directory
-     * @return IndexingServerProperties containing the solr host/port for this case
+     *
+     * @return IndexingServerProperties containing the solr host/port for this
+     *         case
      */
     public static IndexingServerProperties getMultiUserServerProperties(String caseDirectory) {
 
         Path serverFilePath = Paths.get(caseDirectory, "solrserver.txt");
-        if(serverFilePath.toFile().exists()){
+        if (serverFilePath.toFile().exists()) {
-            try{
+            try {
                 List<String> lines = Files.readAllLines(serverFilePath);
-                if(lines.isEmpty()) {
+                if (lines.isEmpty()) {
                     logger.log(Level.SEVERE, "solrserver.txt file does not contain any data");
-                } else if (! lines.get(0).contains(",")) {
+                } else if (!lines.get(0).contains(",")) {
                     logger.log(Level.SEVERE, "solrserver.txt file is corrupt - could not read host/port from " + lines.get(0));
                 } else {
                     String[] parts = lines.get(0).split(",");
-                    if(parts.length != 2) {
+                    if (parts.length != 2) {
                         logger.log(Level.SEVERE, "solrserver.txt file is corrupt - could not read host/port from " + lines.get(0));
                     } else {
                         return new IndexingServerProperties(parts[0], parts[1]);
@@ -921,102 +923,104 @@ public class Server {
                 logger.log(Level.SEVERE, "solrserver.txt file could not be read", ex);
             }
         }
-        
+
         // Default back to the user preferences if the solrserver.txt file was not found or if an error occurred
         String host = UserPreferences.getIndexingServerHost();
         String port = UserPreferences.getIndexingServerPort();
         return new IndexingServerProperties(host, port);
     }
-    
+
     /**
-     * Pick a solr server to use for this case and record it in the case directory.
+     * Pick a solr server to use for this case and record it in the case
-     * Looks for a file named "solrServerList.txt" in the root output directory - 
+     * directory. Looks for a file named "solrServerList.txt" in the root output
-     * if this does not exist then no server is recorded.
+     * directory - if this does not exist then no server is recorded.
-     * 
+     *
-     * Format of solrServerList.txt:
+     * Format of solrServerList.txt: (host),(port) Ex: 10.1.2.34,8983
-     * (host),(port)
+     *
-     * Ex: 10.1.2.34,8983
-     * 
      * @param rootOutputDirectory
      * @param caseDirectoryPath
-     * @throws KeywordSearchModuleException 
+     *
+     * @throws KeywordSearchModuleException
      */
     public static void selectSolrServerForCase(Path rootOutputDirectory, Path caseDirectoryPath) throws KeywordSearchModuleException {
         // Look for the solr server list file
         String serverListName = "solrServerList.txt";
         Path serverListPath = Paths.get(rootOutputDirectory.toString(), serverListName);
-        if(serverListPath.toFile().exists()){
+        if (serverListPath.toFile().exists()) {
-            
+
             // Read the list of solr servers
             List<String> lines;
-            try{
+            try {
                 lines = Files.readAllLines(serverListPath);
-            } catch (IOException ex){
+            } catch (IOException ex) {
                 throw new KeywordSearchModuleException(serverListName + " could not be read", ex);
             }
-            
+
             // Remove any lines that don't contain a comma (these are likely just whitespace)
             for (Iterator<String> iterator = lines.iterator(); iterator.hasNext();) {
                 String line = iterator.next();
-                if (! line.contains(",")) {
+                if (!line.contains(",")) {
                     // Remove the current element from the iterator and the list.
                     iterator.remove();
                 }
             }
-            if(lines.isEmpty()) {
+            if (lines.isEmpty()) {
                 throw new KeywordSearchModuleException(serverListName + " had no valid server information");
             }
-                
+
             // Choose which server to use
             int rnd = new Random().nextInt(lines.size());
             String[] parts = lines.get(rnd).split(",");
-            if(parts.length != 2) {
+            if (parts.length != 2) {
                 throw new KeywordSearchModuleException("Invalid server data: " + lines.get(rnd));
             }
-                
+
             // Split it up just to do a sanity check on the data
             String host = parts[0];
-            String port = parts[1];                
+            String port = parts[1];
-            if(host.isEmpty() || port.isEmpty()) {
+            if (host.isEmpty() || port.isEmpty()) {
                 throw new KeywordSearchModuleException("Invalid server data: " + lines.get(rnd));
             }
-                
+
             // Write the server data to a file
             Path serverFile = Paths.get(caseDirectoryPath.toString(), "solrserver.txt");
             try {
                 caseDirectoryPath.toFile().mkdirs();
-                if (! caseDirectoryPath.toFile().exists()) {
+                if (!caseDirectoryPath.toFile().exists()) {
                     throw new KeywordSearchModuleException("Case directory " + caseDirectoryPath.toString() + " does not exist");
                 }
                 Files.write(serverFile, lines.get(rnd).getBytes());
-            } catch (IOException ex){
+            } catch (IOException ex) {
                 throw new KeywordSearchModuleException(serverFile.toString() + " could not be written", ex);
             }
         }
     }
-    
+
     /**
      * Helper class to store the current server properties
      */
     public static class IndexingServerProperties {
+
         private final String host;
         private final String port;
-        
+
-        IndexingServerProperties (String host, String port) {
+        IndexingServerProperties(String host, String port) {
             this.host = host;
             this.port = port;
         }
 
         /**
          * Get the host
+         *
          * @return host
          */
         public String getHost() {
             return host;
         }
-        
+
         /**
          * Get the port
+         *
          * @return port
          */
         public String getPort() {
@@ -1265,6 +1269,26 @@ public class Server {
         }
     }
 
+    /**
+     * Delete a data source from SOLR.
+     *
+     * @param dataSourceId to delete
+     *
+     * @throws NoOpenCoreException
+     */
+    void deleteDataSource(Long dataSourceId) throws IOException, KeywordSearchModuleException, NoOpenCoreException, SolrServerException {
+        try {
+            currentCoreLock.writeLock().lock();
+            if (null == currentCore) {
+                throw new NoOpenCoreException();
+            }
+            currentCore.deleteDataSource(dataSourceId);
+            currentCore.commit();
+        } finally {
+            currentCoreLock.writeLock().unlock();
+        }
+    }
+
     /**
      * Get the text contents of the given file as stored in SOLR.
      *
@@ -1375,10 +1399,10 @@ public class Server {
      * @throws IOException
      */
     void connectToSolrServer(HttpSolrServer solrServer) throws SolrServerException, IOException {
-        TimingMetric metric = HealthMonitor.getTimingMetric("Solr: Connectivity check");            
+        TimingMetric metric = HealthMonitor.getTimingMetric("Solr: Connectivity check");
         CoreAdminRequest statusRequest = new CoreAdminRequest();
-        statusRequest.setCoreName( null );
+        statusRequest.setCoreName(null);
-        statusRequest.setAction( CoreAdminParams.CoreAdminAction.STATUS );
+        statusRequest.setAction(CoreAdminParams.CoreAdminAction.STATUS);
         statusRequest.setIndexInfoNeeded(false);
         statusRequest.process(solrServer);
         HealthMonitor.submitTimingMetric(metric);
@@ -1435,7 +1459,7 @@ public class Server {
         // the server to access a core needs to be built from a URL with the
         // core in it, and is only good for core-specific operations
         private final HttpSolrServer solrCore;
-        
+
         private final int QUERY_TIMEOUT_MILLISECONDS = 86400000; // 24 Hours = 86,400,000 Milliseconds
 
         private Core(String name, CaseType caseType, Index index) {
@@ -1447,7 +1471,7 @@ public class Server {
 
             //TODO test these settings
             // socket read timeout, make large enough so can index larger files
-            solrCore.setSoTimeout(QUERY_TIMEOUT_MILLISECONDS);  
+            solrCore.setSoTimeout(QUERY_TIMEOUT_MILLISECONDS);
             //solrCore.setConnectionTimeout(1000);
             solrCore.setDefaultMaxConnectionsPerHost(32);
             solrCore.setMaxTotalConnections(32);
@@ -1506,6 +1530,13 @@ public class Server {
             }
         }
 
+        private void deleteDataSource(Long dsObjId) throws IOException, SolrServerException {
+            String dataSourceId = Long.toString(dsObjId);
+            String deleteQuery = "image_id:" + dataSourceId;
+
+            solrCore.deleteByQuery(deleteQuery);
+        }
+
         void addDocument(SolrInputDocument doc) throws KeywordSearchModuleException {
             try {
                 solrCore.add(doc);
@@ -1527,7 +1558,8 @@ public class Server {
          * @param chunkID   Chunk ID of the Solr document
          *
          * @return Text from matching Solr document (as String). Null if no
-         * matching Solr document found or error while getting content from Solr
+         *         matching Solr document found or error while getting content
+         *         from Solr
          */
         private String getSolrContent(long contentID, int chunkID) {
             final SolrQuery q = new SolrQuery();

KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java

@@ -60,8 +60,7 @@ import org.sleuthkit.datamodel.TskCoreException;
  * text indexing and search.
  */
 @ServiceProviders(value = {
-    @ServiceProvider(service = KeywordSearchService.class)
+    @ServiceProvider(service = KeywordSearchService.class),
-    ,
     @ServiceProvider(service = AutopsyService.class)
 })
 public class SolrSearchService implements KeywordSearchService, AutopsyService {
@@ -194,6 +193,26 @@ public class SolrSearchService implements KeywordSearchService, AutopsyService {
         }
     }
 
+    /**
+     * Deletes a data source from Solr for a case.
+     *
+     * @param dataSourceId the id of the data source to delete.
+     *
+     * @throws
+     * org.sleuthkit.autopsy.keywordsearchservice.KeywordSearchServiceException
+     */
+    @Override
+    public void deleteDataSource(Long dataSourceId) throws KeywordSearchServiceException {
+
+        try {
+            Server ddsServer = KeywordSearch.getServer();
+            ddsServer.deleteDataSource(dataSourceId);
+        } catch (IOException | KeywordSearchModuleException | NoOpenCoreException | SolrServerException ex) {
+            logger.log(Level.WARNING, NbBundle.getMessage(SolrSearchService.class, "SolrSearchService.DeleteDataSource.msg", dataSourceId), ex);
+            throw new KeywordSearchServiceException(NbBundle.getMessage(SolrSearchService.class, "SolrSearchService.DeleteDataSource.msg", dataSourceId), ex);
+        }
+    }
+
     /**
      * Deletes Solr core for a case.
      *