mstoeck3/autopsy-flatpak
1
0
Fork 0
mirror of https://github.com/overcuriousity/autopsy-flatpak.git synced 2025-07-14 08:56:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #4819 from markmckinnon/2410-delete-data-sources-from-the-case-text-index

Browse Source 
2410 delete data sources from the case text index
This commit is contained in:
Richard Cordovano 2019-10-28 10:16:49 -04:00 committed by GitHub
commit acaa15c359
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 141 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
Core/src/org/sleuthkit/autopsy/actions/DeleteDataSourceAction.java
View File

@ -21,10 +21,13 @@ package org.sleuthkit.autopsy.actions;
import java.awt.event.ActionEvent;
import java.util.logging.Level;
import javax.swing.AbstractAction;
import org.openide.util.Lookup;
import org.openide.util.NbBundle;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.keywordsearchservice.KeywordSearchService;
import org.sleuthkit.autopsy.keywordsearchservice.KeywordSearchServiceException;
import org.sleuthkit.datamodel.TskCoreException;
/**
@ -32,21 +35,30 @@ import org.sleuthkit.datamodel.TskCoreException;
*/
public final class DeleteDataSourceAction extends AbstractAction {
private static final Logger logger = Logger.getLogger(DeleteDataSourceAction.class.getName());
private final Long selectedDataSource;
private final Long dataSourceId;
@NbBundle.Messages({"DeleteDataSourceAction.name.text=Delete Data Source"})
public DeleteDataSourceAction(Long selectedDataSource) {
public DeleteDataSourceAction(Long dataSourceId) {
super(Bundle.DeleteDataSourceAction_name_text());
this.selectedDataSource = selectedDataSource;
this.dataSourceId = dataSourceId;
}
@Override
public void actionPerformed(ActionEvent event) {
try {
Case.getCurrentCaseThrows().getSleuthkitCase().deleteDataSource(selectedDataSource);
} catch (NoCurrentCaseException | TskCoreException e) {
logger.log(Level.WARNING, "Error Deleting Data source " + selectedDataSource, e);
Case.getCurrentCaseThrows().getSleuthkitCase().deleteDataSource(dataSourceId);
deleteDataSource(dataSourceId);
} catch (NoCurrentCaseException | TskCoreException | KeywordSearchServiceException e) {
logger.log(Level.WARNING, "Error Deleting Data source " + dataSourceId, e);
}
}
private static void deleteDataSource(Long dataSourceId) throws KeywordSearchServiceException {
try {
KeywordSearchService kwsService = Lookup.getDefault().lookup(KeywordSearchService.class);
kwsService.deleteDataSource(dataSourceId);
} catch (KeywordSearchServiceException e) {
logger.log(Level.WARNING, "KWS Error", e);
}
}
}

19
Core/src/org/sleuthkit/autopsy/keywordsearchservice/KeywordSearchService.java
View File

@ -28,11 +28,11 @@ import org.sleuthkit.datamodel.TskCoreException;
/**
* An interface for implementations of a keyword search service. You can find
* the implementations by using Lookup, such as:
*
*
* Lookup.getDefault().lookup(KeywordSearchService.class)
*
*
* although most clients should obtain a keyword search service by calling:
*
*
* Case.getCurrentCase().getServices().getKeywordSearchService()
*
* TODO (AUT-2158): This interface should not extend Closeable.
@ -82,7 +82,7 @@ public interface KeywordSearchService extends Closeable {
* @throws KeywordSearchServiceException if unable to delete.
*/
public void deleteTextIndex(CaseMetadata metadata) throws KeywordSearchServiceException;
/**
* Closes the keyword search service.
*
@ -95,6 +95,15 @@ public interface KeywordSearchService extends Closeable {
* No-op maintained for backwards compatibility. Clients should not
* attempt to close case services.
*/
}
}
/**
* Deletes the keyword search text for a specific data source.
*
* @param dataSourceId The data source id to be deleted.
*
* @throws KeywordSearchServiceException if unable to delete.
*/
void deleteDataSource(Long dataSourceId) throws KeywordSearchServiceException;
}

1
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties
View File

@ -298,6 +298,7 @@ GlobalEditListPanel.editWordButton.text=Edit Keyword
SolrSearchService.ServiceName=Solr Keyword Search Service
SolrSearchService.IndexReadOnlyDialog.title=Text Index Is Read-Only
SolrSearchService.IndexReadOnlyDialog.msg=<html>The text index for this case is read-only. <br />You will be able to see existing keyword search results and perform exact match and substring match keyword searches,<br />but you will not be able to add new text to the index or perform regex searches. You may instead open the case<br /> with your previous version of this application.</html>
SolrSearchService.DeleteDataSource.msg=Error Deleting Solr data for data source id {0}
ExtractedContentPanel.jLabel1.text=Text Source:
ExtractedContentPanel.pagePreviousButton.actionCommand=pagePreviousButton
ExtractedContentPanel.pagePreviousButton.text=

2
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED
View File

@ -348,6 +348,7 @@ SolrSearch.openCore.msg=Opening text index
SolrSearch.openGiantCore.msg=Opening text index. Text index for this case is very large and may take long time to load.
SolrSearch.openLargeCore.msg=Opening text index. This may take several minutes.
SolrSearch.readingIndexes.msg=Reading text index metadata file
SolrSearchService.deleteDataSource.exceptionMessage.noCurrentSolrCore=DeleteDataSource did not contain a current Solr core so could not delete the Data Source
# {0} - index folder path
SolrSearchService.exceptionMessage.failedToDeleteIndexFiles=Failed to delete text index files at {0}
SolrSearchService.exceptionMessage.noCurrentSolrCore=IndexMetadata did not contain a current Solr core so could not delete the case
@ -357,6 +358,7 @@ SolrSearchService.indexingError=Unable to index blackboard artifact.
SolrSearchService.ServiceName=Solr Keyword Search Service
SolrSearchService.IndexReadOnlyDialog.title=Text Index Is Read-Only
SolrSearchService.IndexReadOnlyDialog.msg=<html>The text index for this case is read-only. <br />You will be able to see existing keyword search results and perform exact match and substring match keyword searches,<br />but you will not be able to add new text to the index or perform regex searches. You may instead open the case<br /> with your previous version of this application.</html>
SolrSearchService.DeleteDataSource.msg=Error Deleting Solr data for data source id {0}
ExtractedContentPanel.jLabel1.text=Text Source:
ExtractedContentPanel.pagePreviousButton.actionCommand=pagePreviousButton
ExtractedContentPanel.pagePreviousButton.text=

136
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java
View File

@ -189,16 +189,16 @@ public class Server {
}
},
/**
* termfreq is a function which returns the number of times the term appears.
* This is not an actual field defined in schema.xml, but can be gotten from returned documents
* in the same way as fields.
* termfreq is a function which returns the number of times the term
* appears. This is not an actual field defined in schema.xml, but can
* be gotten from returned documents in the same way as fields.
*/
TERMFREQ {
@Override
public String toString() {
return "termfreq"; //NON-NLS
}
}
}
};
public static final String HL_ANALYZE_CHARS_UNLIMITED = "500000"; //max 1MB in a chunk. use -1 for unlimited, but -1 option may not be supported (not documented)
@ -525,7 +525,7 @@ public class Server {
@Override
public void run() {
MessageNotifyUtil.Notify.error(
NbBundle.getMessage(this.getClass(), "Installer.errorInitKsmMsg"),
NbBundle.getMessage(this.getClass(), "Installer.errorInitKsmMsg"),
Bundle.Server_status_failed_msg());
}
});
@ -890,28 +890,30 @@ public class Server {
throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.cantOpen.msg"), ex);
}
}
/**
* Get the host and port for a multiuser case.
* If the file solrserver.txt exists, then use the values from that file.
* Otherwise use the settings from the properties file.
*
* Get the host and port for a multiuser case. If the file solrserver.txt
* exists, then use the values from that file. Otherwise use the settings
* from the properties file.
*
* @param caseDirectory Current case directory
* @return IndexingServerProperties containing the solr host/port for this case
*
* @return IndexingServerProperties containing the solr host/port for this
* case
*/
public static IndexingServerProperties getMultiUserServerProperties(String caseDirectory) {
Path serverFilePath = Paths.get(caseDirectory, "solrserver.txt");
if(serverFilePath.toFile().exists()){
try{
if (serverFilePath.toFile().exists()) {
try {
List<String> lines = Files.readAllLines(serverFilePath);
if(lines.isEmpty()) {
if (lines.isEmpty()) {
logger.log(Level.SEVERE, "solrserver.txt file does not contain any data");
} else if (! lines.get(0).contains(",")) {
} else if (!lines.get(0).contains(",")) {
logger.log(Level.SEVERE, "solrserver.txt file is corrupt - could not read host/port from " + lines.get(0));
} else {
String[] parts = lines.get(0).split(",");
if(parts.length != 2) {
if (parts.length != 2) {
logger.log(Level.SEVERE, "solrserver.txt file is corrupt - could not read host/port from " + lines.get(0));
} else {
return new IndexingServerProperties(parts[0], parts[1]);
@ -921,102 +923,104 @@ public class Server {
logger.log(Level.SEVERE, "solrserver.txt file could not be read", ex);
}
}
// Default back to the user preferences if the solrserver.txt file was not found or if an error occurred
String host = UserPreferences.getIndexingServerHost();
String port = UserPreferences.getIndexingServerPort();
return new IndexingServerProperties(host, port);
}
/**
* Pick a solr server to use for this case and record it in the case directory.
* Looks for a file named "solrServerList.txt" in the root output directory -
* if this does not exist then no server is recorded.
*
* Format of solrServerList.txt:
* (host),(port)
* Ex: 10.1.2.34,8983
*
* Pick a solr server to use for this case and record it in the case
* directory. Looks for a file named "solrServerList.txt" in the root output
* directory - if this does not exist then no server is recorded.
*
* Format of solrServerList.txt: (host),(port) Ex: 10.1.2.34,8983
*
* @param rootOutputDirectory
* @param caseDirectoryPath
* @throws KeywordSearchModuleException
*
* @throws KeywordSearchModuleException
*/
public static void selectSolrServerForCase(Path rootOutputDirectory, Path caseDirectoryPath) throws KeywordSearchModuleException {
// Look for the solr server list file
String serverListName = "solrServerList.txt";
Path serverListPath = Paths.get(rootOutputDirectory.toString(), serverListName);
if(serverListPath.toFile().exists()){
if (serverListPath.toFile().exists()) {
// Read the list of solr servers
List<String> lines;
try{
try {
lines = Files.readAllLines(serverListPath);
} catch (IOException ex){
} catch (IOException ex) {
throw new KeywordSearchModuleException(serverListName + " could not be read", ex);
}
// Remove any lines that don't contain a comma (these are likely just whitespace)
for (Iterator<String> iterator = lines.iterator(); iterator.hasNext();) {
String line = iterator.next();
if (! line.contains(",")) {
if (!line.contains(",")) {
// Remove the current element from the iterator and the list.
iterator.remove();
}
}
if(lines.isEmpty()) {
if (lines.isEmpty()) {
throw new KeywordSearchModuleException(serverListName + " had no valid server information");
}
// Choose which server to use
int rnd = new Random().nextInt(lines.size());
String[] parts = lines.get(rnd).split(",");
if(parts.length != 2) {
if (parts.length != 2) {
throw new KeywordSearchModuleException("Invalid server data: " + lines.get(rnd));
}
// Split it up just to do a sanity check on the data
String host = parts[0];
String port = parts[1];
if(host.isEmpty() || port.isEmpty()) {
String port = parts[1];
if (host.isEmpty() || port.isEmpty()) {
throw new KeywordSearchModuleException("Invalid server data: " + lines.get(rnd));
}
// Write the server data to a file
Path serverFile = Paths.get(caseDirectoryPath.toString(), "solrserver.txt");
try {
caseDirectoryPath.toFile().mkdirs();
if (! caseDirectoryPath.toFile().exists()) {
if (!caseDirectoryPath.toFile().exists()) {
throw new KeywordSearchModuleException("Case directory " + caseDirectoryPath.toString() + " does not exist");
}
Files.write(serverFile, lines.get(rnd).getBytes());
} catch (IOException ex){
} catch (IOException ex) {
throw new KeywordSearchModuleException(serverFile.toString() + " could not be written", ex);
}
}
}
/**
* Helper class to store the current server properties
*/
public static class IndexingServerProperties {
private final String host;
private final String port;
IndexingServerProperties (String host, String port) {
IndexingServerProperties(String host, String port) {
this.host = host;
this.port = port;
}
/**
* Get the host
*
* @return host
*/
public String getHost() {
return host;
}
/**
* Get the port
*
* @return port
*/
public String getPort() {
@ -1265,6 +1269,26 @@ public class Server {
}
}
/**
* Delete a data source from SOLR.
*
* @param dataSourceId to delete
*
* @throws NoOpenCoreException
*/
void deleteDataSource(Long dataSourceId) throws IOException, KeywordSearchModuleException, NoOpenCoreException, SolrServerException {
try {
currentCoreLock.writeLock().lock();
if (null == currentCore) {
throw new NoOpenCoreException();
}
currentCore.deleteDataSource(dataSourceId);
currentCore.commit();
} finally {
currentCoreLock.writeLock().unlock();
}
}
/**
* Get the text contents of the given file as stored in SOLR.
*
@ -1375,10 +1399,10 @@ public class Server {
* @throws IOException
*/
void connectToSolrServer(HttpSolrServer solrServer) throws SolrServerException, IOException {
TimingMetric metric = HealthMonitor.getTimingMetric("Solr: Connectivity check");
TimingMetric metric = HealthMonitor.getTimingMetric("Solr: Connectivity check");
CoreAdminRequest statusRequest = new CoreAdminRequest();
statusRequest.setCoreName( null );
statusRequest.setAction( CoreAdminParams.CoreAdminAction.STATUS );
statusRequest.setCoreName(null);
statusRequest.setAction(CoreAdminParams.CoreAdminAction.STATUS);
statusRequest.setIndexInfoNeeded(false);
statusRequest.process(solrServer);
HealthMonitor.submitTimingMetric(metric);
@ -1435,7 +1459,7 @@ public class Server {
// the server to access a core needs to be built from a URL with the
// core in it, and is only good for core-specific operations
private final HttpSolrServer solrCore;
private final int QUERY_TIMEOUT_MILLISECONDS = 86400000; // 24 Hours = 86,400,000 Milliseconds
private Core(String name, CaseType caseType, Index index) {
@ -1447,7 +1471,7 @@ public class Server {
//TODO test these settings
// socket read timeout, make large enough so can index larger files
solrCore.setSoTimeout(QUERY_TIMEOUT_MILLISECONDS);
solrCore.setSoTimeout(QUERY_TIMEOUT_MILLISECONDS);
//solrCore.setConnectionTimeout(1000);
solrCore.setDefaultMaxConnectionsPerHost(32);
solrCore.setMaxTotalConnections(32);
@ -1506,6 +1530,13 @@ public class Server {
}
}
private void deleteDataSource(Long dsObjId) throws IOException, SolrServerException {
String dataSourceId = Long.toString(dsObjId);
String deleteQuery = "image_id:" + dataSourceId;
solrCore.deleteByQuery(deleteQuery);
}
void addDocument(SolrInputDocument doc) throws KeywordSearchModuleException {
try {
solrCore.add(doc);
@ -1527,7 +1558,8 @@ public class Server {
* @param chunkID Chunk ID of the Solr document
*
* @return Text from matching Solr document (as String). Null if no
* matching Solr document found or error while getting content from Solr
* matching Solr document found or error while getting content
* from Solr
*/
private String getSolrContent(long contentID, int chunkID) {
final SolrQuery q = new SolrQuery();

23
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java
View File

@ -60,8 +60,7 @@ import org.sleuthkit.datamodel.TskCoreException;
* text indexing and search.
*/
@ServiceProviders(value = {
@ServiceProvider(service = KeywordSearchService.class)
,
@ServiceProvider(service = KeywordSearchService.class),
@ServiceProvider(service = AutopsyService.class)
})
public class SolrSearchService implements KeywordSearchService, AutopsyService {
@ -194,6 +193,26 @@ public class SolrSearchService implements KeywordSearchService, AutopsyService {
}
}
/**
* Deletes a data source from Solr for a case.
*
* @param dataSourceId the id of the data source to delete.
*
* @throws
* org.sleuthkit.autopsy.keywordsearchservice.KeywordSearchServiceException
*/
@Override
public void deleteDataSource(Long dataSourceId) throws KeywordSearchServiceException {
try {
Server ddsServer = KeywordSearch.getServer();
ddsServer.deleteDataSource(dataSourceId);
} catch (IOException | KeywordSearchModuleException | NoOpenCoreException | SolrServerException ex) {
logger.log(Level.WARNING, NbBundle.getMessage(SolrSearchService.class, "SolrSearchService.DeleteDataSource.msg", dataSourceId), ex);
throw new KeywordSearchServiceException(NbBundle.getMessage(SolrSearchService.class, "SolrSearchService.DeleteDataSource.msg", dataSourceId), ex);
}
}
/**
* Deletes Solr core for a case.
*