mstoeck3/autopsy-flatpak
1
0
Fork 0
mirror of https://github.com/overcuriousity/autopsy-flatpak.git synced 2025-07-17 10:17:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/search_improvements' into open_case_resources

Browse Source
This commit is contained in:
Richard Cordovano 2017-01-19 09:08:04 -05:00
commit a95ce4db51
16 changed files with 406 additions and 119 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
KeywordSearch/release/Solr4to5IndexUpgrade/Solr4IndexUpgrade.jar Normal file
View File

Binary file not shown.

BIN
KeywordSearch/release/Solr4to5IndexUpgrade/lib/lucene-analyzers-common-5.5.1.jar Normal file
View File

Binary file not shown.

BIN
KeywordSearch/release/Solr4to5IndexUpgrade/lib/lucene-backward-codecs-5.5.1.jar Normal file
View File

Binary file not shown.

BIN
KeywordSearch/release/Solr4to5IndexUpgrade/lib/lucene-codecs-5.5.1.jar Normal file
View File

Binary file not shown.

BIN
KeywordSearch/release/Solr4to5IndexUpgrade/lib/lucene-core-5.5.1.jar Normal file
View File

Binary file not shown.

BIN
KeywordSearch/release/Solr5to6IndexUpgrade/Solr5IndexUpgrade.jar Normal file
View File

Binary file not shown.

BIN
KeywordSearch/release/Solr5to6IndexUpgrade/lib/lucene-analyzers-common-6.2.1.jar Normal file
View File

Binary file not shown.

BIN
KeywordSearch/release/Solr5to6IndexUpgrade/lib/lucene-backward-codecs-6.2.1.jar Normal file
View File

Binary file not shown.

BIN
KeywordSearch/release/Solr5to6IndexUpgrade/lib/lucene-codecs-6.2.1.jar Normal file
View File

Binary file not shown.

BIN
KeywordSearch/release/Solr5to6IndexUpgrade/lib/lucene-core-6.2.1.jar Normal file
View File

Binary file not shown.

2
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties
View File

@ -313,3 +313,5 @@ GlobalEditListPanel.keywordDupesSkipped.text={0} keyword was already in the list
GlobalEditListPanel.keywordDupesSkippedPlural.text={0} keywords were already in the list. GlobalEditListPanel.keywordDupesSkippedPlural.text={0} keywords were already in the list.
GlobalEditListPanel.keywordErrors.text={0} keyword could not be parsed. Please review and try again. GlobalEditListPanel.keywordErrors.text={0} keyword could not be parsed. Please review and try again.
GlobalEditListPanel.keywordErrorsPlural.text={0} keywords could not be parsed. Please review and try again. GlobalEditListPanel.keywordErrorsPlural.text={0} keywords could not be parsed. Please review and try again.
SolrSearchService.IndexUpgradeDialog.title=Index Upgrade Required In Order To Open Case
SolrSearchService.IndexUpgradeDialog.msg=<html>Index upgrade can be a very lengthy operation that involves copying existing index and calling third party tools to upgrade it. <br />Upon upgrade you will be able to see existing keyword search results and perform literal keyword searches on the existing index.<br />However, you will not be able to add new text to the index or performing regex searches.<br /> You must create a new case and re-run Keyword Search Ingest Module if you want to index new text or performing regex searches.<br /> Do you wish to proceed with the index upgrade?</html>

157
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexHandling.java → KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexFinder.java
View File

@ -24,27 +24,38 @@ import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.Collections; import java.util.Collections;
import java.util.List; import java.util.List;
import java.util.logging.Level;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import org.apache.commons.io.FileUtils;
import org.openide.util.NbBundle;
import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.corecomponentinterfaces.AutopsyService;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.coreutils.UNCPathUtilities; import org.sleuthkit.autopsy.coreutils.UNCPathUtilities;
import org.sleuthkit.autopsy.coreutils.PlatformUtil;
/** /**
* This class handles the task of finding KWS index folders and upgrading old * This class handles the task of finding and identifying KWS index folders.
* indexes to the latest supported Solr version.
*/ */
class IndexHandling { class IndexFinder {
private UNCPathUtilities uncPathUtilities = new UNCPathUtilities(); private static final Logger logger = Logger.getLogger(IndexFinder.class.getName());
private static final String MODULE_OUTPUT = "ModuleOutput"; // ELTODO get "ModuleOutput" somehow... private UNCPathUtilities uncPathUtilities;
private static final String KWS_OUTPUT_FOLDER_NAME = "keywordsearch"; private static final String KWS_OUTPUT_FOLDER_NAME = "keywordsearch";
private static final String KWS_DATA_FOLDER_NAME = "data"; private static final String KWS_DATA_FOLDER_NAME = "data";
private static final String INDEX_FOLDER_NAME = "index"; private static final String INDEX_FOLDER_NAME = "index";
private static final String CURRENT_SOLR_VERSION = "6"; private static final String CURRENT_SOLR_VERSION = "6";
private static final String CURRENT_SOLR_SCHEMA_VERSION = "2.0"; private static final String CURRENT_SOLR_SCHEMA_VERSION = "2.0";
private static final Pattern INDEX_FOLDER_NAME_PATTERN = Pattern.compile("^solr\\d{1,2}_schema_\\d{1,2}.\\d{1,2}$"); private static final Pattern INDEX_FOLDER_NAME_PATTERN = Pattern.compile("^solr\\d{1,2}_schema_\\d{1,2}.\\d{1,2}$");
// If SOLR_HOME environment variable doesn't exist, try these relative paths to find Solr config sets:
private static final String RELATIVE_PATH_TO_CONFIG_SET = "autopsy/solr/solr/configsets/";
private static final String RELATIVE_PATH_TO_CONFIG_SET_2 = "release/solr/solr/configsets/";
IndexFinder() {
uncPathUtilities = new UNCPathUtilities();
}
static String getCurrentSolrVersion() { static String getCurrentSolrVersion() {
return CURRENT_SOLR_VERSION; return CURRENT_SOLR_VERSION;
} }
@ -52,7 +63,7 @@ class IndexHandling {
static String getCurrentSchemaVersion() { static String getCurrentSchemaVersion() {
return CURRENT_SOLR_SCHEMA_VERSION; return CURRENT_SOLR_SCHEMA_VERSION;
} }
static String findLatestVersionIndexDir(List<String> allIndexes) { static String findLatestVersionIndexDir(List<String> allIndexes) {
String indexFolderName = "solr" + CURRENT_SOLR_VERSION + "_schema_" + CURRENT_SOLR_SCHEMA_VERSION; String indexFolderName = "solr" + CURRENT_SOLR_VERSION + "_schema_" + CURRENT_SOLR_SCHEMA_VERSION;
for (String path : allIndexes) { for (String path : allIndexes) {
@ -63,6 +74,82 @@ class IndexHandling {
return ""; return "";
} }
String copyIndexAndConfigSet(Case theCase, String oldIndexDir) throws AutopsyService.AutopsyServiceException {
// Copy the "old" index into ModuleOutput/keywordsearch/data/solrX_schema_Y/index
String newIndexDir = createReferenceIndexCopy(theCase, oldIndexDir);
// Make a “reference copy” of the configset and place it in ModuleOutput/keywordsearch/data/solrX_schema_Y/configset
createReferenceConfigSetCopy(new File(newIndexDir).getParent());
return newIndexDir;
}
private String createReferenceIndexCopy(Case theCase, String indexPath) throws AutopsyService.AutopsyServiceException {
logger.log(Level.INFO, "Creating a reference copy of KWS index in {0} ", indexPath); //NON-NLS
String indexFolderName = "solr" + CURRENT_SOLR_VERSION + "_schema_" + CURRENT_SOLR_SCHEMA_VERSION;
try {
// new index should be stored in "\ModuleOutput\keywordsearch\data\solrX_schema_Y\index"
File targetDirPath = Paths.get(theCase.getModuleDirectory(), KWS_OUTPUT_FOLDER_NAME, KWS_DATA_FOLDER_NAME, indexFolderName, INDEX_FOLDER_NAME).toFile(); //NON-NLS
if (targetDirPath.exists()) {
// targetDirPath should not exist, at least the target directory should be empty
List<File> contents = getAllContentsInFolder(targetDirPath.getAbsolutePath());
if (!contents.isEmpty()) {
// target directory is not empty
logger.log(Level.SEVERE, "Creating a reference copy of KWS index in {0} ", indexPath); //NON-NLS
throw new AutopsyService.AutopsyServiceException("Directory to store the upgraded index must be empty " + targetDirPath.getAbsolutePath());
}
}
targetDirPath.mkdirs();
FileUtils.copyDirectory(new File(indexPath), targetDirPath);
return targetDirPath.getAbsolutePath();
} catch (Exception ex) {
logger.log(Level.SEVERE, "Error occurred while creating a reference copy of keyword search index {0}", ex); //NON-NLS
throw new AutopsyService.AutopsyServiceException("Error occurred while creating a copy of keyword search index", ex);
}
}
// ELTODO This functionality is NTH:
private void createReferenceConfigSetCopy(String indexPath) {
logger.log(Level.INFO, "Creating a reference copy of config set in {0} ", indexPath); //NON-NLS
File pathToConfigSet = new File("");
try {
// See if there is SOLR_HOME environment variable first
String solrHome = System.getenv("SOLR_HOME");
if (solrHome != null && !solrHome.isEmpty()) {
// ELTODO pathToConfigSet =
return; // ELTODO remove
} else {
// if there is no SOLR_HOME:
// this will only work for Windows OS
if (!PlatformUtil.isWindowsOS()) {
throw new AutopsyService.AutopsyServiceException("ELTODO");
}
// config set should be located in "C:/some/directory/AutopsyXYZ/autopsy/solr/solr/configsets/"
pathToConfigSet = Paths.get(System.getProperty("user.dir"), RELATIVE_PATH_TO_CONFIG_SET).toFile();
if (!pathToConfigSet.exists() || !pathToConfigSet.isDirectory()) {
// try the "release/solr/solr/configsets/" folder instead
pathToConfigSet = Paths.get(System.getProperty("user.dir"), RELATIVE_PATH_TO_CONFIG_SET_2).toFile();
if (!pathToConfigSet.exists() || !pathToConfigSet.isDirectory()) {
logger.log(Level.WARNING, "Unable to locate KWS config set in order to create a reference copy"); //NON-NLS
return;
// ELTODO This is NTH: throw new AutopsyService.AutopsyServiceException("ELTODO");
}
}
}
File targetDirPath = new File(indexPath); //NON-NLS
if (!targetDirPath.exists()) {
targetDirPath.mkdirs();
}
// copy config set
if (!pathToConfigSet.getAbsolutePath().isEmpty() && pathToConfigSet.exists()) {
FileUtils.copyDirectory(pathToConfigSet, new File(indexPath));
}
} catch (Exception ex) {
// This feature is a NTH so don't re-throw
logger.log(Level.WARNING, "Error while copying KWS config set to {0}", indexPath); //NON-NLS
}
}
/** /**
* Find index directory location for the case. This is done via subdirectory * Find index directory location for the case. This is done via subdirectory
* search of all existing "ModuleOutput/node_name/keywordsearch/data/" * search of all existing "ModuleOutput/node_name/keywordsearch/data/"
@ -70,9 +157,9 @@ class IndexHandling {
* *
* @param theCase the case to get index dir for * @param theCase the case to get index dir for
* *
* @return absolute path to index dir * @return List of absolute paths to all found index directories
*/ */
static List<String> findAllIndexDirs(Case theCase) { List<String> findAllIndexDirs(Case theCase) {
ArrayList<String> candidateIndexDirs = new ArrayList<>(); ArrayList<String> candidateIndexDirs = new ArrayList<>();
// first find all existing "/ModuleOutput/keywordsearch/data/" folders // first find all existing "/ModuleOutput/keywordsearch/data/" folders
if (theCase.getCaseType() == Case.CaseType.MULTI_USER_CASE) { if (theCase.getCaseType() == Case.CaseType.MULTI_USER_CASE) {
@ -87,15 +174,19 @@ class IndexHandling {
// create a list of all sub-directories // create a list of all sub-directories
List<File> contents = getAllContentsInFolder(theCase.getCaseDirectory()); List<File> contents = getAllContentsInFolder(theCase.getCaseDirectory());
// ELTODO decipher "ModuleOutput" from path
// scan all topLevelOutputDir subfolders for presence of non-empty "/ModuleOutput/keywordsearch/data/" folder if (!contents.isEmpty()) {
for (File item : contents) { // decipher "ModuleOutput" directory name from module output path
File path = Paths.get(item.getAbsolutePath(), MODULE_OUTPUT, KWS_OUTPUT_FOLDER_NAME, KWS_DATA_FOLDER_NAME).toFile(); //NON-NLS // (e.g. X:\Case\ingest4\ModuleOutput\) because there is no other way to get it...
// must be a non-empty directory String moduleOutDirName = new File(theCase.getModuleDirectory()).getName();
if (path.exists() && path.isDirectory()) {
candidateIndexDirs.add(path.toString()); // scan all topLevelOutputDir subfolders for presence of non-empty "/ModuleOutput/keywordsearch/data/" folder
for (File item : contents) {
File path = Paths.get(item.getAbsolutePath(), moduleOutDirName, KWS_OUTPUT_FOLDER_NAME, KWS_DATA_FOLDER_NAME).toFile(); //NON-NLS
// must be a non-empty directory
if (path.exists() && path.isDirectory()) {
candidateIndexDirs.add(path.toString());
}
} }
} }
} else { } else {
@ -112,20 +203,19 @@ class IndexHandling {
candidateIndexDirs.add(path.toString()); candidateIndexDirs.add(path.toString());
} }
} }
// analyze possible index folders // analyze possible index folders
ArrayList<String> indexDirs = new ArrayList<>(); ArrayList<String> indexDirs = new ArrayList<>();
for (String path : candidateIndexDirs) { for (String path : candidateIndexDirs) {
List<String> validIndexPaths = containsValidIndexFolders(path); List<String> validIndexPaths = containsValidIndexFolders(path);
for (String validPath : validIndexPaths) { for (String validPath : validIndexPaths) {
indexDirs.add(validPath); indexDirs.add(convertPathToUNC(validPath));
// ELTODO indexDirs.add(convertPathToUNC(validPath));
// there can be multiple index folders (e.g. current version and "old" version) so keep looking // there can be multiple index folders (e.g. current version and "old" version) so keep looking
} }
} }
return indexDirs; return indexDirs;
} }
String convertPathToUNC(String indexDir) { String convertPathToUNC(String indexDir) {
// ELTODO do we need to do this when searching for old index? // ELTODO do we need to do this when searching for old index?
if (uncPathUtilities == null) { if (uncPathUtilities == null) {
@ -146,7 +236,7 @@ class IndexHandling {
/** /**
* Returns a list of all contents in the folder of interest. * Returns a list of all contents in the folder of interest.
* *
* @param path Absolute path of the folder of interest * @param path Absolute targetDirPath of the folder of interest
* *
* @return List of all contents in the folder of interest * @return List of all contents in the folder of interest
*/ */
@ -156,12 +246,10 @@ class IndexHandling {
if (contents == null) { if (contents == null) {
// the directory file is not really a directory.. // the directory file is not really a directory..
return Collections.emptyList(); return Collections.emptyList();
} } else if (contents.length == 0) {
else if (contents.length == 0) {
// Folder is empty // Folder is empty
return Collections.emptyList(); return Collections.emptyList();
} } else {
else {
// Folder has contents // Folder has contents
return new ArrayList<>(Arrays.asList(contents)); return new ArrayList<>(Arrays.asList(contents));
} }
@ -189,7 +277,7 @@ class IndexHandling {
// keep looking as there may be more index folders // keep looking as there may be more index folders
continue; continue;
} }
// check if the folder matches "solrX_schema_Y" patern // check if the folder matches "solrX_schema_Y" patern
if (matchesIndexFolderNameStandard(item.getName())) { if (matchesIndexFolderNameStandard(item.getName())) {
File nextLevelIndexFolder = Paths.get(item.getAbsolutePath(), INDEX_FOLDER_NAME).toFile(); File nextLevelIndexFolder = Paths.get(item.getAbsolutePath(), INDEX_FOLDER_NAME).toFile();
@ -202,24 +290,23 @@ class IndexHandling {
} }
return indexFolders; return indexFolders;
} }
private static boolean isNonEmptyIndexFolder(File path) { private static boolean isNonEmptyIndexFolder(File path) {
if (path.exists() && path.isDirectory() && path.getName().equals(INDEX_FOLDER_NAME) && path.listFiles().length > 0) { if (path.exists() && path.isDirectory() && path.getName().equals(INDEX_FOLDER_NAME) && path.listFiles().length > 0) {
return true; return true;
} }
return false; return false;
} }
/** /**
* Checks whether a name matches index folder name standard * Checks whether a name matches index folder name standard
* *
* @param inputString The string to check. * @param inputString The string to check.
* *
* @return True or false. * @return True or false.
*/ */
public static boolean matchesIndexFolderNameStandard(String inputString) { private static boolean matchesIndexFolderNameStandard(String inputString) {
Matcher m = INDEX_FOLDER_NAME_PATTERN.matcher(inputString); Matcher m = INDEX_FOLDER_NAME_PATTERN.matcher(inputString);
return m.find(); return m.find();
} }
} }

173
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IndexUpgrader.java Normal file
View File

@ -0,0 +1,173 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2011-2016 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.keywordsearch;
import java.io.File;
import java.io.IOException;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.logging.Level;
import org.openide.modules.InstalledFileLocator;
import org.sleuthkit.autopsy.corecomponentinterfaces.AutopsyService;
import org.sleuthkit.autopsy.coreutils.ExecUtil;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.coreutils.PlatformUtil;
/**
* This class handles the task of upgrading old indexes to the latest supported
* Solr version.
*/
class IndexUpgrader {
private static final Logger logger = Logger.getLogger(IndexFinder.class.getName());
private final String JAVA_PATH;
IndexUpgrader() {
JAVA_PATH = PlatformUtil.getJavaPath();
}
void performIndexUpgrade(String newIndexDir, String tempResultsDir) throws AutopsyService.AutopsyServiceException {
// ELTODO Check for cancellation at whatever points are feasible
// Run the upgrade tools on the contents (core) in ModuleOutput/keywordsearch/data/solrX_schema_Y/index
File tmpDir = Paths.get(tempResultsDir, "IndexUpgrade").toFile(); //NON-NLS
tmpDir.mkdirs();
boolean success = true;
try {
// upgrade from Solr 4 to 5. If index is newer than Solr 4 then the upgrade script will throw exception right away.
upgradeSolrIndexVersion4to5(newIndexDir, tempResultsDir);
} catch (Exception ex) {
// catch-all firewall for exceptions thrown by the Solr 4 to 5 upgrade tool itself
logger.log(Level.SEVERE, "Exception while running Sorl 4 to Solr 5 upgrade tool " + newIndexDir, ex); //NON-NLS
success = false;
}
if (success) {
try {
// upgrade from Solr 5 to 6. This one must complete successfully in order to produce a valid Solr 6 index.
upgradeSolrIndexVersion5to6(newIndexDir, tempResultsDir);
} catch (Exception ex) {
// catch-all firewall for exceptions thrown by Solr 5 to 6 upgrade tool itself
logger.log(Level.SEVERE, "Exception while running Sorl 5 to Solr 6 upgrade tool " + newIndexDir, ex); //NON-NLS
success = false;
}
}
if (!success) {
// delete the new directories
new File(newIndexDir).delete();
throw new AutopsyService.AutopsyServiceException("Failed to upgrade existing keyword search index");
}
}
/**
* Upgrades Solr index from version 4 to 5.
*
* @param solr4IndexPath Full path to Solr v4 index directory
* @param tempResultsDir Path to directory where to store log output
*
* @return True is index upgraded successfully, false otherwise
*/
private void upgradeSolrIndexVersion4to5(String solr4IndexPath, String tempResultsDir) throws AutopsyService.AutopsyServiceException, SecurityException, IOException {
String outputFileName = "output.txt";
logger.log(Level.INFO, "Upgrading KWS index {0} from Sorl 4 to Solr 5", solr4IndexPath); //NON-NLS
// find the index upgrade tool
final File upgradeToolFolder = InstalledFileLocator.getDefault().locate("Solr4to5IndexUpgrade", IndexFinder.class.getPackage().getName(), false); //NON-NLS
if (upgradeToolFolder == null) {
logger.log(Level.SEVERE, "Unable to locate Sorl 4 to Solr 5 upgrade tool"); //NON-NLS
throw new AutopsyService.AutopsyServiceException("Unable to locate Sorl 4 to Solr 5 upgrade tool");
}
// full path to index upgrade jar file
File upgradeJarPath = Paths.get(upgradeToolFolder.getAbsolutePath(), "Solr4IndexUpgrade.jar").toFile();
if (!upgradeJarPath.exists() || !upgradeJarPath.isFile()) {
logger.log(Level.SEVERE, "Unable to locate Sorl 4 to Solr 5 upgrade tool's JAR file at {0}", upgradeJarPath); //NON-NLS
throw new AutopsyService.AutopsyServiceException("Unable to locate Sorl 4 to Solr 5 upgrade tool's JAR file");
}
// create log output directory if it doesn't exist
new File(tempResultsDir).mkdirs();
final String outputFileFullPath = Paths.get(tempResultsDir, outputFileName).toString();
final String errFileFullPath = Paths.get(tempResultsDir, outputFileName + ".err").toString(); //NON-NLS
List<String> commandLine = new ArrayList<>();
commandLine.add(JAVA_PATH);
commandLine.add("-jar");
commandLine.add(upgradeJarPath.getAbsolutePath());
commandLine.add(solr4IndexPath);
ProcessBuilder processBuilder = new ProcessBuilder(commandLine);
processBuilder.redirectOutput(new File(outputFileFullPath));
processBuilder.redirectError(new File(errFileFullPath));
ExecUtil.execute(processBuilder);
// alternatively can execute lucene upgrade command from the folder where lucene jars are located
// java -cp ".;lucene-core-5.5.1.jar;lucene-backward-codecs-5.5.1.jar;lucene-codecs-5.5.1.jar;lucene-analyzers-common-5.5.1.jar" org.apache.lucene.index.IndexUpgrader \path\to\index
}
/**
* Upgrades Solr index from version 5 to 6.
*
* @param solr5IndexPath Full path to Solr v5 index directory
* @param tempResultsDir Path to directory where to store log output
*
* @return True is index upgraded successfully, false otherwise
*/
private void upgradeSolrIndexVersion5to6(String solr5IndexPath, String tempResultsDir) throws AutopsyService.AutopsyServiceException, SecurityException, IOException {
String outputFileName = "output.txt";
logger.log(Level.INFO, "Upgrading KWS index {0} from Sorl 5 to Solr 6", solr5IndexPath); //NON-NLS
// find the index upgrade tool
final File upgradeToolFolder = InstalledFileLocator.getDefault().locate("Solr5to6IndexUpgrade", IndexFinder.class.getPackage().getName(), false); //NON-NLS
if (upgradeToolFolder == null) {
logger.log(Level.SEVERE, "Unable to locate Sorl 5 to Solr 6 upgrade tool"); //NON-NLS
throw new AutopsyService.AutopsyServiceException("Unable to locate Sorl 5 to Solr 6 upgrade tool");
}
// full path to index upgrade jar file
File upgradeJarPath = Paths.get(upgradeToolFolder.getAbsolutePath(), "Solr5IndexUpgrade.jar").toFile();
if (!upgradeJarPath.exists() || !upgradeJarPath.isFile()) {
logger.log(Level.SEVERE, "Unable to locate Sorl 5 to Solr 6 upgrade tool's JAR file at {0}", upgradeJarPath); //NON-NLS
throw new AutopsyService.AutopsyServiceException("Unable to locate Sorl 5 to Solr 6 upgrade tool's JAR file");
}
// create log output directory if it doesn't exist
new File(tempResultsDir).mkdirs();
final String outputFileFullPath = Paths.get(tempResultsDir, outputFileName).toString();
final String errFileFullPath = Paths.get(tempResultsDir, outputFileName + ".err").toString(); //NON-NLS
List<String> commandLine = new ArrayList<>();
commandLine.add(JAVA_PATH);
commandLine.add("-jar");
commandLine.add(upgradeJarPath.getAbsolutePath());
commandLine.add(solr5IndexPath);
ProcessBuilder processBuilder = new ProcessBuilder(commandLine);
processBuilder.redirectOutput(new File(outputFileFullPath));
processBuilder.redirectError(new File(errFileFullPath));
ExecUtil.execute(processBuilder);
// alternatively can execute lucene upgrade command from the folder where lucene jars are located
// java -cp ".;lucene-core-6.2.1.jar;lucene-backward-codecs-6.2.1.jar;lucene-codecs-6.2.1.jar;lucene-analyzers-common-6.2.1.jar" org.apache.lucene.index.IndexUpgrader \path\to\index
}
}

114
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java
View File

@ -21,9 +21,11 @@ package org.sleuthkit.autopsy.keywordsearch;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collection; import java.util.Collection;
import java.util.HashMap; import java.util.HashMap;
import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Map.Entry; import java.util.Map.Entry;
import java.util.Set;
import java.util.Timer; import java.util.Timer;
import java.util.TimerTask; import java.util.TimerTask;
import java.util.concurrent.CancellationException; import java.util.concurrent.CancellationException;
@ -262,12 +264,14 @@ public final class SearchRunner {
// mutable state: // mutable state:
private volatile boolean workerRunning; private volatile boolean workerRunning;
private List<String> keywordListNames; //guarded by SearchJobInfo.this private List<String> keywordListNames; //guarded by SearchJobInfo.this
private Map<Keyword, List<Long>> currentResults; //guarded by SearchJobInfo.this
// Map of keyword to the object ids that contain a hit
private Map<Keyword, Set<Long>> currentResults; //guarded by SearchJobInfo.this
private SearchRunner.Searcher currentSearcher; private SearchRunner.Searcher currentSearcher;
private AtomicLong moduleReferenceCount = new AtomicLong(0); private AtomicLong moduleReferenceCount = new AtomicLong(0);
private final Object finalSearchLock = new Object(); //used for a condition wait private final Object finalSearchLock = new Object(); //used for a condition wait
public SearchJobInfo(long jobId, long dataSourceId, List<String> keywordListNames) { private SearchJobInfo(long jobId, long dataSourceId, List<String> keywordListNames) {
this.jobId = jobId; this.jobId = jobId;
this.dataSourceId = dataSourceId; this.dataSourceId = dataSourceId;
this.keywordListNames = new ArrayList<>(keywordListNames); this.keywordListNames = new ArrayList<>(keywordListNames);
@ -276,53 +280,53 @@ public final class SearchRunner {
currentSearcher = null; currentSearcher = null;
} }
public long getJobId() { private long getJobId() {
return jobId; return jobId;
} }
public long getDataSourceId() { private long getDataSourceId() {
return dataSourceId; return dataSourceId;
} }
public synchronized List<String> getKeywordListNames() { private synchronized List<String> getKeywordListNames() {
return new ArrayList<>(keywordListNames); return new ArrayList<>(keywordListNames);
} }
public synchronized void addKeywordListName(String keywordListName) { private synchronized void addKeywordListName(String keywordListName) {
if (!keywordListNames.contains(keywordListName)) { if (!keywordListNames.contains(keywordListName)) {
keywordListNames.add(keywordListName); keywordListNames.add(keywordListName);
} }
} }
public synchronized List<Long> currentKeywordResults(Keyword k) { private synchronized Set<Long> currentKeywordResults(Keyword k) {
return currentResults.get(k); return currentResults.get(k);
} }
public synchronized void addKeywordResults(Keyword k, List<Long> resultsIDs) { private synchronized void addKeywordResults(Keyword k, Set<Long> resultsIDs) {
currentResults.put(k, resultsIDs); currentResults.put(k, resultsIDs);
} }
public boolean isWorkerRunning() { private boolean isWorkerRunning() {
return workerRunning; return workerRunning;
} }
public void setWorkerRunning(boolean flag) { private void setWorkerRunning(boolean flag) {
workerRunning = flag; workerRunning = flag;
} }
public synchronized SearchRunner.Searcher getCurrentSearcher() { private synchronized SearchRunner.Searcher getCurrentSearcher() {
return currentSearcher; return currentSearcher;
} }
public synchronized void setCurrentSearcher(SearchRunner.Searcher searchRunner) { private synchronized void setCurrentSearcher(SearchRunner.Searcher searchRunner) {
currentSearcher = searchRunner; currentSearcher = searchRunner;
} }
public void incrementModuleReferenceCount() { private void incrementModuleReferenceCount() {
moduleReferenceCount.incrementAndGet(); moduleReferenceCount.incrementAndGet();
} }
public long decrementModuleReferenceCount() { private long decrementModuleReferenceCount() {
return moduleReferenceCount.decrementAndGet(); return moduleReferenceCount.decrementAndGet();
} }
@ -331,7 +335,7 @@ public final class SearchRunner {
* *
* @throws InterruptedException * @throws InterruptedException
*/ */
public void waitForCurrentWorker() throws InterruptedException { private void waitForCurrentWorker() throws InterruptedException {
synchronized (finalSearchLock) { synchronized (finalSearchLock) {
while (workerRunning) { while (workerRunning) {
finalSearchLock.wait(); //wait() releases the lock finalSearchLock.wait(); //wait() releases the lock
@ -342,7 +346,7 @@ public final class SearchRunner {
/** /**
* Unset workerRunning and wake up thread(s) waiting on finalSearchLock * Unset workerRunning and wake up thread(s) waiting on finalSearchLock
*/ */
public void searchNotify() { private void searchNotify() {
synchronized (finalSearchLock) { synchronized (finalSearchLock) {
workerRunning = false; workerRunning = false;
finalSearchLock.notify(); finalSearchLock.notify();
@ -468,8 +472,8 @@ public final class SearchRunner {
return null; return null;
} }
// calculate new results by substracting results already obtained in this ingest // Reduce the results of the query to only those hits we
// this creates a map of each keyword to the list of unique files that have that hit. // have not already seen.
QueryResults newResults = filterResults(queryResults); QueryResults newResults = filterResults(queryResults);
if (!newResults.getKeywords().isEmpty()) { if (!newResults.getKeywords().isEmpty()) {
@ -567,40 +571,68 @@ public final class SearchRunner {
}); });
} }
//calculate new results but substracting results already obtained in this ingest /**
//update currentResults map with the new results * This method filters out all of the hits found in earlier
* periodic searches and returns only the results found by the most
* recent search.
*
* This method will only return hits for objects for which we haven't
* previously seen a hit for the keyword.
*
* @param queryResult The results returned by a keyword search.
* @return The set of hits found by the most recent search for objects
* that have not previously had a hit.
*
*/
private QueryResults filterResults(QueryResults queryResult) { private QueryResults filterResults(QueryResults queryResult) {
// Create a new (empty) QueryResults object to hold the most recently
// found hits.
QueryResults newResults = new QueryResults(queryResult.getQuery(), queryResult.getKeywordList()); QueryResults newResults = new QueryResults(queryResult.getQuery(), queryResult.getKeywordList());
// For each keyword represented in the results.
for (Keyword keyword : queryResult.getKeywords()) { for (Keyword keyword : queryResult.getKeywords()) {
// These are all of the hits across all objects for the most recent search.
// This may well include duplicates of hits we've seen in earlier periodic searches.
List<KeywordHit> queryTermResults = queryResult.getResults(keyword); List<KeywordHit> queryTermResults = queryResult.getResults(keyword);
//translate to list of IDs that we keep track of // This will be used to build up the hits we haven't seen before
List<Long> queryTermResultsIDs = new ArrayList<>(); // for this keyword.
for (KeywordHit ch : queryTermResults) { List<KeywordHit> newUniqueHits = new ArrayList<>();
queryTermResultsIDs.add(ch.getSolrObjectId());
// Get the set of object ids seen in the past by this searcher
// for the given keyword.
Set<Long> curTermResults = job.currentKeywordResults(keyword);
if (curTermResults == null) {
// We create a new empty set if we haven't seen results for
// this keyword before.
curTermResults = new HashSet<>();
} }
List<Long> curTermResults = job.currentKeywordResults(keyword); // For each hit for this keyword.
if (curTermResults == null) { for (KeywordHit hit : queryTermResults) {
job.addKeywordResults(keyword, queryTermResultsIDs); if (curTermResults.contains(hit.getSolrObjectId())) {
newResults.addResult(keyword, queryTermResults); // Skip the hit if we've already seen a hit for
} else { // this keyword in the object.
//some AbstractFile hits already exist for this keyword continue;
for (KeywordHit res : queryTermResults) {
if (!curTermResults.contains(res.getSolrObjectId())) {
//add to new results
List<KeywordHit> newResultsFs = newResults.getResults(keyword);
if (newResultsFs == null) {
newResultsFs = new ArrayList<>();
newResults.addResult(keyword, newResultsFs);
}
newResultsFs.add(res);
curTermResults.add(res.getSolrObjectId());
}
} }
// We haven't seen the hit before so add it to list of new
// unique hits.
newUniqueHits.add(hit);
// Add the object id to the results we've seen for this
// keyword.
curTermResults.add(hit.getSolrObjectId());
} }
// Update the job with the list of objects for which we have
// seen hits for the current keyword.
job.addKeywordResults(keyword, curTermResults);
// Add the new hits for the current keyword into the results
// to be returned.
newResults.addResult(keyword, newUniqueHits);
} }
return newResults; return newResults;

55
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java
View File

@ -18,18 +18,21 @@
*/ */
package org.sleuthkit.autopsy.keywordsearch; package org.sleuthkit.autopsy.keywordsearch;
import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.net.InetAddress; import java.net.InetAddress;
import java.nio.file.Paths;
import java.util.List; import java.util.List;
import java.util.MissingResourceException; import java.util.MissingResourceException;
import java.util.logging.Level;
import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.openide.util.NbBundle; import org.openide.util.NbBundle;
import org.openide.util.lookup.ServiceProvider; import org.openide.util.lookup.ServiceProvider;
import org.openide.util.lookup.ServiceProviders; import org.openide.util.lookup.ServiceProviders;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.core.RuntimeProperties; import org.sleuthkit.autopsy.core.RuntimeProperties;
import org.sleuthkit.autopsy.corecomponentinterfaces.AutopsyService; import org.sleuthkit.autopsy.corecomponentinterfaces.AutopsyService;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.keywordsearchservice.KeywordSearchService; import org.sleuthkit.autopsy.keywordsearchservice.KeywordSearchService;
import org.sleuthkit.autopsy.keywordsearchservice.KeywordSearchServiceException; import org.sleuthkit.autopsy.keywordsearchservice.KeywordSearchServiceException;
import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardArtifact;
@ -45,6 +48,7 @@ import org.sleuthkit.datamodel.TskCoreException;
) )
public class SolrSearchService implements KeywordSearchService, AutopsyService { public class SolrSearchService implements KeywordSearchService, AutopsyService {
private static final Logger logger = Logger.getLogger(IndexFinder.class.getName());
private static final String BAD_IP_ADDRESS_FORMAT = "ioexception occurred when talking to server"; //NON-NLS private static final String BAD_IP_ADDRESS_FORMAT = "ioexception occurred when talking to server"; //NON-NLS
private static final String SERVER_REFUSED_CONNECTION = "server refused connection"; //NON-NLS private static final String SERVER_REFUSED_CONNECTION = "server refused connection"; //NON-NLS
private static final int IS_REACHABLE_TIMEOUT_MS = 1000; private static final int IS_REACHABLE_TIMEOUT_MS = 1000;
@ -144,7 +148,7 @@ public class SolrSearchService implements KeywordSearchService, AutopsyService
* @param context * @param context
* *
* @throws * @throws
* org.sleuthkit.autopsy.corecomponentinterfaces.AutopsyServiceProvider.AutopsyServiceProviderException * org.sleuthkit.autopsy.corecomponentinterfaces.AutopsyService.AutopsyServiceException
*/ */
@Override @Override
public void openCaseResources(CaseContext context) throws AutopsyServiceException { public void openCaseResources(CaseContext context) throws AutopsyServiceException {
@ -153,10 +157,11 @@ public class SolrSearchService implements KeywordSearchService, AutopsyService
*/ */
// do a case subdirectory search to check for the existence and upgrade status of KWS indexes // do a case subdirectory search to check for the existence and upgrade status of KWS indexes
List<String> indexDirs = IndexHandling.findAllIndexDirs(Case.getCurrentCase()); IndexFinder indexFinder = new IndexFinder();
List<String> indexDirs = indexFinder.findAllIndexDirs(context.getCase());
// check if index needs upgrade // check if index needs upgrade
String currentVersionIndexDir = IndexHandling.findLatestVersionIndexDir(indexDirs); String currentVersionIndexDir = IndexFinder.findLatestVersionIndexDir(indexDirs);
if (currentVersionIndexDir.isEmpty()) { if (currentVersionIndexDir.isEmpty()) {
// ELTODO not sure what to do when there are multiple old indexes. grab the first one? // ELTODO not sure what to do when there are multiple old indexes. grab the first one?
@ -165,37 +170,31 @@ public class SolrSearchService implements KeywordSearchService, AutopsyService
if (RuntimeProperties.coreComponentsAreActive()) { if (RuntimeProperties.coreComponentsAreActive()) {
//pop up a message box to indicate the restrictions on adding additional //pop up a message box to indicate the restrictions on adding additional
//text and performing regex searches and give the user the option to decline the upgrade //text and performing regex searches and give the user the option to decline the upgrade
boolean upgradeDeclined = false; if (!KeywordSearchUtil.displayConfirmDialog(NbBundle.getMessage(this.getClass(), "SolrSearchService.IndexUpgradeDialog.title"),
if (upgradeDeclined) { NbBundle.getMessage(this.getClass(), "SolrSearchService.IndexUpgradeDialog.msg"),
throw new AutopsyServiceException("ELTODO"); KeywordSearchUtil.DIALOG_MESSAGE_TYPE.WARN)) {
// upgrade declined - throw exception
throw new AutopsyServiceException("Index upgrade was declined by user");
} }
} }
// ELTODO Check for cancellation at whatever points are feasible // ELTODO Check for cancellation at whatever points are feasible
// Copy the contents (core) of ModuleOutput/keywordsearch/data/index into ModuleOutput/keywordsearch/data/solr6_schema_2.0/index // Copy the "old" index and config set into ModuleOutput/keywordsearch/data/solrX_schema_Y/
String newIndexDir = indexFinder.copyIndexAndConfigSet(context.getCase(), oldIndexDir);
// Make a “reference copy” of the configset and place it in ModuleOutput/keywordsearch/data/solr6_schema_2.0/configset // upgrade the "old" index to the latest supported Solr version
IndexUpgrader indexUpgrader = new IndexUpgrader();
// convert path to UNC path indexUpgrader.performIndexUpgrade(newIndexDir, context.getCase().getTempDirectory());
// Run the upgrade tools on the contents (core) in ModuleOutput/keywordsearch/data/solr6_schema_2.0/index
// Open the upgraded index
// execute a test query
boolean success = true;
if (!success) { // set the upgraded reference index as the index to be used for this case
// delete the new directories currentVersionIndexDir = newIndexDir;
// close the upgraded index?
throw new AutopsyServiceException("ELTODO");
}
// currentVersionIndexDir = upgraded index dir
} }
// open currentVersionIndexDir index
// execute a test query
// if failed, close the upgraded index?
} }
/** /**
@ -203,7 +202,7 @@ public class SolrSearchService implements KeywordSearchService, AutopsyService
* @param context * @param context
* *
* @throws * @throws
* org.sleuthkit.autopsy.corecomponentinterfaces.AutopsyServiceProvider.AutopsyServiceProviderException * org.sleuthkit.autopsy.corecomponentinterfaces.AutopsyService.AutopsyServiceException
*/ */
@Override @Override
public void closeCaseResources(CaseContext context) throws AutopsyServiceException { public void closeCaseResources(CaseContext context) throws AutopsyServiceException {

24
NEWS.txt
View File

@ -1,23 +1,17 @@
---------------- VERSION 4.3.0 -------------- ---------------- VERSION 4.3.0 --------------
Improvements: Improvements:
- Creation and analysis (e.g., keyword search) of virtual files for slack - Support for slack space on files (as separate virtual files) to enable keyword searching and other analysis.
space. - Simple mode for the file extension mismatch module that focuses on only only multimedia and executable files to reduce false positives.
- A preloader in an Android device image does not prevent adding the image as - New view in tree that shows the MIME types.
a data source (reading of secondary GPT tables supported).
- User can add data sources with no file systems or unsupported file systems
as "unallocated space image files" for carving, keyword search, etc.
- File extension mismatch analysis can be configured to check all file types,
all file types except text files, or only multimedia and executable files.
- Column order changes in table views are "sticky" for each type of tree view
item.
- Tree view has new file types by MIME type sub tree.
- User can bulk add list of keywords to a keyword list.
- Tagged items are highlighted in table views. - Tagged items are highlighted in table views.
- Toolbar button for Image/Video Gallery - Ordering of columns is saved when user changes them.
- New "Experimental" module (activate via Tools, Plugins) with auto ingest - Support for Android devices with preloaders (uses backup GPT)
feature. - Support for images with no file systems (all data is added as unallocated space)
- User can bulk add list of keywords to a keyword list.
- New "Experimental" module (activate via Tools, Plugins) with auto ingest feature.
- Assorted bug fixes and minor enhancements. - Assorted bug fixes and minor enhancements.
---------------- VERSION 4.2.0 -------------- ---------------- VERSION 4.2.0 --------------
Improvements: Improvements:
- Credit card account search. - Credit card account search.