Refactoring hash set import

This commit is contained in:
Ann Priestman 2017-11-21 11:17:52 -05:00
parent cd6d0ca14c
commit 49a631f351
5 changed files with 322 additions and 286 deletions

View File

@ -22,31 +22,24 @@ import java.io.InputStream;
import java.io.BufferedInputStream;
import java.io.FileInputStream;
import java.io.IOException;
import java.lang.StringBuilder;
import java.util.Arrays;
import java.util.List;
import java.util.ArrayList;
import java.util.logging.Level;
import javax.swing.JOptionPane;
import org.openide.util.NbBundle;
import org.sleuthkit.autopsy.core.RuntimeProperties;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.datamodel.TskCoreException;
class EncaseHashSetParser {
final byte[] encaseHeader = {(byte)0x48, (byte)0x41, (byte)0x53, (byte)0x48, (byte)0x0d, (byte)0x0a, (byte)0xff, (byte)0x00,
class EncaseHashSetParser implements HashSetParser {
private final byte[] encaseHeader = {(byte)0x48, (byte)0x41, (byte)0x53, (byte)0x48, (byte)0x0d, (byte)0x0a, (byte)0xff, (byte)0x00,
(byte)0x02, (byte)0x00, (byte)0x00, (byte)0x00, (byte)0x01, (byte)0x00, (byte)0x00, (byte)0x00};
InputStream inputStream;
final int expectedHashes;
int totalHashesRead = 0;
private InputStream inputStream;
private final long expectedHashCount;
private int totalHashesRead = 0;
/**
* Opens the import file and parses the header.
* @param filename The Encase hashset
* @throws TskCoreException There was an error opening/reading the file or it is not the correct format
*/
@NbBundle.Messages({"EncaseHashSetParser.fileOpenError.text=Error reading import file",
"EncaseHashSetParser.wrongFormat.text=Hashset is not Encase format"})
EncaseHashSetParser(String filename) throws TskCoreException{
try{
inputStream = new BufferedInputStream(new FileInputStream(filename));
@ -55,16 +48,14 @@ class EncaseHashSetParser {
byte[] header = new byte[16];
readBuffer(header, 16);
if(! Arrays.equals(header, encaseHeader)){
displayError(NbBundle.getMessage(this.getClass(),
"EncaseHashSetParser.wrongFormat.text"));
close();
throw new TskCoreException("File " + filename + " does not have an Encase header");
}
// Read in the expected number of hashes
// Read in the expected number of hashes (little endian)
byte[] sizeBuffer = new byte[4];
readBuffer(sizeBuffer, 4);
expectedHashes = ((sizeBuffer[3] & 0xff) << 24) | ((sizeBuffer[2] & 0xff) << 16)
expectedHashCount = ((sizeBuffer[3] & 0xff) << 24) | ((sizeBuffer[2] & 0xff) << 16)
| ((sizeBuffer[1] & 0xff) << 8) | (sizeBuffer[0] & 0xff);
// Read in a bunch of nulls
@ -80,8 +71,6 @@ class EncaseHashSetParser {
readBuffer(typeBuffer, 0x28);
} catch (IOException ex){
displayError(NbBundle.getMessage(this.getClass(),
"EncaseHashSetParser.fileOpenError.text"));
close();
throw new TskCoreException("Error reading " + filename, ex);
} catch (TskCoreException ex){
@ -90,21 +79,34 @@ class EncaseHashSetParser {
}
}
int getExpectedHashes(){
return expectedHashes;
/**
* Get the expected number of hashes in the file.
* This number can be an estimate.
* @return The expected hash count
*/
@Override
public long getExpectedHashCount(){
return expectedHashCount;
}
synchronized boolean doneReading(){
if(inputStream == null){
return true;
}
return(totalHashesRead >= expectedHashes);
/**
* Check if there are more hashes to read
* @return true if we've read all expected hash values, false otherwise
*/
@Override
public boolean doneReading(){
return(totalHashesRead >= expectedHashCount);
}
synchronized String getNextHash() throws TskCoreException{
/**
* Get the next hash to import
* @return The hash as a string, or null if the end of file was reached without error
* @throws TskCoreException
*/
@Override
public String getNextHash() throws TskCoreException{
if(inputStream == null){
return null;
throw new TskCoreException("Attempting to read from null inputStream");
}
byte[] hashBytes = new byte[16];
@ -122,14 +124,16 @@ class EncaseHashSetParser {
totalHashesRead++;
return sb.toString();
} catch (IOException ex){
// Log it and return what we've got
Logger.getLogger(EncaseHashSetParser.class.getName()).log(Level.SEVERE, "Ran out of data while reading Encase hash sets", ex);
close();
throw new TskCoreException("Error reading hash", ex);
}
}
synchronized final void close(){
/**
* Closes the import file
*/
@Override
public final void close(){
if(inputStream != null){
try{
inputStream.close();
@ -142,26 +146,13 @@ class EncaseHashSetParser {
}
@NbBundle.Messages({"EncaseHashSetParser.outOfData.text=Ran out of data while parsing file"})
private synchronized void readBuffer(byte[] buffer, int length) throws TskCoreException, IOException {
private void readBuffer(byte[] buffer, int length) throws TskCoreException, IOException {
if(inputStream == null){
throw new TskCoreException("readBuffer called on null inputStream");
}
if(length != inputStream.read(buffer)){
displayError(NbBundle.getMessage(this.getClass(),
"EncaseHashSetParser.outOfData.text"));
close();
throw new TskCoreException("Ran out of data while parsing Encase file");
}
}
@NbBundle.Messages({"EncaseHashSetParser.error.title=Error importing Encase hashset"})
private void displayError(String errorText){
if(RuntimeProperties.runningWithGUI()){
JOptionPane.showMessageDialog(null,
errorText,
NbBundle.getMessage(this.getClass(),
"EncaseHashSetParser.error.title"),
JOptionPane.ERROR_MESSAGE);
throw new TskCoreException("Ran out of data unexpectedly while parsing Encase file");
}
}
}

View File

@ -447,7 +447,8 @@ final class HashDbImportDatabaseDialog extends javax.swing.JDialog {
@NbBundle.Messages({"HashDbImportDatabaseDialog.missingVersion=A version must be entered",
"HashDbImportDatabaseDialog.missingOrg=An organization must be selected",
"HashDbImportDatabaseDialog.duplicateName=A hashset with this name and version already exists",
"HashDbImportDatabaseDialog.databaseLookupError=Error accessing central repository"
"HashDbImportDatabaseDialog.databaseLookupError=Error accessing central repository",
"HashDbImportDatabaseDialog.mustEnterHashSetNameMsg=A hash set name must be entered."
})
private void okButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_okButtonActionPerformed
// Note that the error handlers in this method call return without disposing of the
@ -456,7 +457,7 @@ final class HashDbImportDatabaseDialog extends javax.swing.JDialog {
if (hashSetNameTextField.getText().isEmpty()) {
JOptionPane.showMessageDialog(this,
NbBundle.getMessage(this.getClass(),
"HashDbCreateDatabaseDialog.mustEnterHashSetNameMsg"),
"HashDbImportDatabaseDialog.mustEnterHashSetNameMsg"),
NbBundle.getMessage(this.getClass(),
"HashDbImportDatabaseDialog.importHashDbErr"),
JOptionPane.ERROR_MESSAGE);

View File

@ -0,0 +1,49 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2011 - 2017 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.modules.hashdatabase;
import org.sleuthkit.datamodel.TskCoreException;
interface HashSetParser {
/**
* Get the next hash to import
* @return The hash as a string, or null if the end of file was reached without error
* @throws TskCoreException
*/
String getNextHash() throws TskCoreException;
/**
* Check if there are more hashes to read
* @return true if we've read all expected hash values, false otherwise
*/
boolean doneReading();
/**
* Get the expected number of hashes in the file.
* This number can be an estimate.
* @return The expected hash count
*/
long getExpectedHashCount();
/**
* Closes the import file
*/
void close();
}

View File

@ -0,0 +1,113 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2011 - 2017 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.modules.hashdatabase;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.logging.Level;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.datamodel.TskCoreException;
/**
* Parser for idx files
*/
class IdxHashSetParser implements HashSetParser {
private String filename;
private BufferedReader reader;
private final long totalHashes;
private boolean doneReading = false;
IdxHashSetParser(String filename) throws TskCoreException{
this.filename = filename;
try{
reader = new BufferedReader(new FileReader(filename));
} catch (FileNotFoundException ex){
throw new TskCoreException("Error opening file " + filename, ex);
}
// Estimate the total number of hashes in the file since counting them all can be slow
File importFile = new File(filename);
long fileSize = importFile.length();
totalHashes = fileSize / 0x33 + 1; // IDX file lines are generally 0x33 bytes long. We add one to prevent this from being zero
}
/**
* Get the next hash to import
* @return The hash as a string, or null if the end of file was reached without error
* @throws TskCoreException
*/
@Override
public String getNextHash() throws TskCoreException {
String line;
try{
while ((line = reader.readLine()) != null) {
String[] parts = line.split("\\|");
// Header lines start with a 41 character dummy hash, 1 character longer than a SHA-1 hash
if (parts.length != 2 || parts[0].length() == 41) {
continue;
}
return parts[0].toLowerCase();
}
} catch (IOException ex){
throw new TskCoreException("Error reading file " + filename, ex);
}
// We've run out of data
doneReading = true;
return null;
}
/**
* Check if there are more hashes to read
* @return true if we've read all expected hash values, false otherwise
*/
@Override
public boolean doneReading() {
return doneReading;
}
/**
* Get the expected number of hashes in the file.
* This number can be an estimate.
* @return The expected hash count
*/
@Override
public long getExpectedHashCount() {
return totalHashes;
}
/**
* Closes the import file
*/
@Override
public void close() {
try{
reader.close();
} catch (IOException ex){
Logger.getLogger(IdxHashSetParser.class.getName()).log(Level.SEVERE, "Error closing file " + filename, ex);
}
}
}

View File

@ -19,12 +19,8 @@
package org.sleuthkit.autopsy.modules.hashdatabase;
import java.awt.Color;
import java.awt.Cursor;
import java.beans.PropertyChangeListener;
import java.beans.PropertyChangeEvent;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.util.HashSet;
import java.util.Set;
import java.util.logging.Level;
@ -32,8 +28,8 @@ import javax.swing.JFrame;
import javax.swing.SwingWorker;
import javax.swing.WindowConstants;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.Executors;
import javax.swing.JOptionPane;
import org.openide.util.NbBundle;
import org.openide.windows.WindowManager;
import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttribute;
@ -50,18 +46,8 @@ import org.sleuthkit.datamodel.TskData;
*/
class ImportCentralRepoDbProgressDialog extends javax.swing.JDialog implements PropertyChangeListener{
private CentralRepoImportWorker worker;
/**
*
* @param hashSetName
* @param version
* @param orgId
* @param searchDuringIngest
* @param sendIngestMessages
* @param knownFilesType
* @param importFile
*/
private CentralRepoImportWorker worker; // Swing worker that will import the file and send updates to the dialog
@NbBundle.Messages({"ImportCentralRepoDbProgressDialog.title.text=Central Repository Import Progress",
})
ImportCentralRepoDbProgressDialog() {
@ -78,29 +64,16 @@ class ImportCentralRepoDbProgressDialog extends javax.swing.JDialog implements P
bnOk.setEnabled(false);
}
@NbBundle.Messages({"ImportCentralRepoDbProgressDialog.unknownFormat.message=Hash set to import is an unknown format"})
void importFile(String hashSetName, String version, int orgId,
boolean searchDuringIngest, boolean sendIngestMessages, HashDbManager.HashDb.KnownFilesType knownFilesType,
boolean readOnly, String importFileName){
setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR));
File importFile = new File(importFileName);
if(importFileName.toLowerCase().endsWith(".idx")){
worker = new ImportIDXWorker(hashSetName, version, orgId, searchDuringIngest, sendIngestMessages,
knownFilesType, readOnly, importFile);
} else if(importFileName.toLowerCase().endsWith(".hash")){
worker = new ImportEncaseWorker(hashSetName, version, orgId, searchDuringIngest, sendIngestMessages,
knownFilesType, readOnly, importFile);
} else {
// We've gotten here with a format that can't be processed
JOptionPane.showMessageDialog(null, Bundle.ImportCentralRepoDbProgressDialog_unknownFormat_message());
return;
}
boolean readOnly, String importFileName){
worker = new CentralRepoImportWorker(hashSetName, version, orgId, searchDuringIngest, sendIngestMessages,
knownFilesType, readOnly, importFileName);
worker.addPropertyChangeListener(this);
worker.execute();
setLocationRelativeTo((JFrame) WindowManager.getDefault().getMainWindow());
setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR));
this.setVisible(true);
}
@ -111,53 +84,67 @@ class ImportCentralRepoDbProgressDialog extends javax.swing.JDialog implements P
return null;
}
@NbBundle.Messages({"ImportCentralRepoDbProgressDialog.linesProcessed= hashes processed"})
/**
* Updates the dialog from events from the worker.
* The two events we handle are progress updates and
* the done event.
* @param evt
*/
@NbBundle.Messages({"ImportCentralRepoDbProgressDialog.errorParsingFile.message=Error parsing hash set file"})
@Override
public void propertyChange(PropertyChangeEvent evt) {
if("progress".equals(evt.getPropertyName())){
progressBar.setValue(worker.getProgressPercentage());
// The progress has been updated. Update the progress bar and text
progressBar.setValue(worker.getProgress());
lbProgress.setText(getProgressString());
} else if ("state".equals(evt.getPropertyName())
&& (SwingWorker.StateValue.DONE.equals(evt.getNewValue()))) {
// Disable cancel and enable ok
// The worker is done processing
// Disable cancel button and enable ok
bnCancel.setEnabled(false);
bnOk.setEnabled(true);
if(worker.getError().isEmpty()){
if(worker.getImportSuccess()){
// If the import succeeded, finish the progress bar and display the
// total number of imported hashes
progressBar.setValue(progressBar.getMaximum());
lbProgress.setText(getProgressString());
} else {
// If there was an error, reset the progress bar and display an error message
progressBar.setValue(0);
lbProgress.setForeground(Color.red);
lbProgress.setText(worker.getError());
lbProgress.setText(Bundle.ImportCentralRepoDbProgressDialog_errorParsingFile_message());
}
}
}
@NbBundle.Messages({"ImportCentralRepoDbProgressDialog.linesProcessed.message= hashes processed"})
private String getProgressString(){
return worker.getLinesProcessed() + Bundle.ImportCentralRepoDbProgressDialog_linesProcessed();
return worker.getLinesProcessed() + Bundle.ImportCentralRepoDbProgressDialog_linesProcessed_message();
}
abstract class CentralRepoImportWorker extends SwingWorker<Void, Void>{
final int HASH_IMPORT_THRESHOLD = 10000;
final String hashSetName;
final String version;
final int orgId;
final boolean searchDuringIngest;
final boolean sendIngestMessages;
final HashDbManager.HashDb.KnownFilesType knownFilesType;
final boolean readOnly;
final File importFile;
long totalHashes = 1;
int referenceSetID = -1;
HashDbManager.CentralRepoHashSet newHashDb = null;
final AtomicLong numLines = new AtomicLong();
String errorString = "";
class CentralRepoImportWorker extends SwingWorker<Void, Void>{
private final int HASH_IMPORT_THRESHOLD = 10000;
private final String hashSetName;
private final String version;
private final int orgId;
private final boolean searchDuringIngest;
private final boolean sendIngestMessages;
private final HashDbManager.HashDb.KnownFilesType knownFilesType;
private final boolean readOnly;
private final String importFileName;
private long totalHashes = 1;
private int referenceSetID = -1;
private HashDbManager.CentralRepoHashSet newHashDb = null;
private final AtomicLong numLines = new AtomicLong();
private final AtomicBoolean importSuccess = new AtomicBoolean();
CentralRepoImportWorker(String hashSetName, String version, int orgId,
boolean searchDuringIngest, boolean sendIngestMessages, HashDbManager.HashDb.KnownFilesType knownFilesType,
boolean readOnly, File importFile){
boolean readOnly, String importFileName){
this.hashSetName = hashSetName;
this.version = version;
@ -166,11 +153,12 @@ class ImportCentralRepoDbProgressDialog extends javax.swing.JDialog implements P
this.sendIngestMessages = sendIngestMessages;
this.knownFilesType = knownFilesType;
this.readOnly = readOnly;
this.importFile = importFile;
this.importFileName = importFileName;
this.numLines.set(0);
this.importSuccess.set(false);
}
HashDbManager.CentralRepoHashSet getDatabase(){
synchronized HashDbManager.CentralRepoHashSet getDatabase(){
return newHashDb;
}
@ -178,20 +166,81 @@ class ImportCentralRepoDbProgressDialog extends javax.swing.JDialog implements P
return numLines.get();
}
int getProgressPercentage(){
return this.getProgress();
boolean getImportSuccess(){
return importSuccess.get();
}
String getError(){
return errorString;
@Override
protected Void doInBackground() throws Exception {
// Create the hash set parser
HashSetParser hashSetParser;
if(importFileName.toLowerCase().endsWith(".idx")){
hashSetParser = new IdxHashSetParser(importFileName);
} else
if(importFileName.toLowerCase().endsWith(".hash")){
hashSetParser = new EncaseHashSetParser(importFileName);
} else {
// We've gotten here with a format that can't be processed
throw new TskCoreException("Hash set to import is an unknown format : " + importFileName);
}
try{
totalHashes = hashSetParser.getExpectedHashCount();
TskData.FileKnown knownStatus;
if (knownFilesType.equals(HashDbManager.HashDb.KnownFilesType.KNOWN)) {
knownStatus = TskData.FileKnown.KNOWN;
} else {
knownStatus = TskData.FileKnown.BAD;
}
// Create an empty hashset in the central repository
referenceSetID = EamDb.getInstance().newReferenceSet(new EamGlobalSet(orgId, hashSetName, version, knownStatus, readOnly));
EamDb dbManager = EamDb.getInstance();
CorrelationAttribute.Type contentType = dbManager.getCorrelationTypeById(CorrelationAttribute.FILES_TYPE_ID); // get "FILES" type
Set<EamGlobalFileInstance> globalInstances = new HashSet<>();
while (! hashSetParser.doneReading()) {
if(isCancelled()){
return null;
}
String newHash = hashSetParser.getNextHash();
if(newHash != null){
EamGlobalFileInstance eamGlobalFileInstance = new EamGlobalFileInstance(
referenceSetID,
newHash,
knownStatus,
"");
globalInstances.add(eamGlobalFileInstance);
if(numLines.incrementAndGet() % HASH_IMPORT_THRESHOLD == 0){
dbManager.bulkInsertReferenceTypeEntries(globalInstances, contentType);
globalInstances.clear();
int progress = (int)(numLines.get() * 100 / totalHashes);
if(progress < 100){
this.setProgress(progress);
} else {
this.setProgress(99);
}
}
}
}
dbManager.bulkInsertReferenceTypeEntries(globalInstances, contentType);
this.setProgress(100);
return null;
} finally {
hashSetParser.close();
}
}
/**
* Should be called in the constructor to set the max number of hashes.
* The value can be updated later after parsing the import file.
*/
abstract void setEstimatedTotalHashes();
void deleteIncompleteSet(){
if(referenceSetID >= 0){
@ -209,10 +258,8 @@ class ImportCentralRepoDbProgressDialog extends javax.swing.JDialog implements P
}
}
@NbBundle.Messages({"ImportCentralRepoDbProgressDialog.importHashsetError=Error importing hash set",
"ImportCentralRepoDbProgressDialog.addDbError.message=Error adding new hash set"})
@Override
protected void done() {
synchronized protected void done() {
if(isCancelled()){
// If the user hit cancel, delete this incomplete hash set from the central repo
@ -226,184 +273,19 @@ class ImportCentralRepoDbProgressDialog extends javax.swing.JDialog implements P
newHashDb = HashDbManager.getInstance().addExistingCentralRepoHashSet(hashSetName, version,
referenceSetID,
searchDuringIngest, sendIngestMessages, knownFilesType, readOnly);
importSuccess.set(true);
} catch (TskCoreException ex){
JOptionPane.showMessageDialog(null, Bundle.ImportCentralRepoDbProgressDialog_addDbError_message());
Logger.getLogger(ImportCentralRepoDbProgressDialog.class.getName()).log(Level.SEVERE, "Error adding imported hash set", ex);
}
} catch (Exception ex) {
// Delete this incomplete hash set from the central repo
deleteIncompleteSet();
errorString = Bundle.ImportCentralRepoDbProgressDialog_importHashsetError();
Logger.getLogger(ImportCentralRepoDbProgressDialog.class.getName()).log(Level.SEVERE, "Error importing hash set", ex);
}
}
}
class ImportEncaseWorker extends CentralRepoImportWorker{
ImportEncaseWorker(String hashSetName, String version, int orgId,
boolean searchDuringIngest, boolean sendIngestMessages, HashDbManager.HashDb.KnownFilesType knownFilesType,
boolean readOnly, File importFile){
super(hashSetName, version, orgId, searchDuringIngest, sendIngestMessages, knownFilesType, readOnly, importFile);
setEstimatedTotalHashes();
}
/**
* Encase files have a 0x480 byte header, then each hash takes 18 bytes
* @return Approximate number of hashes in the file
*/
@Override
final void setEstimatedTotalHashes(){
long fileSize = importFile.length();
if(fileSize < 0x492){
totalHashes = 1; // There's room for at most one hash
}
totalHashes = (fileSize - 0x492) / 18;
}
@Override
protected Void doInBackground() throws Exception {
EncaseHashSetParser encaseParser = new EncaseHashSetParser(importFile.getAbsolutePath());
totalHashes = encaseParser.getExpectedHashes();
TskData.FileKnown knownStatus;
if (knownFilesType.equals(HashDbManager.HashDb.KnownFilesType.KNOWN)) {
knownStatus = TskData.FileKnown.KNOWN;
} else {
knownStatus = TskData.FileKnown.BAD;
}
// Create an empty hashset in the central repository
referenceSetID = EamDb.getInstance().newReferenceSet(new EamGlobalSet(orgId, hashSetName, version, knownStatus, readOnly));
EamDb dbManager = EamDb.getInstance();
CorrelationAttribute.Type contentType = dbManager.getCorrelationTypeById(CorrelationAttribute.FILES_TYPE_ID); // get "FILES" type
Set<EamGlobalFileInstance> globalInstances = new HashSet<>();
while (! encaseParser.doneReading()) {
if(isCancelled()){
return null;
}
String newHash = encaseParser.getNextHash();
if(newHash != null){
EamGlobalFileInstance eamGlobalFileInstance = new EamGlobalFileInstance(
referenceSetID,
newHash,
knownStatus,
"");
globalInstances.add(eamGlobalFileInstance);
numLines.incrementAndGet();
if(numLines.get() % HASH_IMPORT_THRESHOLD == 0){
dbManager.bulkInsertReferenceTypeEntries(globalInstances, contentType);
globalInstances.clear();
int progress = (int)(numLines.get() * 100 / totalHashes);
if(progress < 100){
this.setProgress(progress);
} else {
this.setProgress(99);
}
}
}
}
dbManager.bulkInsertReferenceTypeEntries(globalInstances, contentType);
this.setProgress(100);
return null;
}
}
class ImportIDXWorker extends CentralRepoImportWorker{
ImportIDXWorker(String hashSetName, String version, int orgId,
boolean searchDuringIngest, boolean sendIngestMessages, HashDbManager.HashDb.KnownFilesType knownFilesType,
boolean readOnly, File importFile){
super(hashSetName, version, orgId, searchDuringIngest, sendIngestMessages, knownFilesType, readOnly, importFile);
setEstimatedTotalHashes();
}
/**
* Doing an actual count of the number of lines in a large idx file (such
* as the nsrl) is slow, so just get something in the general area for the
* progress bar.
* @return Approximate number of hashes in the file
*/
@Override
final void setEstimatedTotalHashes(){
long fileSize = importFile.length();
totalHashes = fileSize / 0x33 + 1; // IDX file lines are generally 0x33 bytes long, and we don't want this to be zero
}
@Override
protected Void doInBackground() throws Exception {
TskData.FileKnown knownStatus;
if (knownFilesType.equals(HashDbManager.HashDb.KnownFilesType.KNOWN)) {
knownStatus = TskData.FileKnown.KNOWN;
} else {
knownStatus = TskData.FileKnown.BAD;
}
// Create an empty hashset in the central repository
referenceSetID = EamDb.getInstance().newReferenceSet(new EamGlobalSet(orgId, hashSetName, version, knownStatus, readOnly));
EamDb dbManager = EamDb.getInstance();
CorrelationAttribute.Type contentType = dbManager.getCorrelationTypeById(CorrelationAttribute.FILES_TYPE_ID); // get "FILES" type
BufferedReader reader = new BufferedReader(new FileReader(importFile));
String line;
Set<EamGlobalFileInstance> globalInstances = new HashSet<>();
while ((line = reader.readLine()) != null) {
if(isCancelled()){
return null;
}
String[] parts = line.split("\\|");
// Header lines start with a 41 character dummy hash, 1 character longer than a SHA-1 hash
if (parts.length != 2 || parts[0].length() == 41) {
continue;
}
EamGlobalFileInstance eamGlobalFileInstance = new EamGlobalFileInstance(
referenceSetID,
parts[0].toLowerCase(),
knownStatus,
"");
globalInstances.add(eamGlobalFileInstance);
numLines.incrementAndGet();
if(numLines.get() % HASH_IMPORT_THRESHOLD == 0){
dbManager.bulkInsertReferenceTypeEntries(globalInstances, contentType);
globalInstances.clear();
int progress = (int)(numLines.get() * 100 / totalHashes);
if(progress < 100){
this.setProgress(progress);
} else {
this.setProgress(99);
}
}
}
dbManager.bulkInsertReferenceTypeEntries(globalInstances, contentType);
this.setProgress(100);
return null;
}
}
/**
* This method is called from within the constructor to initialize the form.
* WARNING: Do NOT modify this code. The content of this method is always