From 49a631f351a276a08910a89e4796edaad133eecd Mon Sep 17 00:00:00 2001 From: Ann Priestman Date: Tue, 21 Nov 2017 11:17:52 -0500 Subject: [PATCH] Refactoring hash set import --- .../hashdatabase/EncaseHashSetParser.java | 83 ++-- .../HashDbImportDatabaseDialog.java | 5 +- .../modules/hashdatabase/HashSetParser.java | 49 +++ .../hashdatabase/IdxHashSetParser.java | 113 ++++++ .../ImportCentralRepoDbProgressDialog.java | 358 ++++++------------ 5 files changed, 322 insertions(+), 286 deletions(-) create mode 100644 Core/src/org/sleuthkit/autopsy/modules/hashdatabase/HashSetParser.java create mode 100644 Core/src/org/sleuthkit/autopsy/modules/hashdatabase/IdxHashSetParser.java diff --git a/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/EncaseHashSetParser.java b/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/EncaseHashSetParser.java index df9d78b7a3..9d2d4709be 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/EncaseHashSetParser.java +++ b/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/EncaseHashSetParser.java @@ -22,31 +22,24 @@ import java.io.InputStream; import java.io.BufferedInputStream; import java.io.FileInputStream; import java.io.IOException; -import java.lang.StringBuilder; import java.util.Arrays; -import java.util.List; -import java.util.ArrayList; import java.util.logging.Level; -import javax.swing.JOptionPane; import org.openide.util.NbBundle; -import org.sleuthkit.autopsy.core.RuntimeProperties; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.datamodel.TskCoreException; -class EncaseHashSetParser { - final byte[] encaseHeader = {(byte)0x48, (byte)0x41, (byte)0x53, (byte)0x48, (byte)0x0d, (byte)0x0a, (byte)0xff, (byte)0x00, +class EncaseHashSetParser implements HashSetParser { + private final byte[] encaseHeader = {(byte)0x48, (byte)0x41, (byte)0x53, (byte)0x48, (byte)0x0d, (byte)0x0a, (byte)0xff, (byte)0x00, (byte)0x02, (byte)0x00, (byte)0x00, (byte)0x00, (byte)0x01, (byte)0x00, (byte)0x00, (byte)0x00}; - InputStream inputStream; - final int expectedHashes; - int totalHashesRead = 0; + private InputStream inputStream; + private final long expectedHashCount; + private int totalHashesRead = 0; /** * Opens the import file and parses the header. * @param filename The Encase hashset * @throws TskCoreException There was an error opening/reading the file or it is not the correct format */ - @NbBundle.Messages({"EncaseHashSetParser.fileOpenError.text=Error reading import file", - "EncaseHashSetParser.wrongFormat.text=Hashset is not Encase format"}) EncaseHashSetParser(String filename) throws TskCoreException{ try{ inputStream = new BufferedInputStream(new FileInputStream(filename)); @@ -55,16 +48,14 @@ class EncaseHashSetParser { byte[] header = new byte[16]; readBuffer(header, 16); if(! Arrays.equals(header, encaseHeader)){ - displayError(NbBundle.getMessage(this.getClass(), - "EncaseHashSetParser.wrongFormat.text")); close(); throw new TskCoreException("File " + filename + " does not have an Encase header"); } - // Read in the expected number of hashes + // Read in the expected number of hashes (little endian) byte[] sizeBuffer = new byte[4]; readBuffer(sizeBuffer, 4); - expectedHashes = ((sizeBuffer[3] & 0xff) << 24) | ((sizeBuffer[2] & 0xff) << 16) + expectedHashCount = ((sizeBuffer[3] & 0xff) << 24) | ((sizeBuffer[2] & 0xff) << 16) | ((sizeBuffer[1] & 0xff) << 8) | (sizeBuffer[0] & 0xff); // Read in a bunch of nulls @@ -80,8 +71,6 @@ class EncaseHashSetParser { readBuffer(typeBuffer, 0x28); } catch (IOException ex){ - displayError(NbBundle.getMessage(this.getClass(), - "EncaseHashSetParser.fileOpenError.text")); close(); throw new TskCoreException("Error reading " + filename, ex); } catch (TskCoreException ex){ @@ -90,21 +79,34 @@ class EncaseHashSetParser { } } - int getExpectedHashes(){ - return expectedHashes; + /** + * Get the expected number of hashes in the file. + * This number can be an estimate. + * @return The expected hash count + */ + @Override + public long getExpectedHashCount(){ + return expectedHashCount; } - synchronized boolean doneReading(){ - if(inputStream == null){ - return true; - } - - return(totalHashesRead >= expectedHashes); + /** + * Check if there are more hashes to read + * @return true if we've read all expected hash values, false otherwise + */ + @Override + public boolean doneReading(){ + return(totalHashesRead >= expectedHashCount); } - synchronized String getNextHash() throws TskCoreException{ + /** + * Get the next hash to import + * @return The hash as a string, or null if the end of file was reached without error + * @throws TskCoreException + */ + @Override + public String getNextHash() throws TskCoreException{ if(inputStream == null){ - return null; + throw new TskCoreException("Attempting to read from null inputStream"); } byte[] hashBytes = new byte[16]; @@ -122,14 +124,16 @@ class EncaseHashSetParser { totalHashesRead++; return sb.toString(); } catch (IOException ex){ - // Log it and return what we've got Logger.getLogger(EncaseHashSetParser.class.getName()).log(Level.SEVERE, "Ran out of data while reading Encase hash sets", ex); - close(); throw new TskCoreException("Error reading hash", ex); } } - synchronized final void close(){ + /** + * Closes the import file + */ + @Override + public final void close(){ if(inputStream != null){ try{ inputStream.close(); @@ -142,26 +146,13 @@ class EncaseHashSetParser { } @NbBundle.Messages({"EncaseHashSetParser.outOfData.text=Ran out of data while parsing file"}) - private synchronized void readBuffer(byte[] buffer, int length) throws TskCoreException, IOException { + private void readBuffer(byte[] buffer, int length) throws TskCoreException, IOException { if(inputStream == null){ throw new TskCoreException("readBuffer called on null inputStream"); } if(length != inputStream.read(buffer)){ - displayError(NbBundle.getMessage(this.getClass(), - "EncaseHashSetParser.outOfData.text")); close(); - throw new TskCoreException("Ran out of data while parsing Encase file"); - } - } - - @NbBundle.Messages({"EncaseHashSetParser.error.title=Error importing Encase hashset"}) - private void displayError(String errorText){ - if(RuntimeProperties.runningWithGUI()){ - JOptionPane.showMessageDialog(null, - errorText, - NbBundle.getMessage(this.getClass(), - "EncaseHashSetParser.error.title"), - JOptionPane.ERROR_MESSAGE); + throw new TskCoreException("Ran out of data unexpectedly while parsing Encase file"); } } } diff --git a/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/HashDbImportDatabaseDialog.java b/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/HashDbImportDatabaseDialog.java index a91ea45a16..670b7e8d19 100755 --- a/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/HashDbImportDatabaseDialog.java +++ b/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/HashDbImportDatabaseDialog.java @@ -447,7 +447,8 @@ final class HashDbImportDatabaseDialog extends javax.swing.JDialog { @NbBundle.Messages({"HashDbImportDatabaseDialog.missingVersion=A version must be entered", "HashDbImportDatabaseDialog.missingOrg=An organization must be selected", "HashDbImportDatabaseDialog.duplicateName=A hashset with this name and version already exists", - "HashDbImportDatabaseDialog.databaseLookupError=Error accessing central repository" + "HashDbImportDatabaseDialog.databaseLookupError=Error accessing central repository", + "HashDbImportDatabaseDialog.mustEnterHashSetNameMsg=A hash set name must be entered." }) private void okButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_okButtonActionPerformed // Note that the error handlers in this method call return without disposing of the @@ -456,7 +457,7 @@ final class HashDbImportDatabaseDialog extends javax.swing.JDialog { if (hashSetNameTextField.getText().isEmpty()) { JOptionPane.showMessageDialog(this, NbBundle.getMessage(this.getClass(), - "HashDbCreateDatabaseDialog.mustEnterHashSetNameMsg"), + "HashDbImportDatabaseDialog.mustEnterHashSetNameMsg"), NbBundle.getMessage(this.getClass(), "HashDbImportDatabaseDialog.importHashDbErr"), JOptionPane.ERROR_MESSAGE); diff --git a/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/HashSetParser.java b/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/HashSetParser.java new file mode 100644 index 0000000000..fc45856af9 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/HashSetParser.java @@ -0,0 +1,49 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2011 - 2017 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.modules.hashdatabase; + +import org.sleuthkit.datamodel.TskCoreException; + +interface HashSetParser { + + /** + * Get the next hash to import + * @return The hash as a string, or null if the end of file was reached without error + * @throws TskCoreException + */ + String getNextHash() throws TskCoreException; + + /** + * Check if there are more hashes to read + * @return true if we've read all expected hash values, false otherwise + */ + boolean doneReading(); + + /** + * Get the expected number of hashes in the file. + * This number can be an estimate. + * @return The expected hash count + */ + long getExpectedHashCount(); + + /** + * Closes the import file + */ + void close(); +} diff --git a/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/IdxHashSetParser.java b/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/IdxHashSetParser.java new file mode 100644 index 0000000000..9176002eda --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/IdxHashSetParser.java @@ -0,0 +1,113 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2011 - 2017 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.modules.hashdatabase; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.logging.Level; +import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.datamodel.TskCoreException; + +/** + * Parser for idx files + */ +class IdxHashSetParser implements HashSetParser { + private String filename; + private BufferedReader reader; + private final long totalHashes; + private boolean doneReading = false; + + IdxHashSetParser(String filename) throws TskCoreException{ + this.filename = filename; + try{ + reader = new BufferedReader(new FileReader(filename)); + } catch (FileNotFoundException ex){ + throw new TskCoreException("Error opening file " + filename, ex); + } + + // Estimate the total number of hashes in the file since counting them all can be slow + File importFile = new File(filename); + long fileSize = importFile.length(); + totalHashes = fileSize / 0x33 + 1; // IDX file lines are generally 0x33 bytes long. We add one to prevent this from being zero + } + + /** + * Get the next hash to import + * @return The hash as a string, or null if the end of file was reached without error + * @throws TskCoreException + */ + @Override + public String getNextHash() throws TskCoreException { + String line; + + try{ + while ((line = reader.readLine()) != null) { + + String[] parts = line.split("\\|"); + + // Header lines start with a 41 character dummy hash, 1 character longer than a SHA-1 hash + if (parts.length != 2 || parts[0].length() == 41) { + continue; + } + + return parts[0].toLowerCase(); + } + } catch (IOException ex){ + throw new TskCoreException("Error reading file " + filename, ex); + } + + // We've run out of data + doneReading = true; + return null; + } + + /** + * Check if there are more hashes to read + * @return true if we've read all expected hash values, false otherwise + */ + @Override + public boolean doneReading() { + return doneReading; + } + + /** + * Get the expected number of hashes in the file. + * This number can be an estimate. + * @return The expected hash count + */ + @Override + public long getExpectedHashCount() { + return totalHashes; + } + + /** + * Closes the import file + */ + @Override + public void close() { + try{ + reader.close(); + } catch (IOException ex){ + Logger.getLogger(IdxHashSetParser.class.getName()).log(Level.SEVERE, "Error closing file " + filename, ex); + } + } +} diff --git a/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/ImportCentralRepoDbProgressDialog.java b/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/ImportCentralRepoDbProgressDialog.java index 240e4c19a5..9712edf904 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/ImportCentralRepoDbProgressDialog.java +++ b/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/ImportCentralRepoDbProgressDialog.java @@ -19,12 +19,8 @@ package org.sleuthkit.autopsy.modules.hashdatabase; import java.awt.Color; -import java.awt.Cursor; import java.beans.PropertyChangeListener; import java.beans.PropertyChangeEvent; -import java.io.BufferedReader; -import java.io.File; -import java.io.FileReader; import java.util.HashSet; import java.util.Set; import java.util.logging.Level; @@ -32,8 +28,8 @@ import javax.swing.JFrame; import javax.swing.SwingWorker; import javax.swing.WindowConstants; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.Executors; -import javax.swing.JOptionPane; import org.openide.util.NbBundle; import org.openide.windows.WindowManager; import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttribute; @@ -50,18 +46,8 @@ import org.sleuthkit.datamodel.TskData; */ class ImportCentralRepoDbProgressDialog extends javax.swing.JDialog implements PropertyChangeListener{ - private CentralRepoImportWorker worker; - - /** - * - * @param hashSetName - * @param version - * @param orgId - * @param searchDuringIngest - * @param sendIngestMessages - * @param knownFilesType - * @param importFile - */ + private CentralRepoImportWorker worker; // Swing worker that will import the file and send updates to the dialog + @NbBundle.Messages({"ImportCentralRepoDbProgressDialog.title.text=Central Repository Import Progress", }) ImportCentralRepoDbProgressDialog() { @@ -78,29 +64,16 @@ class ImportCentralRepoDbProgressDialog extends javax.swing.JDialog implements P bnOk.setEnabled(false); } - @NbBundle.Messages({"ImportCentralRepoDbProgressDialog.unknownFormat.message=Hash set to import is an unknown format"}) void importFile(String hashSetName, String version, int orgId, boolean searchDuringIngest, boolean sendIngestMessages, HashDbManager.HashDb.KnownFilesType knownFilesType, - boolean readOnly, String importFileName){ - setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR)); - - File importFile = new File(importFileName); - if(importFileName.toLowerCase().endsWith(".idx")){ - worker = new ImportIDXWorker(hashSetName, version, orgId, searchDuringIngest, sendIngestMessages, - knownFilesType, readOnly, importFile); - } else if(importFileName.toLowerCase().endsWith(".hash")){ - worker = new ImportEncaseWorker(hashSetName, version, orgId, searchDuringIngest, sendIngestMessages, - knownFilesType, readOnly, importFile); - } else { - // We've gotten here with a format that can't be processed - JOptionPane.showMessageDialog(null, Bundle.ImportCentralRepoDbProgressDialog_unknownFormat_message()); - return; - } + boolean readOnly, String importFileName){ + + worker = new CentralRepoImportWorker(hashSetName, version, orgId, searchDuringIngest, sendIngestMessages, + knownFilesType, readOnly, importFileName); worker.addPropertyChangeListener(this); worker.execute(); setLocationRelativeTo((JFrame) WindowManager.getDefault().getMainWindow()); - setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR)); this.setVisible(true); } @@ -111,53 +84,67 @@ class ImportCentralRepoDbProgressDialog extends javax.swing.JDialog implements P return null; } - @NbBundle.Messages({"ImportCentralRepoDbProgressDialog.linesProcessed= hashes processed"}) + + /** + * Updates the dialog from events from the worker. + * The two events we handle are progress updates and + * the done event. + * @param evt + */ + @NbBundle.Messages({"ImportCentralRepoDbProgressDialog.errorParsingFile.message=Error parsing hash set file"}) @Override public void propertyChange(PropertyChangeEvent evt) { if("progress".equals(evt.getPropertyName())){ - progressBar.setValue(worker.getProgressPercentage()); + // The progress has been updated. Update the progress bar and text + progressBar.setValue(worker.getProgress()); lbProgress.setText(getProgressString()); } else if ("state".equals(evt.getPropertyName()) && (SwingWorker.StateValue.DONE.equals(evt.getNewValue()))) { - // Disable cancel and enable ok + + // The worker is done processing + // Disable cancel button and enable ok bnCancel.setEnabled(false); bnOk.setEnabled(true); - if(worker.getError().isEmpty()){ + if(worker.getImportSuccess()){ + // If the import succeeded, finish the progress bar and display the + // total number of imported hashes progressBar.setValue(progressBar.getMaximum()); lbProgress.setText(getProgressString()); } else { + // If there was an error, reset the progress bar and display an error message progressBar.setValue(0); lbProgress.setForeground(Color.red); - lbProgress.setText(worker.getError()); + lbProgress.setText(Bundle.ImportCentralRepoDbProgressDialog_errorParsingFile_message()); } } } + @NbBundle.Messages({"ImportCentralRepoDbProgressDialog.linesProcessed.message= hashes processed"}) private String getProgressString(){ - return worker.getLinesProcessed() + Bundle.ImportCentralRepoDbProgressDialog_linesProcessed(); + return worker.getLinesProcessed() + Bundle.ImportCentralRepoDbProgressDialog_linesProcessed_message(); } - abstract class CentralRepoImportWorker extends SwingWorker{ - final int HASH_IMPORT_THRESHOLD = 10000; - final String hashSetName; - final String version; - final int orgId; - final boolean searchDuringIngest; - final boolean sendIngestMessages; - final HashDbManager.HashDb.KnownFilesType knownFilesType; - final boolean readOnly; - final File importFile; - long totalHashes = 1; - int referenceSetID = -1; - HashDbManager.CentralRepoHashSet newHashDb = null; - final AtomicLong numLines = new AtomicLong(); - String errorString = ""; + class CentralRepoImportWorker extends SwingWorker{ + private final int HASH_IMPORT_THRESHOLD = 10000; + private final String hashSetName; + private final String version; + private final int orgId; + private final boolean searchDuringIngest; + private final boolean sendIngestMessages; + private final HashDbManager.HashDb.KnownFilesType knownFilesType; + private final boolean readOnly; + private final String importFileName; + private long totalHashes = 1; + private int referenceSetID = -1; + private HashDbManager.CentralRepoHashSet newHashDb = null; + private final AtomicLong numLines = new AtomicLong(); + private final AtomicBoolean importSuccess = new AtomicBoolean(); CentralRepoImportWorker(String hashSetName, String version, int orgId, boolean searchDuringIngest, boolean sendIngestMessages, HashDbManager.HashDb.KnownFilesType knownFilesType, - boolean readOnly, File importFile){ + boolean readOnly, String importFileName){ this.hashSetName = hashSetName; this.version = version; @@ -166,11 +153,12 @@ class ImportCentralRepoDbProgressDialog extends javax.swing.JDialog implements P this.sendIngestMessages = sendIngestMessages; this.knownFilesType = knownFilesType; this.readOnly = readOnly; - this.importFile = importFile; + this.importFileName = importFileName; this.numLines.set(0); + this.importSuccess.set(false); } - HashDbManager.CentralRepoHashSet getDatabase(){ + synchronized HashDbManager.CentralRepoHashSet getDatabase(){ return newHashDb; } @@ -178,20 +166,81 @@ class ImportCentralRepoDbProgressDialog extends javax.swing.JDialog implements P return numLines.get(); } - int getProgressPercentage(){ - return this.getProgress(); + boolean getImportSuccess(){ + return importSuccess.get(); } - String getError(){ - return errorString; + @Override + protected Void doInBackground() throws Exception { + + // Create the hash set parser + HashSetParser hashSetParser; + if(importFileName.toLowerCase().endsWith(".idx")){ + hashSetParser = new IdxHashSetParser(importFileName); + } else + if(importFileName.toLowerCase().endsWith(".hash")){ + hashSetParser = new EncaseHashSetParser(importFileName); + } else { + // We've gotten here with a format that can't be processed + throw new TskCoreException("Hash set to import is an unknown format : " + importFileName); + } + + try{ + totalHashes = hashSetParser.getExpectedHashCount(); + + TskData.FileKnown knownStatus; + if (knownFilesType.equals(HashDbManager.HashDb.KnownFilesType.KNOWN)) { + knownStatus = TskData.FileKnown.KNOWN; + } else { + knownStatus = TskData.FileKnown.BAD; + } + + // Create an empty hashset in the central repository + referenceSetID = EamDb.getInstance().newReferenceSet(new EamGlobalSet(orgId, hashSetName, version, knownStatus, readOnly)); + + EamDb dbManager = EamDb.getInstance(); + CorrelationAttribute.Type contentType = dbManager.getCorrelationTypeById(CorrelationAttribute.FILES_TYPE_ID); // get "FILES" type + + Set globalInstances = new HashSet<>(); + + while (! hashSetParser.doneReading()) { + if(isCancelled()){ + return null; + } + + String newHash = hashSetParser.getNextHash(); + + if(newHash != null){ + EamGlobalFileInstance eamGlobalFileInstance = new EamGlobalFileInstance( + referenceSetID, + newHash, + knownStatus, + ""); + + globalInstances.add(eamGlobalFileInstance); + + if(numLines.incrementAndGet() % HASH_IMPORT_THRESHOLD == 0){ + dbManager.bulkInsertReferenceTypeEntries(globalInstances, contentType); + globalInstances.clear(); + + int progress = (int)(numLines.get() * 100 / totalHashes); + if(progress < 100){ + this.setProgress(progress); + } else { + this.setProgress(99); + } + } + } + } + + dbManager.bulkInsertReferenceTypeEntries(globalInstances, contentType); + this.setProgress(100); + return null; + } finally { + hashSetParser.close(); + } } - /** - * Should be called in the constructor to set the max number of hashes. - * The value can be updated later after parsing the import file. - */ - abstract void setEstimatedTotalHashes(); - void deleteIncompleteSet(){ if(referenceSetID >= 0){ @@ -209,10 +258,8 @@ class ImportCentralRepoDbProgressDialog extends javax.swing.JDialog implements P } } - @NbBundle.Messages({"ImportCentralRepoDbProgressDialog.importHashsetError=Error importing hash set", - "ImportCentralRepoDbProgressDialog.addDbError.message=Error adding new hash set"}) @Override - protected void done() { + synchronized protected void done() { if(isCancelled()){ // If the user hit cancel, delete this incomplete hash set from the central repo @@ -226,184 +273,19 @@ class ImportCentralRepoDbProgressDialog extends javax.swing.JDialog implements P newHashDb = HashDbManager.getInstance().addExistingCentralRepoHashSet(hashSetName, version, referenceSetID, searchDuringIngest, sendIngestMessages, knownFilesType, readOnly); + importSuccess.set(true); } catch (TskCoreException ex){ - JOptionPane.showMessageDialog(null, Bundle.ImportCentralRepoDbProgressDialog_addDbError_message()); Logger.getLogger(ImportCentralRepoDbProgressDialog.class.getName()).log(Level.SEVERE, "Error adding imported hash set", ex); } } catch (Exception ex) { // Delete this incomplete hash set from the central repo deleteIncompleteSet(); - errorString = Bundle.ImportCentralRepoDbProgressDialog_importHashsetError(); + Logger.getLogger(ImportCentralRepoDbProgressDialog.class.getName()).log(Level.SEVERE, "Error importing hash set", ex); } } } - class ImportEncaseWorker extends CentralRepoImportWorker{ - - ImportEncaseWorker(String hashSetName, String version, int orgId, - boolean searchDuringIngest, boolean sendIngestMessages, HashDbManager.HashDb.KnownFilesType knownFilesType, - boolean readOnly, File importFile){ - super(hashSetName, version, orgId, searchDuringIngest, sendIngestMessages, knownFilesType, readOnly, importFile); - - setEstimatedTotalHashes(); - } - - - /** - * Encase files have a 0x480 byte header, then each hash takes 18 bytes - * @return Approximate number of hashes in the file - */ - @Override - final void setEstimatedTotalHashes(){ - long fileSize = importFile.length(); - if(fileSize < 0x492){ - totalHashes = 1; // There's room for at most one hash - } - totalHashes = (fileSize - 0x492) / 18; - } - - @Override - protected Void doInBackground() throws Exception { - - EncaseHashSetParser encaseParser = new EncaseHashSetParser(importFile.getAbsolutePath()); - totalHashes = encaseParser.getExpectedHashes(); - - TskData.FileKnown knownStatus; - if (knownFilesType.equals(HashDbManager.HashDb.KnownFilesType.KNOWN)) { - knownStatus = TskData.FileKnown.KNOWN; - } else { - knownStatus = TskData.FileKnown.BAD; - } - - // Create an empty hashset in the central repository - referenceSetID = EamDb.getInstance().newReferenceSet(new EamGlobalSet(orgId, hashSetName, version, knownStatus, readOnly)); - - EamDb dbManager = EamDb.getInstance(); - CorrelationAttribute.Type contentType = dbManager.getCorrelationTypeById(CorrelationAttribute.FILES_TYPE_ID); // get "FILES" type - - Set globalInstances = new HashSet<>(); - - while (! encaseParser.doneReading()) { - if(isCancelled()){ - return null; - } - - String newHash = encaseParser.getNextHash(); - - if(newHash != null){ - EamGlobalFileInstance eamGlobalFileInstance = new EamGlobalFileInstance( - referenceSetID, - newHash, - knownStatus, - ""); - - globalInstances.add(eamGlobalFileInstance); - numLines.incrementAndGet(); - - if(numLines.get() % HASH_IMPORT_THRESHOLD == 0){ - dbManager.bulkInsertReferenceTypeEntries(globalInstances, contentType); - globalInstances.clear(); - - int progress = (int)(numLines.get() * 100 / totalHashes); - if(progress < 100){ - this.setProgress(progress); - } else { - this.setProgress(99); - } - } - } - } - - dbManager.bulkInsertReferenceTypeEntries(globalInstances, contentType); - this.setProgress(100); - return null; - } - } - - - class ImportIDXWorker extends CentralRepoImportWorker{ - - ImportIDXWorker(String hashSetName, String version, int orgId, - boolean searchDuringIngest, boolean sendIngestMessages, HashDbManager.HashDb.KnownFilesType knownFilesType, - boolean readOnly, File importFile){ - super(hashSetName, version, orgId, searchDuringIngest, sendIngestMessages, knownFilesType, readOnly, importFile); - - setEstimatedTotalHashes(); - } - - /** - * Doing an actual count of the number of lines in a large idx file (such - * as the nsrl) is slow, so just get something in the general area for the - * progress bar. - * @return Approximate number of hashes in the file - */ - @Override - final void setEstimatedTotalHashes(){ - long fileSize = importFile.length(); - totalHashes = fileSize / 0x33 + 1; // IDX file lines are generally 0x33 bytes long, and we don't want this to be zero - } - - @Override - protected Void doInBackground() throws Exception { - - TskData.FileKnown knownStatus; - if (knownFilesType.equals(HashDbManager.HashDb.KnownFilesType.KNOWN)) { - knownStatus = TskData.FileKnown.KNOWN; - } else { - knownStatus = TskData.FileKnown.BAD; - } - - // Create an empty hashset in the central repository - referenceSetID = EamDb.getInstance().newReferenceSet(new EamGlobalSet(orgId, hashSetName, version, knownStatus, readOnly)); - - EamDb dbManager = EamDb.getInstance(); - CorrelationAttribute.Type contentType = dbManager.getCorrelationTypeById(CorrelationAttribute.FILES_TYPE_ID); // get "FILES" type - BufferedReader reader = new BufferedReader(new FileReader(importFile)); - String line; - Set globalInstances = new HashSet<>(); - - while ((line = reader.readLine()) != null) { - if(isCancelled()){ - return null; - } - - String[] parts = line.split("\\|"); - - // Header lines start with a 41 character dummy hash, 1 character longer than a SHA-1 hash - if (parts.length != 2 || parts[0].length() == 41) { - continue; - } - - EamGlobalFileInstance eamGlobalFileInstance = new EamGlobalFileInstance( - referenceSetID, - parts[0].toLowerCase(), - knownStatus, - ""); - - globalInstances.add(eamGlobalFileInstance); - numLines.incrementAndGet(); - - if(numLines.get() % HASH_IMPORT_THRESHOLD == 0){ - dbManager.bulkInsertReferenceTypeEntries(globalInstances, contentType); - globalInstances.clear(); - - int progress = (int)(numLines.get() * 100 / totalHashes); - if(progress < 100){ - this.setProgress(progress); - } else { - this.setProgress(99); - } - } - } - - dbManager.bulkInsertReferenceTypeEntries(globalInstances, contentType); - this.setProgress(100); - - return null; - } - } - /** * This method is called from within the constructor to initialize the form. * WARNING: Do NOT modify this code. The content of this method is always