From c9913f24e8b54119aa38335e6eccb8f40e60b147 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Tue, 9 Oct 2018 15:05:42 -0400 Subject: [PATCH 01/44] Fixed the source of the memory issue and did some light refactoring --- .../SevenZipExtractor.java | 224 ++++++------------ 1 file changed, 74 insertions(+), 150 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java index 704bdac4f8..df208b14c6 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java +++ b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java @@ -19,6 +19,7 @@ package org.sleuthkit.autopsy.modules.embeddedfileextractor; import java.io.File; +import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStream; @@ -46,6 +47,7 @@ import net.sf.sevenzipjbinding.IArchiveExtractCallback; import net.sf.sevenzipjbinding.ICryptoGetTextPassword; import net.sf.sevenzipjbinding.PropID; import org.netbeans.api.progress.ProgressHandle; +import org.openide.util.Exceptions; import org.openide.util.NbBundle; import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.casemodule.Case; @@ -93,7 +95,7 @@ class SevenZipExtractor { private String moduleDirAbsolute; private Blackboard blackboard; - + private ProgressHandle progress; private int numItems; private String currentArchiveName; @@ -164,18 +166,19 @@ class SevenZipExtractor { * * More heuristics to be added here * - * @param archiveFile the AbstractFile for the parent archive which - * which we are checking - * @param inArchive The SevenZip archive currently open for extraction - * - * @param inArchiveItemIndex Index of item inside the SevenZip archive. Each - * file inside an archive is associated with a unique - * integer - * - * @param depthMap a concurrent hashmap which keeps track of the - * depth of all nested archives, key of objectID - * @param escapedFilePath the path to the archiveFileItem which has been - * escaped + * @param archiveFile the AbstractFile for the parent archive which + * which we are checking + * @param inArchive The SevenZip archive currently open for + * extraction + * + * @param inArchiveItemIndex Index of item inside the SevenZip archive. Each + * file inside an archive is associated with a + * unique integer + * + * @param depthMap a concurrent hashmap which keeps track of the + * depth of all nested archives, key of objectID + * @param escapedFilePath the path to the archiveFileItem which has been + * escaped * * @return true if potential zip bomb, false otherwise */ @@ -543,7 +546,7 @@ class SevenZipExtractor { logger.log(Level.INFO, "Count of items in archive: {0}: {1}", new Object[]{escapedArchiveFilePath, numItems}); //NON-NLS progress.start(numItems); progressStarted = true; - + //setup the archive local root folder final String uniqueArchiveFileName = FileUtil.escapeFileName(EmbeddedFileExtractorIngestModule.getUniqueName(archiveFile)); try { @@ -597,7 +600,7 @@ class SevenZipExtractor { inArchiveItemIndex, PropID.SIZE); if (freeDiskSpace != IngestMonitor.DISK_FREE_SPACE_UNKNOWN && archiveItemSize != null && archiveItemSize > 0) { //if free space is known and file is not empty. String archiveItemPath = (String) inArchive.getProperty( - inArchiveItemIndex, PropID.PATH); + inArchiveItemIndex, PropID.PATH); long newDiskSpace = freeDiskSpace - archiveItemSize; if (newDiskSpace < MIN_FREE_DISK_SPACE) { String msg = NbBundle.getMessage(SevenZipExtractor.class, @@ -669,7 +672,7 @@ class SevenZipExtractor { inArchive.extract(extractionIndices, false, archiveCallBack); unpackSuccessful = unpackSuccessful & archiveCallBack.wasSuccessful(); - + archiveDetailsMap = null; // add them to the DB. We wait until the end so that we have the metadata on all of the @@ -788,62 +791,22 @@ class SevenZipExtractor { .toArray(); } - /** - * Stream used to unpack the archive to local file - */ - private abstract static class UnpackStream implements ISequentialOutStream { - - private OutputStream output; - private String localAbsPath; - - UnpackStream(String localAbsPath) { - this.localAbsPath = localAbsPath; - try { - output = new EncodedFileOutputStream(new FileOutputStream(localAbsPath), TskData.EncodingType.XOR1); - } catch (IOException ex) { - logger.log(Level.SEVERE, "Error writing extracted file: " + localAbsPath, ex); //NON-NLS - } - - } - - public abstract long getSize(); - - OutputStream getOutput() { - return output; - } - - String getLocalAbsPath() { - return localAbsPath; - } - - public void close() { - if (output != null) { - try { - output.flush(); - output.close(); - output = null; - } catch (IOException e) { - logger.log(Level.SEVERE, "Error closing unpack stream for file: {0}", localAbsPath); //NON-NLS - } - } - } - } - /** * Stream used to unpack the archive of unknown size to local file */ - private static class UnknownSizeUnpackStream extends UnpackStream { + private static class UnpackStream implements ISequentialOutStream, AutoCloseable { - private long freeDiskSpace; - private boolean outOfSpace = false; - private long bytesWritten = 0; + private EncodedFileOutputStream output; + private String localAbsPath; - UnknownSizeUnpackStream(String localAbsPath, long freeDiskSpace) { - super(localAbsPath); - this.freeDiskSpace = freeDiskSpace; + private long bytesWritten; + + UnpackStream(String localAbsPath) throws IOException { + output = new EncodedFileOutputStream(new FileOutputStream(localAbsPath), TskData.EncodingType.XOR1); + this.localAbsPath = localAbsPath; + this.bytesWritten = 0; } - @Override public long getSize() { return this.bytesWritten; } @@ -851,78 +814,36 @@ class SevenZipExtractor { @Override public int write(byte[] bytes) throws SevenZipException { try { - // If the content size is unknown, cautiously write to disk. - // Write only if byte array is less than 80% of the current - // free disk space. - if (freeDiskSpace == IngestMonitor.DISK_FREE_SPACE_UNKNOWN || bytes.length < 0.8 * freeDiskSpace) { - getOutput().write(bytes); - // NOTE: this method is called multiple times for a - // single extractSlow() call. Update bytesWritten and - // freeDiskSpace after every write operation. - this.bytesWritten += bytes.length; - this.freeDiskSpace -= bytes.length; - } else { - this.outOfSpace = true; - logger.log(Level.INFO, NbBundle.getMessage( - SevenZipExtractor.class, - "EmbeddedFileExtractorIngestModule.ArchiveExtractor.UnpackStream.write.noSpace.msg")); - throw new SevenZipException( - NbBundle.getMessage(SevenZipExtractor.class, "EmbeddedFileExtractorIngestModule.ArchiveExtractor.UnpackStream.write.noSpace.msg")); - } + output.write(bytes); + // Update bytesWritten + this.bytesWritten += bytes.length; } catch (IOException ex) { throw new SevenZipException( NbBundle.getMessage(SevenZipExtractor.class, "EmbeddedFileExtractorIngestModule.ArchiveExtractor.UnpackStream.write.exception.msg", - getLocalAbsPath()), ex); + localAbsPath), ex); } return bytes.length; } + public void setNewOutputStream(String localAbsPath) throws IOException { + this.output.setOutputStream(new FileOutputStream(localAbsPath), TskData.EncodingType.XOR1); + this.localAbsPath = localAbsPath; + this.bytesWritten = 0; + } + @Override public void close() { - if (getOutput() != null) { + if (output != null) { try { - getOutput().flush(); - getOutput().close(); - if (this.outOfSpace) { - Files.delete(Paths.get(getLocalAbsPath())); - } + output.flush(); + output.close(); } catch (IOException e) { - logger.log(Level.SEVERE, "Error closing unpack stream for file: {0}", getLocalAbsPath()); //NON-NLS + logger.log(Level.WARNING, "Error closing unpack stream for file: {0}", localAbsPath); //NON-NLS } } } } - /** - * Stream used to unpack the archive of known size to local file - */ - private static class KnownSizeUnpackStream extends UnpackStream { - - private long size; - - KnownSizeUnpackStream(String localAbsPath, long size) { - super(localAbsPath); - this.size = size; - } - - @Override - public long getSize() { - return this.size; - } - - @Override - public int write(byte[] bytes) throws SevenZipException { - try { - getOutput().write(bytes); - } catch (IOException ex) { - throw new SevenZipException( - NbBundle.getMessage(SevenZipExtractor.class, "EmbeddedFileExtractorIngestModule.ArchiveExtractor.UnpackStream.write.exception.msg", - getLocalAbsPath()), ex); - } - return bytes.length; - } - } - /** * Wrapper for necessary details used in StandardIArchiveExtractCallback */ @@ -965,9 +886,8 @@ class SevenZipExtractor { private UnpackStream unpackStream = null; private final Map archiveDetailsMap; private final ProgressHandle progressHandle; - + private int inArchiveItemIndex; - private final long freeDiskSpace; private long createTimeInSeconds; private long modTimeInSeconds; @@ -984,7 +904,6 @@ class SevenZipExtractor { String password, long freeDiskSpace) { this.inArchive = inArchive; - this.freeDiskSpace = freeDiskSpace; this.progressHandle = progressHandle; this.archiveFile = archiveFile; this.archiveDetailsMap = archiveDetailsMap; @@ -992,19 +911,21 @@ class SevenZipExtractor { } /** - * Get stream is called by the internal framework as it traverses - * the archive structure. The ISequentialOutStream is where the - * archive file contents will be expanded and written to the local disk. - * + * Get stream is called by the internal framework as it traverses the + * archive structure. The ISequentialOutStream is where the archive file + * contents will be expanded and written to the local disk. + * * Skips folders, as there is nothing to extract. - * - * @param inArchiveItemIndex current location of the - * @param mode Will always be EXTRACT + * + * @param inArchiveItemIndex current location of the + * @param mode Will always be EXTRACT + * * @return - * @throws SevenZipException + * + * @throws SevenZipException */ @Override - public ISequentialOutStream getStream(int inArchiveItemIndex, + public ISequentialOutStream getStream(int inArchiveItemIndex, ExtractAskMode mode) throws SevenZipException { this.inArchiveItemIndex = inArchiveItemIndex; @@ -1015,28 +936,31 @@ class SevenZipExtractor { return null; } - final Long archiveItemSize = (Long) inArchive.getProperty( - inArchiveItemIndex, PropID.SIZE); final String localAbsPath = archiveDetailsMap.get( inArchiveItemIndex).getLocalAbsPath(); - - if (archiveItemSize != null) { - unpackStream = new SevenZipExtractor.KnownSizeUnpackStream( - localAbsPath, archiveItemSize); - } else { - unpackStream = new SevenZipExtractor.UnknownSizeUnpackStream( - localAbsPath, freeDiskSpace); + + try { + if (unpackStream != null) { + unpackStream.setNewOutputStream(localAbsPath); + } else { + unpackStream = new UnpackStream(localAbsPath); + } + } catch (IOException ex) { + logger.log(Level.WARNING, String.format("Error opening or setting new stream " //NON-NLS + + "for archive file at %s", localAbsPath), ex); //NON-NLS + return null; } return unpackStream; } /** - * Retrieves the file metadata from the archive before extraction. + * Retrieves the file metadata from the archive before extraction. * Called after getStream. - * + * * @param mode Will always be EXTRACT. - * @throws SevenZipException + * + * @throws SevenZipException */ @Override public void prepareOperation(ExtractAskMode mode) throws SevenZipException { @@ -1053,18 +977,18 @@ class SevenZipExtractor { : writeTime.getTime() / 1000; accessTimeInSeconds = accessTime == null ? 0L : accessTime.getTime() / 1000; - + progressHandle.progress(archiveFile.getName() + ": " + (String) inArchive.getProperty(inArchiveItemIndex, PropID.PATH), inArchiveItemIndex); - + } /** * Updates the unpackedNode data in the tree after the archive has been - * expanded to local disk. + * expanded to local disk. * - * @param result - ExtractOperationResult + * @param result - ExtractOperationResult * * @throws SevenZipException */ @@ -1081,7 +1005,7 @@ class SevenZipExtractor { localRelPath); return; } - + final String localAbsPath = archiveDetailsMap.get( inArchiveItemIndex).getLocalAbsPath(); if (result != ExtractOperationResult.OK) { From c88ea15bbcedfd340a23ffb5672295a92f18cba0 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Tue, 9 Oct 2018 16:22:50 -0400 Subject: [PATCH 02/44] Added some comments and did some testing, everything looks good --- .../SevenZipExtractor.java | 23 +++++++++++++------ 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java index df208b14c6..977340600b 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java +++ b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java @@ -19,12 +19,8 @@ package org.sleuthkit.autopsy.modules.embeddedfileextractor; import java.io.File; -import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; -import java.io.OutputStream; -import java.nio.file.Files; -import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -47,7 +43,6 @@ import net.sf.sevenzipjbinding.IArchiveExtractCallback; import net.sf.sevenzipjbinding.ICryptoGetTextPassword; import net.sf.sevenzipjbinding.PropID; import org.netbeans.api.progress.ProgressHandle; -import org.openide.util.Exceptions; import org.openide.util.NbBundle; import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.casemodule.Case; @@ -792,7 +787,8 @@ class SevenZipExtractor { } /** - * Stream used to unpack the archive of unknown size to local file + * Stream used to unpack the archive item to local file. This will be used when + * the 7ZIP bindings call getStream() on the StandardIArchiveExtractCallback. */ private static class UnpackStream implements ISequentialOutStream, AutoCloseable { @@ -815,7 +811,6 @@ class SevenZipExtractor { public int write(byte[] bytes) throws SevenZipException { try { output.write(bytes); - // Update bytesWritten this.bytesWritten += bytes.length; } catch (IOException ex) { throw new SevenZipException( @@ -825,6 +820,15 @@ class SevenZipExtractor { return bytes.length; } + /** + * Update the OutputStream to point to a new File. This method mutates the state so + * that there is no overhead of creating a new object and buffer. Additionally, the + * 7zip binding has a memory leak, so this prevents multiple streams from being created + * and never GC'd since the 7zip lib never frees a reference to them. + * + * @param localAbsPath Path to local file + * @throws IOException + */ public void setNewOutputStream(String localAbsPath) throws IOException { this.output.setOutputStream(new FileOutputStream(localAbsPath), TskData.EncodingType.XOR1); this.localAbsPath = localAbsPath; @@ -939,6 +943,11 @@ class SevenZipExtractor { final String localAbsPath = archiveDetailsMap.get( inArchiveItemIndex).getLocalAbsPath(); + //If the Unpackstream has been allocated, then set the Outputstream + //to another file rather than creating a new unpack stream. The 7Zip + //binding has a memory leak, so creating new unpack streams will not be + //dereferenced. As a fix, we create one UnpackStream, and mutate its state, + //so that there only exists one 8192 byte buffer in memory per archive. try { if (unpackStream != null) { unpackStream.setNewOutputStream(localAbsPath); From 611353d8f1132755f00e925757cc38451be1061a Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Thu, 11 Oct 2018 16:36:13 -0400 Subject: [PATCH 03/44] Created a solution entirely within the EFE module and not changing any public API in datamodel --- .../SevenZipExtractor.java | 123 ++++++++++++------ 1 file changed, 80 insertions(+), 43 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java index 977340600b..c8f0fce062 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java +++ b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java @@ -43,6 +43,7 @@ import net.sf.sevenzipjbinding.IArchiveExtractCallback; import net.sf.sevenzipjbinding.ICryptoGetTextPassword; import net.sf.sevenzipjbinding.PropID; import org.netbeans.api.progress.ProgressHandle; +import org.openide.util.Exceptions; import org.openide.util.NbBundle; import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.casemodule.Case; @@ -790,34 +791,19 @@ class SevenZipExtractor { * Stream used to unpack the archive item to local file. This will be used when * the 7ZIP bindings call getStream() on the StandardIArchiveExtractCallback. */ - private static class UnpackStream implements ISequentialOutStream, AutoCloseable { + private final static class MutableEncodedFileOutputStream extends EncodedFileOutputStream + implements AutoCloseable { + + private static final int HEADER_LENGTH = 32; + private static final String HEADER = "TSK_CONTAINER_XOR1_xxxxxxxxxxxxx"; - private EncodedFileOutputStream output; - private String localAbsPath; - - private long bytesWritten; - - UnpackStream(String localAbsPath) throws IOException { - output = new EncodedFileOutputStream(new FileOutputStream(localAbsPath), TskData.EncodingType.XOR1); - this.localAbsPath = localAbsPath; - this.bytesWritten = 0; - } - - public long getSize() { - return this.bytesWritten; + MutableEncodedFileOutputStream(String localAbsPath) throws IOException { + super(new FileOutputStream(localAbsPath), TskData.EncodingType.XOR1); } @Override - public int write(byte[] bytes) throws SevenZipException { - try { - output.write(bytes); - this.bytesWritten += bytes.length; - } catch (IOException ex) { - throw new SevenZipException( - NbBundle.getMessage(SevenZipExtractor.class, "EmbeddedFileExtractorIngestModule.ArchiveExtractor.UnpackStream.write.exception.msg", - localAbsPath), ex); - } - return bytes.length; + public void write(byte[] bytes) throws IOException { + super.write(bytes); } /** @@ -830,22 +816,69 @@ class SevenZipExtractor { * @throws IOException */ public void setNewOutputStream(String localAbsPath) throws IOException { - this.output.setOutputStream(new FileOutputStream(localAbsPath), TskData.EncodingType.XOR1); + this.out.close(); + this.out = new FileOutputStream(localAbsPath); + writeHeader(); + } + + private byte[] getEncodedHeader() { + byte [] encHeader = new byte[HEADER_LENGTH]; + byte [] plainHeader = HEADER.getBytes(); + + for(int i = 0; i < HEADER_LENGTH; i++){ + encHeader[i] = ((byte)(plainHeader[i] ^ 0xca)); + } + return encHeader; + } + + private void writeHeader() throws IOException { + // We get the encoded header here so it will be in plaintext after encoding + write(getEncodedHeader(), 0, HEADER_LENGTH); + } + } + + private final static class UnpackStream implements ISequentialOutStream { + + private final MutableEncodedFileOutputStream output; + private String localAbsPath; + private int bytesWritten; + + UnpackStream(String localAbsPath) throws IOException { + this.output = new MutableEncodedFileOutputStream(localAbsPath); + this.localAbsPath = localAbsPath; + this.bytesWritten = 0; + } + + public void setNewOutputStream(String localAbsPath) throws IOException { + this.output.setNewOutputStream(localAbsPath); this.localAbsPath = localAbsPath; this.bytesWritten = 0; } - - @Override - public void close() { - if (output != null) { - try { - output.flush(); - output.close(); - } catch (IOException e) { - logger.log(Level.WARNING, "Error closing unpack stream for file: {0}", localAbsPath); //NON-NLS - } - } + + public int getSize() { + return bytesWritten; } + + @Override + public int write(byte[] bytes) throws SevenZipException { + try { + output.write(bytes); + this.bytesWritten += bytes.length; + } catch (IOException ex) { + throw new SevenZipException( + NbBundle.getMessage(SevenZipExtractor.class, + "EmbeddedFileExtractorIngestModule.ArchiveExtractor.UnpackStream.write.exception.msg", + localAbsPath), ex); + } + return bytes.length; + } + + public void close() throws IOException { + try(MutableEncodedFileOutputStream out = output) { + out.flush(); + } + } + } /** @@ -1028,7 +1061,11 @@ class SevenZipExtractor { !(Boolean) inArchive.getProperty(inArchiveItemIndex, PropID.IS_FOLDER), 0L, createTimeInSeconds, accessTimeInSeconds, modTimeInSeconds, localRelPath); - unpackStream.close(); + try { + unpackStream.close(); + } catch (IOException e) { + logger.log(Level.WARNING, "Error closing unpack stream for file: {0}", localAbsPath); //NON-NLS + } } @Override @@ -1139,9 +1176,9 @@ class SevenZipExtractor { */ List getRootFileObjects() { List ret = new ArrayList<>(); - for (UnpackedNode child : rootNode.getChildren()) { + rootNode.getChildren().forEach((child) -> { ret.add(child.getFile()); - } + }); return ret; } @@ -1153,17 +1190,17 @@ class SevenZipExtractor { */ List getAllFileObjects() { List ret = new ArrayList<>(); - for (UnpackedNode child : rootNode.getChildren()) { + rootNode.getChildren().forEach((child) -> { getAllFileObjectsRec(ret, child); - } + }); return ret; } private void getAllFileObjectsRec(List list, UnpackedNode parent) { list.add(parent.getFile()); - for (UnpackedNode child : parent.getChildren()) { + parent.getChildren().forEach((child) -> { getAllFileObjectsRec(list, child); - } + }); } /** From 1dda52aa03e7d4a5fa97dd563b5b6f3c253b9032 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Wed, 24 Oct 2018 14:45:43 -0400 Subject: [PATCH 04/44] 4305 added correlation attribute for wifi networks --- .../CorrelationAttributeInstance.java | 41 ++++++++++++------- .../CorrelationAttributeNormalizer.java | 4 ++ .../datamodel/EamArtifactUtil.java | 26 ++++++++---- .../recentactivity/ExtractRegistry.java | 8 +++- 4 files changed, 55 insertions(+), 24 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeInstance.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeInstance.java index c26134c5b8..eb790f866b 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeInstance.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeInstance.java @@ -171,7 +171,7 @@ public class CorrelationAttributeInstance implements Serializable { * Is this a database instance? * * @return True if the instance ID is greater or equal to zero; otherwise - * false. + * false. */ public boolean isDatabaseInstance() { return (ID >= 0); @@ -234,7 +234,7 @@ public class CorrelationAttributeInstance implements Serializable { * as notable and should never be set to KNOWN. * * @param knownStatus Should be BAD if the item is tagged as notable, - * UNKNOWN otherwise + * UNKNOWN otherwise */ public void setKnownStatus(TskData.FileKnown knownStatus) { this.knownStatus = knownStatus; @@ -246,18 +246,24 @@ public class CorrelationAttributeInstance implements Serializable { public static final int EMAIL_TYPE_ID = 2; public static final int PHONE_TYPE_ID = 3; public static final int USBID_TYPE_ID = 4; + public static final int SSID_TYPE_ID = 5; + public static final int MAC_TYPE_ID = 6; + public static final int IMEI_TYPE_ID = 7; /** * Load the default correlation types * * @throws EamDbException if the Type's dbTableName has invalid - * characters/format + * characters/format */ @Messages({"CorrelationType.FILES.displayName=Files", "CorrelationType.DOMAIN.displayName=Domains", "CorrelationType.EMAIL.displayName=Email Addresses", "CorrelationType.PHONE.displayName=Phone Numbers", - "CorrelationType.USBID.displayName=USB Devices"}) + "CorrelationType.USBID.displayName=USB Devices", + "CorrelationType.SSID.displayName=Wireless Networks", + "CorrelationType.MAC.displayName=MAC Address", + "CorrelationType.IMEI.displayName=IMEI Number"}) public static List getDefaultCorrelationTypes() throws EamDbException { List DEFAULT_CORRELATION_TYPES = new ArrayList<>(); DEFAULT_CORRELATION_TYPES.add(new CorrelationAttributeInstance.Type(FILES_TYPE_ID, Bundle.CorrelationType_FILES_displayName(), "file", true, true)); // NON-NLS @@ -265,6 +271,9 @@ public class CorrelationAttributeInstance implements Serializable { DEFAULT_CORRELATION_TYPES.add(new CorrelationAttributeInstance.Type(EMAIL_TYPE_ID, Bundle.CorrelationType_EMAIL_displayName(), "email_address", true, true)); // NON-NLS DEFAULT_CORRELATION_TYPES.add(new CorrelationAttributeInstance.Type(PHONE_TYPE_ID, Bundle.CorrelationType_PHONE_displayName(), "phone_number", true, true)); // NON-NLS DEFAULT_CORRELATION_TYPES.add(new CorrelationAttributeInstance.Type(USBID_TYPE_ID, Bundle.CorrelationType_USBID_displayName(), "usb_devices", true, true)); // NON-NLS + DEFAULT_CORRELATION_TYPES.add(new CorrelationAttributeInstance.Type(SSID_TYPE_ID, Bundle.CorrelationType_SSID_displayName(), "wireless_networks", true, true)); // NON-NLS + DEFAULT_CORRELATION_TYPES.add(new CorrelationAttributeInstance.Type(MAC_TYPE_ID, Bundle.CorrelationType_MAC_displayName(), "mac_address", true, true)); // NON-NLS + DEFAULT_CORRELATION_TYPES.add(new CorrelationAttributeInstance.Type(IMEI_TYPE_ID, Bundle.CorrelationType_IMEI_displayName(), "imei_number", true, true)); // NON-NLS return DEFAULT_CORRELATION_TYPES; } @@ -283,13 +292,14 @@ public class CorrelationAttributeInstance implements Serializable { /** * - * @param typeId Unique ID for this Correlation Type + * @param typeId Unique ID for this Correlation Type * @param displayName Name of this type displayed in the UI. * @param dbTableName Central repository db table where data of this - * type is stored. Must start with a lowercase letter and only contain - * lowercase letters, numbers, and '_' characters. - * @param supported Is this Type currently supported - * @param enabled Is this Type currently enabled. + * type is stored. Must start with a lowercase letter + * and only contain lowercase letters, numbers, and + * '_' characters. + * @param supported Is this Type currently supported + * @param enabled Is this Type currently enabled. */ public Type(int typeId, String displayName, String dbTableName, Boolean supported, Boolean enabled) throws EamDbException { if (dbTableName == null) { @@ -312,10 +322,11 @@ public class CorrelationAttributeInstance implements Serializable { * * @param displayName Name of this type displayed in the UI. * @param dbTableName Central repository db table where data of this - * type is stored Must start with a lowercase letter and only contain - * lowercase letters, numbers, and '_' characters. - * @param supported Is this Type currently supported - * @param enabled Is this Type currently enabled. + * type is stored Must start with a lowercase letter + * and only contain lowercase letters, numbers, and + * '_' characters. + * @param supported Is this Type currently supported + * @param enabled Is this Type currently enabled. */ public Type(String displayName, String dbTableName, Boolean supported, Boolean enabled) throws EamDbException { this(-1, displayName, dbTableName, supported, enabled); @@ -477,8 +488,8 @@ public class CorrelationAttributeInstance implements Serializable { * custom_instances) * * @param dbTableName the dbTableName to set. Must start with lowercase - * letter and can only contain lowercase letters, numbers, and '_' - * characters. + * letter and can only contain lowercase letters, + * numbers, and '_' characters. * * @throws EamDbException if dbTableName contains invalid characters */ diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeNormalizer.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeNormalizer.java index 772e1c517e..c7787389f4 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeNormalizer.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeNormalizer.java @@ -63,6 +63,10 @@ final public class CorrelationAttributeNormalizer { return normalizePhone(data); case CorrelationAttributeInstance.USBID_TYPE_ID: return normalizeUsbId(data); + case CorrelationAttributeInstance.SSID_TYPE_ID: + case CorrelationAttributeInstance.MAC_TYPE_ID: + case CorrelationAttributeInstance.IMEI_TYPE_ID: + return data; default: final String errorMessage = String.format( "Validator function not found for attribute type: %s", diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactUtil.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactUtil.java index 30d539e87f..c3b18e62f5 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactUtil.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactUtil.java @@ -54,9 +54,9 @@ public class EamArtifactUtil { * EamArtifact with a single EamArtifactInstance within. If not, return * null. * - * @param bbArtifact BlackboardArtifact to examine + * @param bbArtifact BlackboardArtifact to examine * @param checkEnabled If true, only create a CorrelationAttribute if it is - * enabled + * enabled * * @return List of EamArtifacts */ @@ -93,10 +93,10 @@ public class EamArtifactUtil { * based on the data in the blackboard artifact. * * @param correlationType The Central Repository artifact type to create - * @param bbArtifact The blackboard artifact to pull data from + * @param bbArtifact The blackboard artifact to pull data from * * @return the new EamArtifact, or null if one was not created because - * bbArtifact did not contain the needed data + * bbArtifact did not contain the needed data */ private static CorrelationAttributeInstance makeInstanceFromBlackboardArtifact(CorrelationAttributeInstance.Type correlationType, BlackboardArtifact bbArtifact) throws EamDbException { @@ -164,7 +164,16 @@ public class EamArtifactUtil { && BlackboardArtifact.ARTIFACT_TYPE.TSK_DEVICE_ATTACHED.getTypeID() == artifactTypeID) { value = bbArtifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DEVICE_ID)).getValueString(); - } + } + else if (correlationType.getId() == CorrelationAttributeInstance.SSID_TYPE_ID + && BlackboardArtifact.ARTIFACT_TYPE.TSK_WIFI_NETWORK.getTypeID() == artifactTypeID) { + value = bbArtifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SSID)).getValueString(); + } +// else if (correlationType.getId() == CorrelationAttributeInstance.MAC_TYPE_ID) { +// value = bbArtifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_BSSID)).getValueString(); +// } else if (correlationType.getId() == CorrelationAttributeInstance.IMEI_TYPE_ID) { +// value = bbArtifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_BSSID)).getValueString(); +// } } catch (TskCoreException ex) { logger.log(Level.SEVERE, "Error getting attribute while getting type from BlackboardArtifact.", ex); // NON-NLS @@ -185,9 +194,10 @@ public class EamArtifactUtil { * Uses the determined type and vallue, then looks up instance details to * create proper CorrelationAttributeInstance. * - * @param bbArtifact the blackboard artifatc + * @param bbArtifact the blackboard artifatc * @param correlationType the given type - * @param value the artifact value + * @param value the artifact value + * * @return CorrelationAttributeInstance from details */ private static CorrelationAttributeInstance makeCorrelationAttributeInstanceUsingTypeValue(BlackboardArtifact bbArtifact, CorrelationAttributeInstance.Type correlationType, String value) { @@ -340,7 +350,7 @@ public class EamArtifactUtil { * @param file The file to test * * @return true if the file should be added to the central repo, false - * otherwise + * otherwise */ public static boolean isSupportedAbstractFileType(AbstractFile file) { if (file == null) { diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java index 70607f361c..6612feb60c 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java @@ -374,7 +374,9 @@ class ExtractRegistry extends Extract { // Add all "usb" dataType nodes to collection of BlackboardArtifacts // that we will submit in a ModuleDataEvent for additional processing. Collection usbBBartifacts = new ArrayList<>(); - + // Add all "ssid" dataType nodes to collection of BlackboardArtifacts + // that we will submit in a ModuleDataEvent for additional processing. + Collection wifiBBartifacts = new ArrayList<>(); for (int i = 0; i < len; i++) { Element tempnode = (Element) children.item(i); @@ -734,6 +736,7 @@ class ExtractRegistry extends Extract { bbart.addAttributes(bbattributes); // index the artifact for keyword search this.indexArtifact(bbart); + wifiBBartifacts.add(bbart); } catch (TskCoreException ex) { logger.log(Level.SEVERE, "Error adding SSID artifact to blackboard."); //NON-NLS } @@ -756,6 +759,9 @@ class ExtractRegistry extends Extract { if (!usbBBartifacts.isEmpty()) { IngestServices.getInstance().fireModuleDataEvent(new ModuleDataEvent(moduleName, BlackboardArtifact.ARTIFACT_TYPE.TSK_DEVICE_ATTACHED, usbBBartifacts)); } + if (!wifiBBartifacts.isEmpty()){ + IngestServices.getInstance().fireModuleDataEvent(new ModuleDataEvent(moduleName, BlackboardArtifact.ARTIFACT_TYPE.TSK_WIFI_NETWORK, wifiBBartifacts)); + } return true; } catch (FileNotFoundException ex) { logger.log(Level.SEVERE, "Error finding the registry file."); //NON-NLS From fcc356beaa8496f9c1a0e77155d62814735e4f07 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Wed, 24 Oct 2018 23:58:15 -0400 Subject: [PATCH 05/44] CASE_INSENSITIVE flags used. --- .../autopsy/modules/interestingitems/FilesSetRulePanel.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/interestingitems/FilesSetRulePanel.java b/Core/src/org/sleuthkit/autopsy/modules/interestingitems/FilesSetRulePanel.java index 36888279a9..fd3514f68c 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/interestingitems/FilesSetRulePanel.java +++ b/Core/src/org/sleuthkit/autopsy/modules/interestingitems/FilesSetRulePanel.java @@ -435,7 +435,7 @@ final class FilesSetRulePanel extends javax.swing.JPanel { if (!this.nameTextField.getText().isEmpty()) { if (this.nameRegexCheckbox.isSelected()) { try { - Pattern pattern = Pattern.compile(this.nameTextField.getText()); + Pattern pattern = Pattern.compile(this.nameTextField.getText(), Pattern.CASE_INSENSITIVE); if (this.fullNameRadioButton.isSelected()) { condition = new FilesSet.Rule.FullNameCondition(pattern); } else { @@ -520,7 +520,7 @@ final class FilesSetRulePanel extends javax.swing.JPanel { if (!this.pathTextField.getText().isEmpty()) { if (this.pathRegexCheckBox.isSelected()) { try { - condition = new FilesSet.Rule.ParentPathCondition(Pattern.compile(this.pathTextField.getText())); + condition = new FilesSet.Rule.ParentPathCondition(Pattern.compile(this.pathTextField.getText(), Pattern.CASE_INSENSITIVE)); } catch (PatternSyntaxException ex) { logger.log(Level.SEVERE, "Attempt to get malformed path condition", ex); // NON-NLS throw new IllegalStateException("The files set rule panel path condition is not in a valid state"); // NON-NLS From 3765e264319eb22b7fc50de51a07ac5102479d35 Mon Sep 17 00:00:00 2001 From: Brian Carrier Date: Thu, 25 Oct 2018 09:48:45 -0400 Subject: [PATCH 06/44] Make error message more clear --- .../sleuthkit/autopsy/modules/hashdatabase/Bundle.properties | 2 +- .../autopsy/modules/hashdatabase/HashDbIngestModule.java | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/Bundle.properties b/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/Bundle.properties index c461fc193f..11ace044fb 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/Bundle.properties @@ -91,7 +91,7 @@ HashDbImportDatabaseDialog.errorMessage.failedToOpenHashDbMsg=Failed to open has HashLookupModuleFactory.moduleName.text=Hash Lookup HashLookupModuleFactory.moduleDescription.text=Identifies known and notable files using supplied hash sets, such as a standard NSRL hash set. HashDbIngestModule.fileReadErrorMsg=Read Error\: {0} -HashDbIngestModule.calcHashValueErr=Error encountered while calculating the hash value for {0}. +HashDbIngestModule.calcHashValueErr=Error encountered while calculating the hash value for {0} ({1}). HashDbIngestModule.hashLookupErrorMsg=Hash Lookup Error\: {0} HashDbIngestModule.settingKnownBadStateErr=Error encountered while setting notable state for {0}. HashDbIngestModule.lookingUpKnownBadHashValueErr=Error encountered while looking up notable hash value for {0}. diff --git a/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/HashDbIngestModule.java b/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/HashDbIngestModule.java index ee3ba1c089..23749b8cba 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/HashDbIngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/modules/hashdatabase/HashDbIngestModule.java @@ -229,7 +229,9 @@ public class HashDbIngestModule implements FileIngestModule { services.postMessage(IngestMessage.createErrorMessage( HashLookupModuleFactory.getModuleName(), NbBundle.getMessage(this.getClass(), "HashDbIngestModule.fileReadErrorMsg", name), - NbBundle.getMessage(this.getClass(), "HashDbIngestModule.calcHashValueErr", name))); + NbBundle.getMessage(this.getClass(), "HashDbIngestModule.calcHashValueErr", + file.getParentPath() + file.getName(), + file.isMetaFlagSet(TskData.TSK_FS_META_FLAG_ENUM.ALLOC)?"Allocated File" : "Deleted File"))); return ProcessResult.ERROR; } } From ba84b4c74bead012662cdd62dd310239835f4e99 Mon Sep 17 00:00:00 2001 From: Ann Priestman Date: Thu, 25 Oct 2018 12:27:54 -0400 Subject: [PATCH 07/44] Prepend the archive name to any images extracted in the archive DSP. Don't use the device ID for the name in the Local Files DSP. --- .../autopsy/casemodule/LocalFilesDSProcessor.java | 2 +- .../experimental/autoingest/AddArchiveTask.java | 12 +++++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/LocalFilesDSProcessor.java b/Core/src/org/sleuthkit/autopsy/casemodule/LocalFilesDSProcessor.java index 8d6dcffdd9..4d5e3eac36 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/LocalFilesDSProcessor.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/LocalFilesDSProcessor.java @@ -369,7 +369,7 @@ public class LocalFilesDSProcessor implements DataSourceProcessor, AutoIngestDat @Override public void process(String deviceId, Path dataSourcePath, DataSourceProcessorProgressMonitor progressMonitor, DataSourceProcessorCallback callBack) { List filePaths = Arrays.asList(new String[]{dataSourcePath.toString()}); - run(deviceId, deviceId, filePaths, progressMonitor, callBack); + run(deviceId, "", filePaths, progressMonitor, callBack); } /** diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AddArchiveTask.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AddArchiveTask.java index e62c58343c..9d6fddd1fe 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AddArchiveTask.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AddArchiveTask.java @@ -41,6 +41,7 @@ import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.datasourceprocessors.AutoIngestDataSourceProcessor; import org.sleuthkit.autopsy.coreutils.TimeStampUtils; import org.sleuthkit.datamodel.Content; +import org.sleuthkit.datamodel.DataSource; /* * A runnable that adds an archive data source as well as data sources contained @@ -195,9 +196,18 @@ class AddArchiveTask implements Runnable { continue; } - // if we are here it means the data source was addedd successfully + // if we are here it means the data source was added successfully success = true; newDataSources.addAll(internalDataSource.getContent()); + + // Update the names for all new data sources to be the root archive plus the name of the data source + for (Content c:internalDataSource.getContent()) { + if (c instanceof DataSource) { + DataSource ds = (DataSource) c; + String newName = Paths.get(archivePath).getFileName() + "/" + ds.getName(); + ds.setDisplayName(currentCase.getSleuthkitCase(), newName); + } + } // skip all other DSPs for this data source break; From 0e8d2ce27ff9d0b69d1e0bddbc79ae216a5f09a6 Mon Sep 17 00:00:00 2001 From: Brian Carrier Date: Thu, 25 Oct 2018 15:52:22 -0400 Subject: [PATCH 08/44] Invalidate settings even when a file is updated - JIRA 1109 --- .../datamodel/grouping/DrawableGroup.java | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/DrawableGroup.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/DrawableGroup.java index bdb54c448f..5469afe4d3 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/DrawableGroup.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/DrawableGroup.java @@ -68,13 +68,6 @@ public class DrawableGroup implements Comparable { DrawableGroup(GroupKey groupKey, Set filesInGroup, boolean seen) { this.groupKey = groupKey; this.fileIDs.setAll(filesInGroup); - fileIDs.addListener((ListChangeListener.Change listchange) -> { - boolean seenChanged = false; - while (false == seenChanged && listchange.next()) { - seenChanged |= listchange.wasAdded(); - } - invalidateProperties(seenChanged); - }); this.seen.set(seen); } @@ -183,15 +176,21 @@ public class DrawableGroup implements Comparable { if (fileIDs.contains(f) == false) { fileIDs.add(f); } + // invalidate no matter what because the file could have new hash hits, etc. + invalidateProperties(true); } synchronized void setFiles(Set newFileIds) { fileIDs.removeIf(fileID -> newFileIds.contains(fileID) == false); + invalidateProperties(false); newFileIds.stream().forEach(this::addFile); } synchronized void removeFile(Long f) { - fileIDs.removeAll(f); + if (fileIDs.contains(f)) { + fileIDs.removeAll(f); + invalidateProperties(false); + } } private void invalidateProperties(boolean seenChanged) { From b4c8abad8f83447a9dfd822a373d8991bf615897 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Thu, 25 Oct 2018 16:02:39 -0400 Subject: [PATCH 09/44] 4305 allow correlation attrs to be added when ingest not running --- .../eventlisteners/IngestEventsListener.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java index ccca659574..cdbf490e9e 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java @@ -183,7 +183,10 @@ public class IngestEventsListener { @Override public void propertyChange(PropertyChangeEvent evt) { - if (getCeModuleInstanceCount() > 0) { + //if ingest is running we want there to check if there is a Correlation Engine module running + //sometimes artifacts are generated by DSPs or other sources while ingest is not running + //in these cases we still want to create correlation attributes for those artifacts when appropriate + if (!IngestManager.getInstance().isIngestRunning() || getCeModuleInstanceCount() > 0) { EamDb dbManager; try { dbManager = EamDb.getInstance(); @@ -197,7 +200,7 @@ public class IngestEventsListener { break; } } - } + } } } From af882afd424f26fa712882e20dec2a9bd9329e46 Mon Sep 17 00:00:00 2001 From: Ann Priestman Date: Fri, 26 Oct 2018 08:55:13 -0400 Subject: [PATCH 10/44] Don't pass in sleuthkit case --- .../autopsy/experimental/autoingest/AddArchiveTask.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AddArchiveTask.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AddArchiveTask.java index 9d6fddd1fe..2275b63d5f 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AddArchiveTask.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AddArchiveTask.java @@ -205,7 +205,7 @@ class AddArchiveTask implements Runnable { if (c instanceof DataSource) { DataSource ds = (DataSource) c; String newName = Paths.get(archivePath).getFileName() + "/" + ds.getName(); - ds.setDisplayName(currentCase.getSleuthkitCase(), newName); + ds.setDisplayName(newName); } } From 5ec17adf8e0943176f5f26766496d89d4f3b4cfb Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Mon, 29 Oct 2018 12:30:46 -0400 Subject: [PATCH 11/44] 4305 add icon for tsk_wifi_network artifacts --- .../autopsy/datamodel/ExtractedContent.java | 2 ++ .../org/sleuthkit/autopsy/images/network-wifi.png | Bin 0 -> 453 bytes .../org/sleuthkit/autopsy/report/ReportHTML.java | 2 ++ 3 files changed, 4 insertions(+) create mode 100644 Core/src/org/sleuthkit/autopsy/images/network-wifi.png diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/ExtractedContent.java b/Core/src/org/sleuthkit/autopsy/datamodel/ExtractedContent.java index 3ab5f6cb90..e950d0e870 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/ExtractedContent.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/ExtractedContent.java @@ -156,6 +156,8 @@ public class ExtractedContent implements AutopsyVisitableItem { return filePath + "drive_network.png"; //NON-NLS } else if (typeID == BlackboardArtifact.ARTIFACT_TYPE.TSK_FACE_DETECTED.getTypeID()) { return filePath + "face.png"; //NON-NLS + } else if (typeID == BlackboardArtifact.ARTIFACT_TYPE.TSK_WIFI_NETWORK.getTypeID()) { + return filePath + "network-wifi.png"; //NON-NLS } return filePath + "artifact-icon.png"; //NON-NLS } diff --git a/Core/src/org/sleuthkit/autopsy/images/network-wifi.png b/Core/src/org/sleuthkit/autopsy/images/network-wifi.png new file mode 100644 index 0000000000000000000000000000000000000000..5d7cb76ca165db10f3df4f8b07813eee8b335a7c GIT binary patch literal 453 zcmV;$0XqJPP)xN5fD!z_WZTrMgZ%0MZJdrZ`nVY0$CtoPti%e>BfZr& v_5MM;>Ij(+V Date: Mon, 29 Oct 2018 12:40:23 -0400 Subject: [PATCH 12/44] 4305 remove partial code for adding imei and mac address correlation attrs --- .../datamodel/CorrelationAttributeInstance.java | 8 +------- .../datamodel/CorrelationAttributeNormalizer.java | 2 -- .../centralrepository/datamodel/EamArtifactUtil.java | 7 ------- 3 files changed, 1 insertion(+), 16 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeInstance.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeInstance.java index eb790f866b..20845ff447 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeInstance.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeInstance.java @@ -247,8 +247,6 @@ public class CorrelationAttributeInstance implements Serializable { public static final int PHONE_TYPE_ID = 3; public static final int USBID_TYPE_ID = 4; public static final int SSID_TYPE_ID = 5; - public static final int MAC_TYPE_ID = 6; - public static final int IMEI_TYPE_ID = 7; /** * Load the default correlation types @@ -261,9 +259,7 @@ public class CorrelationAttributeInstance implements Serializable { "CorrelationType.EMAIL.displayName=Email Addresses", "CorrelationType.PHONE.displayName=Phone Numbers", "CorrelationType.USBID.displayName=USB Devices", - "CorrelationType.SSID.displayName=Wireless Networks", - "CorrelationType.MAC.displayName=MAC Address", - "CorrelationType.IMEI.displayName=IMEI Number"}) + "CorrelationType.SSID.displayName=Wireless Networks"}) public static List getDefaultCorrelationTypes() throws EamDbException { List DEFAULT_CORRELATION_TYPES = new ArrayList<>(); DEFAULT_CORRELATION_TYPES.add(new CorrelationAttributeInstance.Type(FILES_TYPE_ID, Bundle.CorrelationType_FILES_displayName(), "file", true, true)); // NON-NLS @@ -272,8 +268,6 @@ public class CorrelationAttributeInstance implements Serializable { DEFAULT_CORRELATION_TYPES.add(new CorrelationAttributeInstance.Type(PHONE_TYPE_ID, Bundle.CorrelationType_PHONE_displayName(), "phone_number", true, true)); // NON-NLS DEFAULT_CORRELATION_TYPES.add(new CorrelationAttributeInstance.Type(USBID_TYPE_ID, Bundle.CorrelationType_USBID_displayName(), "usb_devices", true, true)); // NON-NLS DEFAULT_CORRELATION_TYPES.add(new CorrelationAttributeInstance.Type(SSID_TYPE_ID, Bundle.CorrelationType_SSID_displayName(), "wireless_networks", true, true)); // NON-NLS - DEFAULT_CORRELATION_TYPES.add(new CorrelationAttributeInstance.Type(MAC_TYPE_ID, Bundle.CorrelationType_MAC_displayName(), "mac_address", true, true)); // NON-NLS - DEFAULT_CORRELATION_TYPES.add(new CorrelationAttributeInstance.Type(IMEI_TYPE_ID, Bundle.CorrelationType_IMEI_displayName(), "imei_number", true, true)); // NON-NLS return DEFAULT_CORRELATION_TYPES; } diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeNormalizer.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeNormalizer.java index c7787389f4..4ce04769c8 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeNormalizer.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeNormalizer.java @@ -64,8 +64,6 @@ final public class CorrelationAttributeNormalizer { case CorrelationAttributeInstance.USBID_TYPE_ID: return normalizeUsbId(data); case CorrelationAttributeInstance.SSID_TYPE_ID: - case CorrelationAttributeInstance.MAC_TYPE_ID: - case CorrelationAttributeInstance.IMEI_TYPE_ID: return data; default: final String errorMessage = String.format( diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactUtil.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactUtil.java index c3b18e62f5..2477d0fc41 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactUtil.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactUtil.java @@ -159,7 +159,6 @@ public class EamArtifactUtil { return null; } } - } else if (correlationType.getId() == CorrelationAttributeInstance.USBID_TYPE_ID && BlackboardArtifact.ARTIFACT_TYPE.TSK_DEVICE_ATTACHED.getTypeID() == artifactTypeID) { @@ -169,12 +168,6 @@ public class EamArtifactUtil { && BlackboardArtifact.ARTIFACT_TYPE.TSK_WIFI_NETWORK.getTypeID() == artifactTypeID) { value = bbArtifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SSID)).getValueString(); } -// else if (correlationType.getId() == CorrelationAttributeInstance.MAC_TYPE_ID) { -// value = bbArtifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_BSSID)).getValueString(); -// } else if (correlationType.getId() == CorrelationAttributeInstance.IMEI_TYPE_ID) { -// value = bbArtifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_BSSID)).getValueString(); -// } - } catch (TskCoreException ex) { logger.log(Level.SEVERE, "Error getting attribute while getting type from BlackboardArtifact.", ex); // NON-NLS return null; From 6ae37630a432f4bb31c8f122294182053f1e3418 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Mon, 29 Oct 2018 15:41:59 -0400 Subject: [PATCH 13/44] Fixed codacy suggestions --- .../embeddedfileextractor/SevenZipExtractor.java | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java index c8f0fce062..dc4f190c09 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java +++ b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java @@ -800,12 +800,7 @@ class SevenZipExtractor { MutableEncodedFileOutputStream(String localAbsPath) throws IOException { super(new FileOutputStream(localAbsPath), TskData.EncodingType.XOR1); } - - @Override - public void write(byte[] bytes) throws IOException { - super.write(bytes); - } - + /** * Update the OutputStream to point to a new File. This method mutates the state so * that there is no overhead of creating a new object and buffer. Additionally, the @@ -837,6 +832,14 @@ class SevenZipExtractor { } } + /** + * UnpackStream used by the SevenZipBindings to do archive extraction. A memory + * leak exists in the SevenZip library that will not let go of the Streams until + * the entire archive extraction is complete. So, to compensate this UnpackStream + * uses a MutableEncodedFileOutputStream, which supports setting a new disk + * location for the contents to be written to (as opposed to creating a new steam + * for each file). + */ private final static class UnpackStream implements ISequentialOutStream { private final MutableEncodedFileOutputStream output; From 82aa5a063923218902f732d1816c19b376e80075 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Mon, 29 Oct 2018 16:00:46 -0400 Subject: [PATCH 14/44] 4305 CR upgrade code for wireless_networks correlation attr --- .../datamodel/AbstractSqlEamDb.java | 47 +++++++++++ .../centralrepository/datamodel/EamDb.java | 4 +- .../datamodel/PostgresEamDbSettings.java | 78 +++++++++++------ .../datamodel/SqliteEamDbSettings.java | 83 ++++++++++++------- 4 files changed, 155 insertions(+), 57 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java index 68e5b4f8c2..b14d9043af 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java @@ -3085,6 +3085,7 @@ abstract class AbstractSqlEamDb implements EamDb { ResultSet resultSet = null; Statement statement = null; + PreparedStatement preparedStatement = null; Connection conn = null; try { @@ -3131,7 +3132,52 @@ abstract class AbstractSqlEamDb implements EamDb { // regardless of whether this succeeds. EamDbUtil.insertDefaultOrganization(conn); } + if (dbSchemaVersion.compareTo(new CaseDbSchemaVersionNumber(1, 2)) < 0) { + //update central repository to be able to store new correlation attributes + EamDbPlatformEnum selectedPlatform = EamDbPlatformEnum.getSelectedPlatform(); + final String addSsidTableTemplate; + final String addCaseIdIndexTemplate; + final String addDataSourceIdIndexTemplate; + final String addValueIndexTemplate; + final String addKnownStatusIndexTemplate; + switch (selectedPlatform) { + case POSTGRESQL: + //create table //settings.getStatementTemplate then statement.execute here with the added table + //add correlation attr to correlation_attrs table + addSsidTableTemplate = PostgresEamDbSettings.getCreateArtifactInstancesTableTemplate(); + addCaseIdIndexTemplate = PostgresEamDbSettings.getAddCaseIdIndexTemplate(); + addDataSourceIdIndexTemplate = PostgresEamDbSettings.getAddDataSourceIdIndexTemplate(); + addValueIndexTemplate = PostgresEamDbSettings.getAddValueIndexTemplate(); + addKnownStatusIndexTemplate = PostgresEamDbSettings.getAddKnownStatusIndexTemplate(); + break; + case SQLITE: + addSsidTableTemplate = SqliteEamDbSettings.getCreateArtifactInstancesTableTemplate(); + addCaseIdIndexTemplate = SqliteEamDbSettings.getAddCaseIdIndexTemplate(); + addDataSourceIdIndexTemplate = SqliteEamDbSettings.getAddDataSourceIdIndexTemplate(); + addValueIndexTemplate = SqliteEamDbSettings.getAddValueIndexTemplate(); + addKnownStatusIndexTemplate = SqliteEamDbSettings.getAddKnownStatusIndexTemplate(); + break; + default: + throw new EamDbException("Currently selected database platform \"" + selectedPlatform.name() + "\" can not be upgraded."); + } + final String wirelessNetworsDbTableName = "wireless_networks"; + final String wirelessNetworksTableInstanceName = wirelessNetworsDbTableName + "_instances"; + final String addAttributeSql = "INSERT INTO correlation_types(id, display_name, db_table_name, supported, enabled) VALUES (?, ?, ?, ?, ?)"; + preparedStatement = conn.prepareStatement(addAttributeSql); + preparedStatement.setInt(1, CorrelationAttributeInstance.SSID_TYPE_ID); + preparedStatement.setString(2, Bundle.CorrelationType_SSID_displayName()); + preparedStatement.setString(3, wirelessNetworsDbTableName); + preparedStatement.setInt(4, 1); + preparedStatement.setInt(5, 1); + preparedStatement.execute(); + statement.execute(String.format(addSsidTableTemplate, wirelessNetworksTableInstanceName, wirelessNetworksTableInstanceName)); + statement.execute(String.format(addCaseIdIndexTemplate, wirelessNetworksTableInstanceName, wirelessNetworksTableInstanceName)); + statement.execute(String.format(addDataSourceIdIndexTemplate, wirelessNetworksTableInstanceName, wirelessNetworksTableInstanceName)); + statement.execute(String.format(addValueIndexTemplate, wirelessNetworksTableInstanceName, wirelessNetworksTableInstanceName)); + statement.execute(String.format(addKnownStatusIndexTemplate, wirelessNetworksTableInstanceName, wirelessNetworksTableInstanceName)); + + } if (!updateSchemaVersion(conn)) { throw new EamDbException("Error updating schema version"); } @@ -3149,6 +3195,7 @@ abstract class AbstractSqlEamDb implements EamDb { throw ex; } finally { EamDbUtil.closeResultSet(resultSet); + EamDbUtil.closeStatement(preparedStatement); EamDbUtil.closeStatement(statement); EamDbUtil.closeConnection(conn); } diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDb.java index c7e385928d..418181d8ab 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDb.java @@ -31,9 +31,9 @@ import org.sleuthkit.datamodel.CaseDbSchemaVersionNumber; */ public interface EamDb { - public static final int SCHEMA_VERSION = 1; + public static final int SCHEMA_VERSION = 2; public static final CaseDbSchemaVersionNumber CURRENT_DB_SCHEMA_VERSION - = new CaseDbSchemaVersionNumber(1, 1); + = new CaseDbSchemaVersionNumber(1, 2); /** diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDbSettings.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDbSettings.java index 77ab8c23db..adaf7c0d9c 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDbSettings.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDbSettings.java @@ -35,8 +35,9 @@ import org.sleuthkit.autopsy.coreutils.TextConverterException; /** * Settings for the Postgres implementation of the Central Repository database - * - * NOTE: This is public scope because the options panel calls it directly to set/get + * + * NOTE: This is public scope because the options panel calls it directly to + * set/get */ public final class PostgresEamDbSettings { @@ -266,7 +267,7 @@ public final class PostgresEamDbSettings { return true; } - + public boolean deleteDatabase() { Connection conn = getEphemeralConnection(true); if (null == conn) { @@ -391,26 +392,13 @@ public final class PostgresEamDbSettings { createCorrelationTypesTable.append("CONSTRAINT correlation_types_names UNIQUE (display_name, db_table_name)"); createCorrelationTypesTable.append(")"); - // Each "%s" will be replaced with the relevant TYPE_instances table name. - StringBuilder createArtifactInstancesTableTemplate = new StringBuilder(); - createArtifactInstancesTableTemplate.append("CREATE TABLE IF NOT EXISTS %s ("); - createArtifactInstancesTableTemplate.append("id SERIAL PRIMARY KEY,"); - createArtifactInstancesTableTemplate.append("case_id integer NOT NULL,"); - createArtifactInstancesTableTemplate.append("data_source_id integer NOT NULL,"); - createArtifactInstancesTableTemplate.append("value text NOT NULL,"); - createArtifactInstancesTableTemplate.append("file_path text NOT NULL,"); - createArtifactInstancesTableTemplate.append("known_status integer NOT NULL,"); - createArtifactInstancesTableTemplate.append("comment text,"); - createArtifactInstancesTableTemplate.append("CONSTRAINT %s_multi_unique_ UNIQUE (data_source_id, value, file_path),"); - createArtifactInstancesTableTemplate.append("foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL,"); - createArtifactInstancesTableTemplate.append("foreign key (data_source_id) references data_sources(id) ON UPDATE SET NULL ON DELETE SET NULL"); - createArtifactInstancesTableTemplate.append(")"); + String createArtifactInstancesTableTemplate = getCreateArtifactInstancesTableTemplate(); - // Each "%s" will be replaced with the relevant TYPE_instances table name. - String instancesIdx1 = "CREATE INDEX IF NOT EXISTS %s_case_id ON %s (case_id)"; - String instancesIdx2 = "CREATE INDEX IF NOT EXISTS %s_data_source_id ON %s (data_source_id)"; - String instancesIdx3 = "CREATE INDEX IF NOT EXISTS %s_value ON %s (value)"; - String instancesIdx4 = "CREATE INDEX IF NOT EXISTS %s_value_known_status ON %s (value, known_status)"; + String instancesIdx1 = getAddCaseIdIndexTemplate(); + String instancesIdx2 = getAddDataSourceIdIndexTemplate(); + + String instancesIdx3 = getAddValueIndexTemplate(); + String instancesIdx4 = getAddKnownStatusIndexTemplate(); StringBuilder createDbInfoTable = new StringBuilder(); createDbInfoTable.append("CREATE TABLE IF NOT EXISTS db_info ("); @@ -447,14 +435,14 @@ public final class PostgresEamDbSettings { // Create a separate instance and reference table for each correlation type List DEFAULT_CORRELATION_TYPES = CorrelationAttributeInstance.getDefaultCorrelationTypes(); - + String reference_type_dbname; String instance_type_dbname; for (CorrelationAttributeInstance.Type type : DEFAULT_CORRELATION_TYPES) { reference_type_dbname = EamDbUtil.correlationTypeToReferenceTableName(type); instance_type_dbname = EamDbUtil.correlationTypeToInstanceTableName(type); - - stmt.execute(String.format(createArtifactInstancesTableTemplate.toString(), instance_type_dbname, instance_type_dbname)); + + stmt.execute(String.format(createArtifactInstancesTableTemplate, instance_type_dbname, instance_type_dbname)); stmt.execute(String.format(instancesIdx1, instance_type_dbname, instance_type_dbname)); stmt.execute(String.format(instancesIdx2, instance_type_dbname, instance_type_dbname)); stmt.execute(String.format(instancesIdx3, instance_type_dbname, instance_type_dbname)); @@ -465,7 +453,7 @@ public final class PostgresEamDbSettings { stmt.execute(String.format(createReferenceTypesTableTemplate.toString(), reference_type_dbname, reference_type_dbname)); stmt.execute(String.format(referenceTypesIdx1, reference_type_dbname, reference_type_dbname)); stmt.execute(String.format(referenceTypesIdx2, reference_type_dbname, reference_type_dbname)); - } + } } } catch (SQLException ex) { @@ -480,6 +468,44 @@ public final class PostgresEamDbSettings { return true; } + static String getCreateArtifactInstancesTableTemplate() { + // Each "%s" will be replaced with the relevant TYPE_instances table name. + StringBuilder createArtifactInstancesTableTemplate = new StringBuilder(); + createArtifactInstancesTableTemplate.append("CREATE TABLE IF NOT EXISTS %s ("); + createArtifactInstancesTableTemplate.append("id SERIAL PRIMARY KEY,"); + createArtifactInstancesTableTemplate.append("case_id integer NOT NULL,"); + createArtifactInstancesTableTemplate.append("data_source_id integer NOT NULL,"); + createArtifactInstancesTableTemplate.append("value text NOT NULL,"); + createArtifactInstancesTableTemplate.append("file_path text NOT NULL,"); + createArtifactInstancesTableTemplate.append("known_status integer NOT NULL,"); + createArtifactInstancesTableTemplate.append("comment text,"); + createArtifactInstancesTableTemplate.append("CONSTRAINT %s_multi_unique_ UNIQUE (data_source_id, value, file_path),"); + createArtifactInstancesTableTemplate.append("foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL,"); + createArtifactInstancesTableTemplate.append("foreign key (data_source_id) references data_sources(id) ON UPDATE SET NULL ON DELETE SET NULL"); + createArtifactInstancesTableTemplate.append(")"); + return createArtifactInstancesTableTemplate.toString(); + } + + static String getAddCaseIdIndexTemplate() { + // Each "%s" will be replaced with the relevant TYPE_instances table name. + return "CREATE INDEX IF NOT EXISTS %s_case_id ON %s (case_id)"; + } + + static String getAddDataSourceIdIndexTemplate() { + // Each "%s" will be replaced with the relevant TYPE_instances table name. + return "CREATE INDEX IF NOT EXISTS %s_data_source_id ON %s (data_source_id)"; + } + + static String getAddValueIndexTemplate() { + // Each "%s" will be replaced with the relevant TYPE_instances table name. + return "CREATE INDEX IF NOT EXISTS %s_value ON %s (value)"; + } + + static String getAddKnownStatusIndexTemplate() { + // Each "%s" will be replaced with the relevant TYPE_instances table name. + return "CREATE INDEX IF NOT EXISTS %s_value_known_status ON %s (value, known_status)"; + } + public boolean insertDefaultDatabaseContent() { Connection conn = getEphemeralConnection(false); if (null == conn) { diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDbSettings.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDbSettings.java index 4894a570e6..c321e3b412 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDbSettings.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDbSettings.java @@ -35,8 +35,9 @@ import org.sleuthkit.autopsy.coreutils.PlatformUtil; /** * Settings for the sqlite implementation of the Central Repository database - * - * NOTE: This is public scope because the options panel calls it directly to set/get + * + * NOTE: This is public scope because the options panel calls it directly to + * set/get */ public final class SqliteEamDbSettings { @@ -95,7 +96,7 @@ public final class SqliteEamDbSettings { ModuleSettings.setConfigSetting("CentralRepository", "db.sqlite.dbDirectory", getDbDirectory()); // NON-NLS ModuleSettings.setConfigSetting("CentralRepository", "db.sqlite.bulkThreshold", Integer.toString(getBulkThreshold())); // NON-NLS } - + /** * Verify that the db file exists. * @@ -103,11 +104,11 @@ public final class SqliteEamDbSettings { */ public boolean dbFileExists() { File dbFile = new File(getFileNameWithPath()); - if(! dbFile.exists()){ + if (!dbFile.exists()) { return false; } // It's unlikely, but make sure the file isn't actually a directory - return ( ! dbFile.isDirectory()); + return (!dbFile.isDirectory()); } /** @@ -148,10 +149,11 @@ public final class SqliteEamDbSettings { return true; } - + /** * Delete the database - * @return + * + * @return */ public boolean deleteDatabase() { File dbFile = new File(this.getFileNameWithPath()); @@ -333,26 +335,13 @@ public final class SqliteEamDbSettings { createCorrelationTypesTable.append("CONSTRAINT correlation_types_names UNIQUE (display_name, db_table_name)"); createCorrelationTypesTable.append(")"); - // Each "%s" will be replaced with the relevant TYPE_instances table name. - StringBuilder createArtifactInstancesTableTemplate = new StringBuilder(); - createArtifactInstancesTableTemplate.append("CREATE TABLE IF NOT EXISTS %s ("); - createArtifactInstancesTableTemplate.append("id integer primary key autoincrement NOT NULL,"); - createArtifactInstancesTableTemplate.append("case_id integer NOT NULL,"); - createArtifactInstancesTableTemplate.append("data_source_id integer NOT NULL,"); - createArtifactInstancesTableTemplate.append("value text NOT NULL,"); - createArtifactInstancesTableTemplate.append("file_path text NOT NULL,"); - createArtifactInstancesTableTemplate.append("known_status integer NOT NULL,"); - createArtifactInstancesTableTemplate.append("comment text,"); - createArtifactInstancesTableTemplate.append("CONSTRAINT %s_multi_unique UNIQUE(data_source_id, value, file_path) ON CONFLICT IGNORE,"); - createArtifactInstancesTableTemplate.append("foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL,"); - createArtifactInstancesTableTemplate.append("foreign key (data_source_id) references data_sources(id) ON UPDATE SET NULL ON DELETE SET NULL"); - createArtifactInstancesTableTemplate.append(")"); + String createArtifactInstancesTableTemplate = getCreateArtifactInstancesTableTemplate(); - // Each "%s" will be replaced with the relevant TYPE_instances table name. - String instancesIdx1 = "CREATE INDEX IF NOT EXISTS %s_case_id ON %s (case_id)"; - String instancesIdx2 = "CREATE INDEX IF NOT EXISTS %s_data_source_id ON %s (data_source_id)"; - String instancesIdx3 = "CREATE INDEX IF NOT EXISTS %s_value ON %s (value)"; - String instancesIdx4 = "CREATE INDEX IF NOT EXISTS %s_value_known_status ON %s (value, known_status)"; + String instancesIdx1 = getAddCaseIdIndexTemplate(); + String instancesIdx2 = getAddDataSourceIdIndexTemplate(); + + String instancesIdx3 = getAddValueIndexTemplate(); + String instancesIdx4 = getAddKnownStatusIndexTemplate(); StringBuilder createDbInfoTable = new StringBuilder(); createDbInfoTable.append("CREATE TABLE IF NOT EXISTS db_info ("); @@ -402,7 +391,7 @@ public final class SqliteEamDbSettings { reference_type_dbname = EamDbUtil.correlationTypeToReferenceTableName(type); instance_type_dbname = EamDbUtil.correlationTypeToInstanceTableName(type); - stmt.execute(String.format(createArtifactInstancesTableTemplate.toString(), instance_type_dbname, instance_type_dbname)); + stmt.execute(String.format(createArtifactInstancesTableTemplate, instance_type_dbname, instance_type_dbname)); stmt.execute(String.format(instancesIdx1, instance_type_dbname, instance_type_dbname)); stmt.execute(String.format(instancesIdx2, instance_type_dbname, instance_type_dbname)); stmt.execute(String.format(instancesIdx3, instance_type_dbname, instance_type_dbname)); @@ -427,6 +416,44 @@ public final class SqliteEamDbSettings { return true; } + static String getCreateArtifactInstancesTableTemplate() { + // Each "%s" will be replaced with the relevant TYPE_instances table name. + StringBuilder createArtifactInstancesTableTemplate = new StringBuilder(); + createArtifactInstancesTableTemplate.append("CREATE TABLE IF NOT EXISTS %s ("); + createArtifactInstancesTableTemplate.append("id integer primary key autoincrement NOT NULL,"); + createArtifactInstancesTableTemplate.append("case_id integer NOT NULL,"); + createArtifactInstancesTableTemplate.append("data_source_id integer NOT NULL,"); + createArtifactInstancesTableTemplate.append("value text NOT NULL,"); + createArtifactInstancesTableTemplate.append("file_path text NOT NULL,"); + createArtifactInstancesTableTemplate.append("known_status integer NOT NULL,"); + createArtifactInstancesTableTemplate.append("comment text,"); + createArtifactInstancesTableTemplate.append("CONSTRAINT %s_multi_unique UNIQUE(data_source_id, value, file_path) ON CONFLICT IGNORE,"); + createArtifactInstancesTableTemplate.append("foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL,"); + createArtifactInstancesTableTemplate.append("foreign key (data_source_id) references data_sources(id) ON UPDATE SET NULL ON DELETE SET NULL"); + createArtifactInstancesTableTemplate.append(")"); + return createArtifactInstancesTableTemplate.toString(); + } + + static String getAddCaseIdIndexTemplate() { + // Each "%s" will be replaced with the relevant TYPE_instances table name. + return "CREATE INDEX IF NOT EXISTS %s_case_id ON %s (case_id)"; + } + + static String getAddDataSourceIdIndexTemplate() { + // Each "%s" will be replaced with the relevant TYPE_instances table name. + return "CREATE INDEX IF NOT EXISTS %s_data_source_id ON %s (data_source_id)"; + } + + static String getAddValueIndexTemplate() { + // Each "%s" will be replaced with the relevant TYPE_instances table name. + return "CREATE INDEX IF NOT EXISTS %s_value ON %s (value)"; + } + + static String getAddKnownStatusIndexTemplate() { + // Each "%s" will be replaced with the relevant TYPE_instances table name. + return "CREATE INDEX IF NOT EXISTS %s_value_known_status ON %s (value, known_status)"; + } + public boolean insertDefaultDatabaseContent() { Connection conn = getEphemeralConnection(); if (null == conn) { @@ -490,8 +517,6 @@ public final class SqliteEamDbSettings { } } - - /** * @return the dbDirectory */ From e4142387929817e2ad47810ea2e047a1ca333045 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Tue, 30 Oct 2018 08:39:03 -0400 Subject: [PATCH 15/44] This import keeps getting automatically added and its causing Codacy to complain --- .../autopsy/modules/embeddedfileextractor/SevenZipExtractor.java | 1 - 1 file changed, 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java index dc4f190c09..0deea2baf0 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java +++ b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java @@ -43,7 +43,6 @@ import net.sf.sevenzipjbinding.IArchiveExtractCallback; import net.sf.sevenzipjbinding.ICryptoGetTextPassword; import net.sf.sevenzipjbinding.PropID; import org.netbeans.api.progress.ProgressHandle; -import org.openide.util.Exceptions; import org.openide.util.NbBundle; import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.casemodule.Case; From b5c5c64970b74bf3459517226fa3c9430c3aae5f Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Tue, 30 Oct 2018 11:22:23 -0400 Subject: [PATCH 16/44] 4305 correct logic for enabling other occurances content viewer --- .../DataContentViewerOtherCases.java | 56 +++++++++---------- 1 file changed, 25 insertions(+), 31 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/contentviewer/DataContentViewerOtherCases.java b/Core/src/org/sleuthkit/autopsy/centralrepository/contentviewer/DataContentViewerOtherCases.java index c6301bfb89..0080ade4fb 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/contentviewer/DataContentViewerOtherCases.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/contentviewer/DataContentViewerOtherCases.java @@ -127,7 +127,7 @@ public class DataContentViewerOtherCases extends JPanel implements DataContentVi } } else if (jmi.equals(showCommonalityMenuItem)) { showCommonalityDetails(); - } + } } }; @@ -419,7 +419,7 @@ public class DataContentViewerOtherCases extends JPanel implements DataContentVi } // we can correlate based on the MD5 if it is enabled - if (this.file != null && EamDb.isEnabled()) { + if (this.file != null && EamDb.isEnabled() && this.file.getSize() > 0) { try { List artifactTypes = EamDb.getInstance().getDefinedCorrelationTypes(); @@ -447,27 +447,23 @@ public class DataContentViewerOtherCases extends JPanel implements DataContentVi } catch (EamDbException | TskCoreException ex) { LOGGER.log(Level.SEVERE, "Error connecting to DB", ex); // NON-NLS } + // If EamDb not enabled, get the Files default correlation type to allow Other Occurances to be enabled. + } else if (this.file != null && this.file.getSize() > 0) { + String md5 = this.file.getMd5Hash(); + if (md5 != null && !md5.isEmpty()) { + try { + final CorrelationAttributeInstance.Type fileAttributeType + = CorrelationAttributeInstance.getDefaultCorrelationTypes() + .stream() + .filter(attrType -> attrType.getId() == CorrelationAttributeInstance.FILES_TYPE_ID) + .findAny() + .get(); - } else { - - // If EamDb not enabled, get the Files default correlation type to allow Other Occurances to be enabled. - if (this.file != null) { - String md5 = this.file.getMd5Hash(); - if (md5 != null && !md5.isEmpty()) { - try { - final CorrelationAttributeInstance.Type fileAttributeType - = CorrelationAttributeInstance.getDefaultCorrelationTypes() - .stream() - .filter(attrType -> attrType.getId() == CorrelationAttributeInstance.FILES_TYPE_ID) - .findAny() - .get(); - - ret.add(new CorrelationAttributeInstance(fileAttributeType, md5)); - } catch (EamDbException ex) { - LOGGER.log(Level.SEVERE, "Error connecting to DB", ex); // NON-NLS - } catch (CorrelationAttributeNormalizationException ex) { - LOGGER.log(Level.INFO, String.format("Unable to create CorrelationAttributeInstance for value %s", md5), ex); // NON-NLS - } + ret.add(new CorrelationAttributeInstance(fileAttributeType, md5)); + } catch (EamDbException ex) { + LOGGER.log(Level.SEVERE, "Error connecting to DB", ex); // NON-NLS + } catch (CorrelationAttributeNormalizationException ex) { + LOGGER.log(Level.INFO, String.format("Unable to create CorrelationAttributeInstance for value %s", md5), ex); // NON-NLS } } } @@ -515,9 +511,9 @@ public class DataContentViewerOtherCases extends JPanel implements DataContentVi * artifact. If the central repo is not enabled, this will only return files * from the current case with matching MD5 hashes. * - * @param corAttr CorrelationAttribute to query for + * @param corAttr CorrelationAttribute to query for * @param dataSourceName Data source to filter results - * @param deviceId Device Id to filter results + * @param deviceId Device Id to filter results * * @return A collection of correlated artifact instances */ @@ -580,7 +576,7 @@ public class DataContentViewerOtherCases extends JPanel implements DataContentVi * Get all other abstract files in the current case with the same MD5 as the * selected node. * - * @param corAttr The CorrelationAttribute containing the MD5 to search for + * @param corAttr The CorrelationAttribute containing the MD5 to search for * @param openCase The current case * * @return List of matching AbstractFile objects @@ -657,11 +653,9 @@ public class DataContentViewerOtherCases extends JPanel implements DataContentVi // - The central repo is disabled and the backing file has a valid MD5 hash this.file = this.getAbstractFileFromNode(node); if (EamDb.isEnabled()) { - return this.file != null - && this.file.getSize() > 0 - && !getCorrelationAttributesFromNode(node).isEmpty(); + return !getCorrelationAttributesFromNode(node).isEmpty(); } else { - return this.file != null + return this.file != null && this.file.getSize() > 0 && ((this.file.getMd5Hash() != null) && (!this.file.getMd5Hash().isEmpty())); } @@ -733,8 +727,8 @@ public class DataContentViewerOtherCases extends JPanel implements DataContentVi * Adjust a given column for the text provided. * * @param columnIndex The index of the column to adjust. - * @param text The text whose length will be used to adjust the column - * width. + * @param text The text whose length will be used to adjust the + * column width. */ private void setColumnWidthToText(int columnIndex, String text) { TableColumn column = otherCasesTable.getColumnModel().getColumn(columnIndex); From 8576c46f829bb83b38e181e044e61b69b3bcb828 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Tue, 30 Oct 2018 16:00:05 -0400 Subject: [PATCH 17/44] 4305 add comments to new methods regarding getting templates for database statements --- .../datamodel/AbstractSqlEamDb.java | 5 ++- .../datamodel/EamArtifactUtil.java | 7 ++-- .../datamodel/PostgresEamDbSettings.java | 39 ++++++++++++++++++ .../datamodel/SqliteEamDbSettings.java | 41 ++++++++++++++++++- 4 files changed, 85 insertions(+), 7 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java index b14d9043af..4364845648 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java @@ -3140,10 +3140,9 @@ abstract class AbstractSqlEamDb implements EamDb { final String addDataSourceIdIndexTemplate; final String addValueIndexTemplate; final String addKnownStatusIndexTemplate; + //get the data base specific code for creating a new _instance table switch (selectedPlatform) { case POSTGRESQL: - //create table //settings.getStatementTemplate then statement.execute here with the added table - //add correlation attr to correlation_attrs table addSsidTableTemplate = PostgresEamDbSettings.getCreateArtifactInstancesTableTemplate(); addCaseIdIndexTemplate = PostgresEamDbSettings.getAddCaseIdIndexTemplate(); addDataSourceIdIndexTemplate = PostgresEamDbSettings.getAddDataSourceIdIndexTemplate(); @@ -3162,6 +3161,7 @@ abstract class AbstractSqlEamDb implements EamDb { } final String wirelessNetworsDbTableName = "wireless_networks"; final String wirelessNetworksTableInstanceName = wirelessNetworsDbTableName + "_instances"; + //add the wireless_networks attribute to the correlation_types table final String addAttributeSql = "INSERT INTO correlation_types(id, display_name, db_table_name, supported, enabled) VALUES (?, ?, ?, ?, ?)"; preparedStatement = conn.prepareStatement(addAttributeSql); preparedStatement.setInt(1, CorrelationAttributeInstance.SSID_TYPE_ID); @@ -3171,6 +3171,7 @@ abstract class AbstractSqlEamDb implements EamDb { preparedStatement.setInt(5, 1); preparedStatement.execute(); + //create a new wireless_networks_instances table and add indexes for its columns statement.execute(String.format(addSsidTableTemplate, wirelessNetworksTableInstanceName, wirelessNetworksTableInstanceName)); statement.execute(String.format(addCaseIdIndexTemplate, wirelessNetworksTableInstanceName, wirelessNetworksTableInstanceName)); statement.execute(String.format(addDataSourceIdIndexTemplate, wirelessNetworksTableInstanceName, wirelessNetworksTableInstanceName)); diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactUtil.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactUtil.java index 2477d0fc41..aca0471345 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactUtil.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactUtil.java @@ -163,11 +163,10 @@ public class EamArtifactUtil { && BlackboardArtifact.ARTIFACT_TYPE.TSK_DEVICE_ATTACHED.getTypeID() == artifactTypeID) { value = bbArtifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DEVICE_ID)).getValueString(); - } - else if (correlationType.getId() == CorrelationAttributeInstance.SSID_TYPE_ID + } else if (correlationType.getId() == CorrelationAttributeInstance.SSID_TYPE_ID && BlackboardArtifact.ARTIFACT_TYPE.TSK_WIFI_NETWORK.getTypeID() == artifactTypeID) { value = bbArtifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SSID)).getValueString(); - } + } } catch (TskCoreException ex) { logger.log(Level.SEVERE, "Error getting attribute while getting type from BlackboardArtifact.", ex); // NON-NLS return null; @@ -187,7 +186,7 @@ public class EamArtifactUtil { * Uses the determined type and vallue, then looks up instance details to * create proper CorrelationAttributeInstance. * - * @param bbArtifact the blackboard artifatc + * @param bbArtifact the blackboard artifact * @param correlationType the given type * @param value the artifact value * diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDbSettings.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDbSettings.java index adaf7c0d9c..6ee454f915 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDbSettings.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDbSettings.java @@ -468,6 +468,13 @@ public final class PostgresEamDbSettings { return true; } + /** + * Get the template String for creating a new _instances table in a Postgres + * central repository. %s will exist in the template where the name of the + * new table will be addedd. + * + * @return a String which is a template for cretating a new _instances table + */ static String getCreateArtifactInstancesTableTemplate() { // Each "%s" will be replaced with the relevant TYPE_instances table name. StringBuilder createArtifactInstancesTableTemplate = new StringBuilder(); @@ -486,21 +493,53 @@ public final class PostgresEamDbSettings { return createArtifactInstancesTableTemplate.toString(); } + /** + * Get the template for creating an index on the case_id column of an + * instance table. %s will exist in the template where the name of the new + * table will be addedd. + * + * @return a String which is a template for adding an index to the case_id + * column of a _instances table + */ static String getAddCaseIdIndexTemplate() { // Each "%s" will be replaced with the relevant TYPE_instances table name. return "CREATE INDEX IF NOT EXISTS %s_case_id ON %s (case_id)"; } + /** + * Get the template for creating an index on the data_source_id column of an + * instance table. %s will exist in the template where the name of the new + * table will be addedd. + * + * @return a String which is a template for adding an index to the + * data_source_id column of a _instances table + */ static String getAddDataSourceIdIndexTemplate() { // Each "%s" will be replaced with the relevant TYPE_instances table name. return "CREATE INDEX IF NOT EXISTS %s_data_source_id ON %s (data_source_id)"; } + /** + * Get the template for creating an index on the value column of an instance + * table. %s will exist in the template where the name of the new table will + * be addedd. + * + * @return a String which is a template for adding an index to the value + * column of a _instances table + */ static String getAddValueIndexTemplate() { // Each "%s" will be replaced with the relevant TYPE_instances table name. return "CREATE INDEX IF NOT EXISTS %s_value ON %s (value)"; } + /** + * Get the template for creating an index on the known_status column of an + * instance table. %s will exist in the template where the name of the new + * table will be addedd. + * + * @return a String which is a template for adding an index to the + * known_status column of a _instances table + */ static String getAddKnownStatusIndexTemplate() { // Each "%s" will be replaced with the relevant TYPE_instances table name. return "CREATE INDEX IF NOT EXISTS %s_value_known_status ON %s (value, known_status)"; diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDbSettings.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDbSettings.java index c321e3b412..615e49e523 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDbSettings.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDbSettings.java @@ -415,7 +415,14 @@ public final class SqliteEamDbSettings { } return true; } - + + /** + * Get the template String for creating a new _instances table in a Sqlite + * central repository. %s will exist in the template where the name of the + * new table will be addedd. + * + * @return a String which is a template for cretating a new _instances table + */ static String getCreateArtifactInstancesTableTemplate() { // Each "%s" will be replaced with the relevant TYPE_instances table name. StringBuilder createArtifactInstancesTableTemplate = new StringBuilder(); @@ -434,21 +441,53 @@ public final class SqliteEamDbSettings { return createArtifactInstancesTableTemplate.toString(); } + /** + * Get the template for creating an index on the case_id column of an + * instance table. %s will exist in the template where the name of the new + * table will be addedd. + * + * @return a String which is a template for adding an index to the case_id + * column of a _instances table + */ static String getAddCaseIdIndexTemplate() { // Each "%s" will be replaced with the relevant TYPE_instances table name. return "CREATE INDEX IF NOT EXISTS %s_case_id ON %s (case_id)"; } + /** + * Get the template for creating an index on the data_source_id column of an + * instance table. %s will exist in the template where the name of the new + * table will be addedd. + * + * @return a String which is a template for adding an index to the + * data_source_id column of a _instances table + */ static String getAddDataSourceIdIndexTemplate() { // Each "%s" will be replaced with the relevant TYPE_instances table name. return "CREATE INDEX IF NOT EXISTS %s_data_source_id ON %s (data_source_id)"; } + /** + * Get the template for creating an index on the value column of an instance + * table. %s will exist in the template where the name of the new table will + * be addedd. + * + * @return a String which is a template for adding an index to the value + * column of a _instances table + */ static String getAddValueIndexTemplate() { // Each "%s" will be replaced with the relevant TYPE_instances table name. return "CREATE INDEX IF NOT EXISTS %s_value ON %s (value)"; } + /** + * Get the template for creating an index on the known_status column of an + * instance table. %s will exist in the template where the name of the new + * table will be addedd. + * + * @return a String which is a template for adding an index to the + * known_status column of a _instances table + */ static String getAddKnownStatusIndexTemplate() { // Each "%s" will be replaced with the relevant TYPE_instances table name. return "CREATE INDEX IF NOT EXISTS %s_value_known_status ON %s (value, known_status)"; From 8c7f987049ad11525abd35a104f9a0144072fd1a Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Tue, 30 Oct 2018 16:53:37 -0400 Subject: [PATCH 18/44] 4305 prevent older versions of Autopsy from lowering CR schema version number --- .../centralrepository/datamodel/AbstractSqlEamDb.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java index 4364845648..981a793f8d 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java @@ -3120,7 +3120,11 @@ abstract class AbstractSqlEamDb implements EamDb { logger.log(Level.INFO, "Central Repository is up to date"); return; } - + if (dbSchemaVersion.compareTo(CURRENT_DB_SCHEMA_VERSION) > 0) { + logger.log(Level.INFO, "Central Repository is of newer version than software creates") + return; + } + // Update from 1.0 to 1.1 if (dbSchemaVersion.compareTo(new CaseDbSchemaVersionNumber(1, 1)) < 0) { statement.execute("ALTER TABLE reference_sets ADD COLUMN known_status INTEGER;"); //NON-NLS From e4c08c87ef628fe52a0b21c4ea2ff30e67e005e3 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Tue, 30 Oct 2018 16:55:39 -0400 Subject: [PATCH 19/44] 4305 add missing semi-colon --- .../autopsy/centralrepository/datamodel/AbstractSqlEamDb.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java index 981a793f8d..fb80bbdacc 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java @@ -3121,7 +3121,7 @@ abstract class AbstractSqlEamDb implements EamDb { return; } if (dbSchemaVersion.compareTo(CURRENT_DB_SCHEMA_VERSION) > 0) { - logger.log(Level.INFO, "Central Repository is of newer version than software creates") + logger.log(Level.INFO, "Central Repository is of newer version than software creates"); return; } From 1c1163ed09bd5489a83b1b204df0d7107a7a5a06 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Tue, 30 Oct 2018 16:58:55 -0400 Subject: [PATCH 20/44] 4305 add update to 1.2 comment --- .../autopsy/centralrepository/datamodel/AbstractSqlEamDb.java | 1 + 1 file changed, 1 insertion(+) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java index fb80bbdacc..766bab44e3 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java @@ -3136,6 +3136,7 @@ abstract class AbstractSqlEamDb implements EamDb { // regardless of whether this succeeds. EamDbUtil.insertDefaultOrganization(conn); } + //Update to 1.2 if (dbSchemaVersion.compareTo(new CaseDbSchemaVersionNumber(1, 2)) < 0) { //update central repository to be able to store new correlation attributes EamDbPlatformEnum selectedPlatform = EamDbPlatformEnum.getSelectedPlatform(); From 18aa381882b098e2a3c8101d57470c7a6c4c9f0a Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Tue, 30 Oct 2018 17:44:50 -0400 Subject: [PATCH 21/44] Sort time zones by GMT offset. --- .../autopsy/coreutils/TimeZoneUtils.java | 78 ++++++++++++++++++- 1 file changed, 76 insertions(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/coreutils/TimeZoneUtils.java b/Core/src/org/sleuthkit/autopsy/coreutils/TimeZoneUtils.java index fa175a995a..91fb5d80f1 100644 --- a/Core/src/org/sleuthkit/autopsy/coreutils/TimeZoneUtils.java +++ b/Core/src/org/sleuthkit/autopsy/coreutils/TimeZoneUtils.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2011-2016 Basis Technology Corp. + * Copyright 2011-2018 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,7 +20,13 @@ package org.sleuthkit.autopsy.coreutils; import java.text.DateFormat; import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; import java.util.GregorianCalendar; +import java.util.List; +import java.util.SimpleTimeZone; +import java.util.TimeZone; /** * Utility methods for workig with time zones. @@ -41,7 +47,7 @@ public class TimeZoneUtils { java.util.TimeZone zone = java.util.TimeZone.getTimeZone(timeZoneId); int offset = zone.getRawOffset() / 1000; int hour = offset / 3600; - int min = (offset % 3600) / 60; + int min = Math.abs((offset % 3600) / 60); DateFormat dfm = new SimpleDateFormat("z"); dfm.setTimeZone(zone); @@ -59,6 +65,74 @@ public class TimeZoneUtils { return result; } + + /** + * Generate a time zone string containing the GMT offset and ID. + * + * @param timeZone The time zone. + * + * @return The time zone string. + */ + public static String createTimeZoneString(TimeZone timeZone) { + int offset = timeZone.getRawOffset() / 1000; + int hour = offset / 3600; + int minutes = Math.abs((offset % 3600) / 60); + + return String.format("(GMT%+d:%02d) %s", hour, minutes, timeZone.getID()); //NON-NLS + } + + /** + * Generates a list of time zones. + */ + public static List createTimeZoneList() { + /* + * Create a list of time zones. + */ + List timeZoneList = new ArrayList<>(); //DLG: NEW! + + // load and add all timezone + String[] ids = SimpleTimeZone.getAvailableIDs(); + for (String id : ids) { + /* + * DateFormat dfm = new SimpleDateFormat("z"); + * dfm.setTimeZone(zone); boolean hasDaylight = + * zone.useDaylightTime(); String first = dfm.format(new Date(2010, + * 1, 1)); String second = dfm.format(new Date(2011, 6, 6)); int mid + * = hour * -1; String result = first + Integer.toString(mid); + * if(hasDaylight){ result = result + second; } + * timeZoneComboBox.addItem(item + " (" + result + ")"); + */ + timeZoneList.add(TimeZone.getTimeZone(id)); + } + + /* + * Sort the list of time zones first by offset, then by ID. + */ + Collections.sort(timeZoneList, new Comparator(){ + @Override + public int compare(TimeZone o1, TimeZone o2){ + int offsetDelta = Integer.compare(o1.getRawOffset(), o2.getRawOffset()); + + if (offsetDelta == 0) { + return o1.getID().compareToIgnoreCase(o2.getID()); + } + + return offsetDelta; + } + }); + + /* + * Create a list of Strings encompassing both the GMT offset and the + * time zone ID. + */ + List outputList = new ArrayList<>(); + + for (TimeZone timeZone : timeZoneList) { + outputList.add(createTimeZoneString(timeZone)); + } + + return outputList; + } /** * Prevents instantiation. From 99af094b38b099b472e1eeaa45deff6fa458a6c7 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Tue, 30 Oct 2018 17:50:25 -0400 Subject: [PATCH 22/44] Cleanup. --- Core/src/org/sleuthkit/autopsy/coreutils/TimeZoneUtils.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/coreutils/TimeZoneUtils.java b/Core/src/org/sleuthkit/autopsy/coreutils/TimeZoneUtils.java index 91fb5d80f1..c844d9ce97 100644 --- a/Core/src/org/sleuthkit/autopsy/coreutils/TimeZoneUtils.java +++ b/Core/src/org/sleuthkit/autopsy/coreutils/TimeZoneUtils.java @@ -88,9 +88,8 @@ public class TimeZoneUtils { /* * Create a list of time zones. */ - List timeZoneList = new ArrayList<>(); //DLG: NEW! + List timeZoneList = new ArrayList<>(); - // load and add all timezone String[] ids = SimpleTimeZone.getAvailableIDs(); for (String id : ids) { /* From 79993506abc9ac103a14682e1eb73f4a2226e9f6 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Wed, 31 Oct 2018 12:09:33 -0400 Subject: [PATCH 23/44] Using Application tab as the default. --- .../autopsy/corecomponents/DataContentViewerArtifact.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/corecomponents/DataContentViewerArtifact.java b/Core/src/org/sleuthkit/autopsy/corecomponents/DataContentViewerArtifact.java index 2cccd31428..39d3203b13 100644 --- a/Core/src/org/sleuthkit/autopsy/corecomponents/DataContentViewerArtifact.java +++ b/Core/src/org/sleuthkit/autopsy/corecomponents/DataContentViewerArtifact.java @@ -487,7 +487,8 @@ public class DataContentViewerArtifact extends javax.swing.JPanel implements Dat || (artifact.getArtifactTypeID() == ARTIFACT_TYPE.TSK_KEYWORD_HIT.getTypeID()) || (artifact.getArtifactTypeID() == ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT.getTypeID()) || (artifact.getArtifactTypeID() == ARTIFACT_TYPE.TSK_OBJECT_DETECTED.getTypeID()) - || (artifact.getArtifactTypeID() == ARTIFACT_TYPE.TSK_METADATA_EXIF.getTypeID())) { + || (artifact.getArtifactTypeID() == ARTIFACT_TYPE.TSK_METADATA_EXIF.getTypeID()) + || (artifact.getArtifactTypeID() == ARTIFACT_TYPE.TSK_EXT_MISMATCH_DETECTED.getTypeID())) { return 3; } else { return 6; From 2dc3021e1a8b653817920c9bc49e14bc0167f1f2 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Wed, 31 Oct 2018 12:41:57 -0400 Subject: [PATCH 24/44] 4305 flag previously tagged as notable changed to being global setting --- .../datamodel/EamDbUtil.java | 59 ++++++++--- .../eventlisteners/IngestEventsListener.java | 13 ++- .../ingestmodule/IngestModule.java | 11 +-- .../ingestmodule/IngestModuleFactory.java | 39 +------- .../ingestmodule/IngestSettings.java | 71 -------------- .../ingestmodule/IngestSettingsPanel.form | 63 ------------ .../ingestmodule/IngestSettingsPanel.java | 98 ------------------- .../optionspanel/Bundle.properties | 3 +- .../optionspanel/GlobalSettingsPanel.form | 41 +++++--- .../optionspanel/GlobalSettingsPanel.java | 35 ++++--- 10 files changed, 111 insertions(+), 322 deletions(-) delete mode 100755 Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettings.java delete mode 100755 Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.form delete mode 100755 Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.java diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDbUtil.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDbUtil.java index e30ed174eb..db6687222a 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDbUtil.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDbUtil.java @@ -39,6 +39,7 @@ public class EamDbUtil { private final static Logger LOGGER = Logger.getLogger(EamDbUtil.class.getName()); private static final String CENTRAL_REPO_NAME = "CentralRepository"; private static final String CENTRAL_REPO_USE_KEY = "db.useCentralRepo"; + private static final String CENTRAL_REPO_FLAG_ITEMS_KEY = "db.flagNotableItems"; private static final String DEFAULT_ORG_NAME = "Not Specified"; /** @@ -192,9 +193,9 @@ public class EamDbUtil { } /** - * Upgrade the current central reposity to the newest version. If the upgrade - * fails, the central repository will be disabled and the current settings - * will be cleared. + * Upgrade the current central reposity to the newest version. If the + * upgrade fails, the central repository will be disabled and the current + * settings will be cleared. * * @return true if the upgrade succeeds, false otherwise. */ @@ -202,7 +203,7 @@ public class EamDbUtil { if (!EamDb.isEnabled()) { return true; } - + CoordinationService.Lock lock = null; try { EamDb db = EamDb.getInstance(); @@ -218,23 +219,23 @@ public class EamDbUtil { LOGGER.log(Level.SEVERE, "Error updating central repository", ex); // Disable the central repo and clear the current settings. - try{ + try { if (null != EamDb.getInstance()) { EamDb.getInstance().shutdownConnections(); } - } catch (EamDbException ex2){ + } catch (EamDbException ex2) { LOGGER.log(Level.SEVERE, "Error shutting down central repo connection pool", ex); - } + } setUseCentralRepo(false); EamDbPlatformEnum.setSelectedPlatform(EamDbPlatformEnum.DISABLED.name()); EamDbPlatformEnum.saveSelectedPlatform(); - + return false; } finally { - if(lock != null){ - try{ + if (lock != null) { + try { lock.release(); - } catch (CoordinationServiceException ex){ + } catch (CoordinationServiceException ex) { LOGGER.log(Level.SEVERE, "Error releasing database lock", ex); } } @@ -255,6 +256,7 @@ public class EamDbUtil { * Check whether the given org is the default organization. * * @param org + * * @return true if it is the default org, false otherwise */ public static boolean isDefaultOrg(EamOrganization org) { @@ -265,6 +267,7 @@ public class EamDbUtil { * Add the default organization to the database * * @param conn + * * @return true if successful, false otherwise */ static boolean insertDefaultOrganization(Connection conn) { @@ -295,7 +298,7 @@ public class EamDbUtil { * If the Central Repos use has been enabled. * * @return true if the Central Repo may be configured, false if it should - * not be able to be + * not be able to be */ public static boolean useCentralRepo() { return Boolean.parseBoolean(ModuleSettings.getConfigSetting(CENTRAL_REPO_NAME, CENTRAL_REPO_USE_KEY)); @@ -306,12 +309,40 @@ public class EamDbUtil { * configured. * * @param centralRepoCheckBoxIsSelected - true if the central repo can be - * used + * used */ public static void setUseCentralRepo(boolean centralRepoCheckBoxIsSelected) { ModuleSettings.setConfigSetting(CENTRAL_REPO_NAME, CENTRAL_REPO_USE_KEY, Boolean.toString(centralRepoCheckBoxIsSelected)); } + /** + * If items previously tagged as notable should be flagged after adding them + * to the Central Repo. + * + * @return true if the previously tagged as notable items should be flagged, + * false if previously tagged as notable items should not be flagged + */ + public static boolean flagNotableItems() { + String flagNotableItems = ModuleSettings.getConfigSetting(CENTRAL_REPO_NAME, CENTRAL_REPO_FLAG_ITEMS_KEY); + if (flagNotableItems == null){ //getConfigSetting can return null if the setting did not exist + setFlagNotableItems(true); + return true; + } + return Boolean.parseBoolean(flagNotableItems); + } + + /** + * Saves the setting for whether items previously tagged as notable should + * be flagged after adding them to the Central Repo. + * + * @param flagNotableItems - true if the previously tagged as notable items + * should be flagged, false if previously tagged as + * notable items should not be flagged + */ + public static void setFlagNotableItems(boolean flagNotableItems) { + ModuleSettings.setConfigSetting(CENTRAL_REPO_NAME, CENTRAL_REPO_FLAG_ITEMS_KEY, Boolean.toString(flagNotableItems)); + } + /** * Use the current settings and the validation query to test the connection * to the database. @@ -365,7 +396,7 @@ public class EamDbUtil { * Close the prepared statement. * * @param preparedStatement The prepared statement to be closed. - * + * * @deprecated Use closeStatement() instead. * * @throws EamDbException diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java index cdbf490e9e..7cdf06f82c 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java @@ -47,6 +47,7 @@ import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardAttribute; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.autopsy.centralrepository.datamodel.EamDb; +import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbUtil; import org.sleuthkit.autopsy.coreutils.ThreadUtils; /** @@ -183,9 +184,9 @@ public class IngestEventsListener { @Override public void propertyChange(PropertyChangeEvent evt) { - //if ingest is running we want there to check if there is a Correlation Engine module running - //sometimes artifacts are generated by DSPs or other sources while ingest is not running - //in these cases we still want to create correlation attributes for those artifacts when appropriate + //if ingest is running we want there to check if there is a Correlation Engine module running + //sometimes artifacts are generated by DSPs or other sources while ingest is not running + //in these cases we still want to create correlation attributes for those artifacts when appropriate if (!IngestManager.getInstance().isIngestRunning() || getCeModuleInstanceCount() > 0) { EamDb dbManager; try { @@ -196,11 +197,13 @@ public class IngestEventsListener { } switch (IngestManager.IngestModuleEvent.valueOf(evt.getPropertyName())) { case DATA_ADDED: { - jobProcessingExecutor.submit(new DataAddedTask(dbManager, evt, isFlagNotableItems())); + //if ingest isn't running read the flag notable items global setting otherwise use the value that existed when ingest was started + boolean flagNotable = IngestManager.getInstance().isIngestRunning() ? isFlagNotableItems() : EamDbUtil.flagNotableItems(); + jobProcessingExecutor.submit(new DataAddedTask(dbManager, evt, flagNotable)); break; } } - } + } } } diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java index 21baf59454..e091902b85 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java @@ -42,6 +42,7 @@ import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationDataSource; import org.sleuthkit.autopsy.centralrepository.datamodel.EamDb; import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbPlatformEnum; import org.sleuthkit.autopsy.centralrepository.datamodel.EamArtifactUtil; +import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbUtil; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardAttribute; @@ -71,7 +72,6 @@ final class IngestModule implements FileIngestModule { private CorrelationDataSource eamDataSource; private Blackboard blackboard; private CorrelationAttributeInstance.Type filesType; - private final boolean flagTaggedNotableItems; /** @@ -79,8 +79,8 @@ final class IngestModule implements FileIngestModule { * * @param settings The ingest settings for the module instance. */ - IngestModule(IngestSettings settings) { - flagTaggedNotableItems = settings.isFlagTaggedNotableItems(); + IngestModule() { + flagTaggedNotableItems = EamDbUtil.flagNotableItems(); } @Override @@ -214,9 +214,8 @@ final class IngestModule implements FileIngestModule { /* * Tell the IngestEventsListener to flag notable items based on the - * current module's configuration. This is a work around for the lack of - * an artifacts pipeline. Note that this can be changed by another - * module instance. All modules are affected by the value. While not + * global setting. This is a work around for the lack of + * an artifacts pipeline. All modules are affected by the value. While not * ideal, this will be good enough until a better solution can be * posited. * diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModuleFactory.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModuleFactory.java index fc5dcd9c81..8d6068a408 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModuleFactory.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModuleFactory.java @@ -26,8 +26,6 @@ import org.sleuthkit.autopsy.ingest.IngestModuleGlobalSettingsPanel; import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettings; import org.sleuthkit.autopsy.centralrepository.optionspanel.GlobalSettingsPanel; import org.sleuthkit.autopsy.coreutils.Version; -import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettingsPanel; -import org.sleuthkit.autopsy.ingest.NoIngestModuleIngestJobSettings; /** * Factory for Central Repository ingest modules @@ -68,17 +66,7 @@ public class IngestModuleFactory extends IngestModuleFactoryAdapter { @Override public FileIngestModule createFileIngestModule(IngestModuleIngestJobSettings settings) { - if (settings instanceof IngestSettings) { - return new IngestModule((IngestSettings) settings); - } - /* - * Compatibility check for older versions. - */ - if (settings instanceof NoIngestModuleIngestJobSettings) { - return new IngestModule(new IngestSettings()); - } - - throw new IllegalArgumentException("Expected settings argument to be an instance of IngestSettings"); + return new IngestModule(); } @Override @@ -92,30 +80,5 @@ public class IngestModuleFactory extends IngestModuleFactoryAdapter { globalOptionsPanel.load(); return globalOptionsPanel; } - - @Override - public IngestModuleIngestJobSettings getDefaultIngestJobSettings() { - return new IngestSettings(); - } - - @Override - public boolean hasIngestJobSettingsPanel() { - return true; - } - - @Override - public IngestModuleIngestJobSettingsPanel getIngestJobSettingsPanel(IngestModuleIngestJobSettings settings) { - if (settings instanceof IngestSettings) { - return new IngestSettingsPanel((IngestSettings) settings); - } - /* - * Compatibility check for older versions. - */ - if (settings instanceof NoIngestModuleIngestJobSettings) { - return new IngestSettingsPanel(new IngestSettings()); - } - - throw new IllegalArgumentException("Expected settings argument to be an instance of IngestSettings"); - } } diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettings.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettings.java deleted file mode 100755 index 32ab9e9f2d..0000000000 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettings.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Central Repository - * - * Copyright 2018 Basis Technology Corp. - * Contact: carrier sleuthkit org - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.sleuthkit.autopsy.centralrepository.ingestmodule; - -import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettings; - -/** - * Ingest job settings for the Correlation Engine module. - */ -final class IngestSettings implements IngestModuleIngestJobSettings { - - private static final long serialVersionUID = 1L; - - private boolean flagTaggedNotableItems; - - /** - * Instantiate the ingest job settings with default values. - */ - IngestSettings() { - this.flagTaggedNotableItems = IngestModule.DEFAULT_FLAG_TAGGED_NOTABLE_ITEMS; - } - - /** - * Instantiate the ingest job settings. - * - * @param flagTaggedNotableItems Flag previously tagged notable items. - */ - IngestSettings(boolean flagTaggedNotableItems) { - this.flagTaggedNotableItems = flagTaggedNotableItems; - } - - @Override - public long getVersionNumber() { - return serialVersionUID; - } - - /** - * Are previously tagged notable items to be flagged? - * - * @return True if flagging; otherwise false. - */ - boolean isFlagTaggedNotableItems() { - return flagTaggedNotableItems; - } - - /** - * Flag or ignore previously identified notable items. - * - * @param ignorePreviousNotableItems Are previously tagged notable items to - * be flagged? - */ - void setFlagTaggedNotableItems(boolean flagTaggedNotableItems) { - this.flagTaggedNotableItems = flagTaggedNotableItems; - } -} diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.form b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.form deleted file mode 100755 index 564031cb72..0000000000 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.form +++ /dev/null @@ -1,63 +0,0 @@ - - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.java deleted file mode 100755 index ed36c71287..0000000000 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Central Repository - * - * Copyright 2018 Basis Technology Corp. - * Contact: carrier sleuthkit org - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.sleuthkit.autopsy.centralrepository.ingestmodule; - -import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettings; -import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettingsPanel; - -/** - * Ingest job settings panel for the Correlation Engine module. - */ -@SuppressWarnings("PMD.SingularField") // UI widgets cause lots of false positives -final class IngestSettingsPanel extends IngestModuleIngestJobSettingsPanel { - - /** - * Creates new form IngestSettingsPanel - */ - public IngestSettingsPanel(IngestSettings settings) { - initComponents(); - customizeComponents(settings); - } - - /** - * Update components with values from the ingest job settings. - * - * @param settings The ingest job settings. - */ - private void customizeComponents(IngestSettings settings) { - flagTaggedNotableItemsCheckbox.setSelected(settings.isFlagTaggedNotableItems()); - } - - @Override - public IngestModuleIngestJobSettings getSettings() { - return new IngestSettings(flagTaggedNotableItemsCheckbox.isSelected()); - } - - /** - * This method is called from within the constructor to initialize the form. - * WARNING: Do NOT modify this code. The content of this method is always - * regenerated by the Form Editor. - */ - @SuppressWarnings("unchecked") - // //GEN-BEGIN:initComponents - private void initComponents() { - - ingestSettingsLabel = new javax.swing.JLabel(); - flagTaggedNotableItemsCheckbox = new javax.swing.JCheckBox(); - - ingestSettingsLabel.setFont(new java.awt.Font("Tahoma", 1, 11)); // NOI18N - org.openide.awt.Mnemonics.setLocalizedText(ingestSettingsLabel, org.openide.util.NbBundle.getMessage(IngestSettingsPanel.class, "IngestSettingsPanel.ingestSettingsLabel.text")); // NOI18N - - org.openide.awt.Mnemonics.setLocalizedText(flagTaggedNotableItemsCheckbox, org.openide.util.NbBundle.getMessage(IngestSettingsPanel.class, "IngestSettingsPanel.flagTaggedNotableItemsCheckbox.text")); // NOI18N - - javax.swing.GroupLayout layout = new javax.swing.GroupLayout(this); - this.setLayout(layout); - layout.setHorizontalGroup( - layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addGroup(layout.createSequentialGroup() - .addContainerGap() - .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addGroup(layout.createSequentialGroup() - .addGap(10, 10, 10) - .addComponent(flagTaggedNotableItemsCheckbox)) - .addComponent(ingestSettingsLabel)) - .addContainerGap(65, Short.MAX_VALUE)) - ); - layout.setVerticalGroup( - layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addGroup(layout.createSequentialGroup() - .addContainerGap() - .addComponent(ingestSettingsLabel) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) - .addComponent(flagTaggedNotableItemsCheckbox) - .addContainerGap(245, Short.MAX_VALUE)) - ); - }// //GEN-END:initComponents - - // Variables declaration - do not modify//GEN-BEGIN:variables - private javax.swing.JCheckBox flagTaggedNotableItemsCheckbox; - private javax.swing.JLabel ingestSettingsLabel; - // End of variables declaration//GEN-END:variables - -} diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/Bundle.properties b/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/Bundle.properties index b45b6f7579..a5bbe04992 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/Bundle.properties @@ -59,7 +59,6 @@ GlobalSettingsPanel.bnManageProperties.text=Manage Correlation Properties EamDbSettingsDialog.lbDatabaseDesc.text=Database File: EamDbSettingsDialog.lbFullDbPath.text= GlobalSettingsPanel.cbUseCentralRepo.text=Use a central repository -GlobalSettingsPanel.correlationPropertiesTextArea.text=Choose which file and result properties to store in the central repository for later correlation.\n GlobalSettingsPanel.organizationTextArea.text=Organization information can be tracked in the central repository. GlobalSettingsPanel.manageOrganizationButton.text=Manage Organizations GlobalSettingsPanel.lbCentralRepository.text=A central repository allows you to correlate files and results between cases. @@ -76,3 +75,5 @@ GlobalSettingsPanel.Case\ Details.AccessibleContext.accessibleName=Cases Details ShowCasesDialog.caseDetailsTable.AccessibleContext.accessibleDescription=Click column name to sort. GlobalSettingsPanel.casesTextArea.text=Display table that lists central repository case details. GlobalSettingsPanel.ingestRunningWarningLabel.text=Cannot make changes to central repository settings when ingest is running! +GlobalSettingsPanel.flagTaggedNotableItemsCheckbox.text=Flag items previously tagged as notable +GlobalSettingsPanel.correlationPropertiesTextArea.text=Choose which file and result properties to store in the central repository for later correlation.\n diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/GlobalSettingsPanel.form b/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/GlobalSettingsPanel.form index c3a8f678d6..cb9f3c1b5a 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/GlobalSettingsPanel.form +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/GlobalSettingsPanel.form @@ -89,13 +89,13 @@ - + - + - + - + @@ -242,24 +242,30 @@ - + - - - - + + + + + - + - - + + + + + + + @@ -301,7 +307,7 @@ - + @@ -314,6 +320,13 @@ + + + + + + + @@ -334,7 +347,7 @@ - + diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/GlobalSettingsPanel.java b/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/GlobalSettingsPanel.java index b8c42eb66d..356c5475e6 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/GlobalSettingsPanel.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/GlobalSettingsPanel.java @@ -120,6 +120,7 @@ public final class GlobalSettingsPanel extends IngestModuleGlobalSettingsPanel i bnManageTypes = new javax.swing.JButton(); correlationPropertiesScrollPane = new javax.swing.JScrollPane(); correlationPropertiesTextArea = new javax.swing.JTextArea(); + flagTaggedNotableItemsCheckbox = new javax.swing.JCheckBox(); organizationPanel = new javax.swing.JPanel(); manageOrganizationButton = new javax.swing.JButton(); organizationScrollPane = new javax.swing.JScrollPane(); @@ -222,13 +223,15 @@ public final class GlobalSettingsPanel extends IngestModuleGlobalSettingsPanel i correlationPropertiesTextArea.setColumns(20); correlationPropertiesTextArea.setFont(new java.awt.Font("Tahoma", 0, 11)); // NOI18N correlationPropertiesTextArea.setLineWrap(true); - correlationPropertiesTextArea.setRows(2); + correlationPropertiesTextArea.setRows(1); correlationPropertiesTextArea.setText(org.openide.util.NbBundle.getMessage(GlobalSettingsPanel.class, "GlobalSettingsPanel.correlationPropertiesTextArea.text")); // NOI18N correlationPropertiesTextArea.setToolTipText(""); correlationPropertiesTextArea.setWrapStyleWord(true); correlationPropertiesTextArea.setBorder(null); correlationPropertiesScrollPane.setViewportView(correlationPropertiesTextArea); + org.openide.awt.Mnemonics.setLocalizedText(flagTaggedNotableItemsCheckbox, org.openide.util.NbBundle.getMessage(GlobalSettingsPanel.class, "GlobalSettingsPanel.flagTaggedNotableItemsCheckbox.text")); // NOI18N + javax.swing.GroupLayout pnCorrelationPropertiesLayout = new javax.swing.GroupLayout(pnCorrelationProperties); pnCorrelationProperties.setLayout(pnCorrelationPropertiesLayout); pnCorrelationPropertiesLayout.setHorizontalGroup( @@ -236,17 +239,21 @@ public final class GlobalSettingsPanel extends IngestModuleGlobalSettingsPanel i .addGroup(pnCorrelationPropertiesLayout.createSequentialGroup() .addContainerGap() .addGroup(pnCorrelationPropertiesLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addComponent(correlationPropertiesScrollPane) .addGroup(pnCorrelationPropertiesLayout.createSequentialGroup() - .addComponent(bnManageTypes) - .addGap(0, 0, Short.MAX_VALUE))) - .addContainerGap()) + .addComponent(correlationPropertiesScrollPane, javax.swing.GroupLayout.PREFERRED_SIZE, 497, javax.swing.GroupLayout.PREFERRED_SIZE) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addComponent(flagTaggedNotableItemsCheckbox)) + .addComponent(bnManageTypes)) + .addContainerGap(javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)) ); pnCorrelationPropertiesLayout.setVerticalGroup( pnCorrelationPropertiesLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addGroup(javax.swing.GroupLayout.Alignment.TRAILING, pnCorrelationPropertiesLayout.createSequentialGroup() - .addGap(7, 7, 7) - .addComponent(correlationPropertiesScrollPane, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE) + .addGroup(pnCorrelationPropertiesLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addGroup(pnCorrelationPropertiesLayout.createSequentialGroup() + .addContainerGap() + .addComponent(flagTaggedNotableItemsCheckbox)) + .addComponent(correlationPropertiesScrollPane, javax.swing.GroupLayout.DEFAULT_SIZE, 32, Short.MAX_VALUE)) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) .addComponent(bnManageTypes) .addGap(8, 8, 8)) @@ -281,7 +288,7 @@ public final class GlobalSettingsPanel extends IngestModuleGlobalSettingsPanel i .addGroup(organizationPanelLayout.createSequentialGroup() .addContainerGap() .addGroup(organizationPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addComponent(organizationScrollPane, javax.swing.GroupLayout.DEFAULT_SIZE, 992, Short.MAX_VALUE) + .addComponent(organizationScrollPane, javax.swing.GroupLayout.DEFAULT_SIZE, 980, Short.MAX_VALUE) .addGroup(organizationPanelLayout.createSequentialGroup() .addComponent(manageOrganizationButton) .addGap(0, 0, Short.MAX_VALUE))) @@ -382,13 +389,13 @@ public final class GlobalSettingsPanel extends IngestModuleGlobalSettingsPanel i .addComponent(ingestRunningWarningLabel)) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) .addComponent(pnDatabaseConfiguration, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addGap(0, 0, 0) .addComponent(pnCorrelationProperties, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addGap(0, 0, 0) .addComponent(organizationPanel, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addGap(0, 0, 0) .addComponent(casesPanel, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addGap(0, 0, 0) .addComponent(tbOops, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE) .addContainerGap()) ); @@ -451,6 +458,7 @@ public final class GlobalSettingsPanel extends IngestModuleGlobalSettingsPanel i enableAllSubComponents(false); EamDbPlatformEnum selectedPlatform = EamDbPlatformEnum.getSelectedPlatform(); cbUseCentralRepo.setSelected(EamDbUtil.useCentralRepo()); // NON-NLS + flagTaggedNotableItemsCheckbox.setSelected(EamDbUtil.flagNotableItems()); switch (selectedPlatform) { case POSTGRESQL: PostgresEamDbSettings dbSettingsPg = new PostgresEamDbSettings(); @@ -480,6 +488,7 @@ public final class GlobalSettingsPanel extends IngestModuleGlobalSettingsPanel i @Override public void store() { // Click OK or Apply on Options Panel EamDbUtil.setUseCentralRepo(cbUseCentralRepo.isSelected()); + EamDbUtil.setFlagNotableItems(flagTaggedNotableItemsCheckbox.isSelected()); } /** @@ -625,6 +634,7 @@ public final class GlobalSettingsPanel extends IngestModuleGlobalSettingsPanel i showCasesButton.setEnabled(enable && !ingestRunning); casesPanel.setEnabled(enable && !ingestRunning); casesTextArea.setEnabled(enable && !ingestRunning); + flagTaggedNotableItemsCheckbox.setEnabled(enable && !ingestRunning); return true; } @@ -637,6 +647,7 @@ public final class GlobalSettingsPanel extends IngestModuleGlobalSettingsPanel i private javax.swing.JCheckBox cbUseCentralRepo; private javax.swing.JScrollPane correlationPropertiesScrollPane; private javax.swing.JTextArea correlationPropertiesTextArea; + private javax.swing.JCheckBox flagTaggedNotableItemsCheckbox; private javax.swing.JLabel ingestRunningWarningLabel; private javax.swing.JPanel jPanel1; private javax.swing.JScrollPane jScrollPane1; From 650dbb963701cfd09b9ccc17ccac929576262b8d Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Wed, 31 Oct 2018 14:03:18 -0400 Subject: [PATCH 25/44] 4305 modify placement of flag previously notable setting, restore cr ingestsettings file --- .../ingestmodule/IngestSettings.java | 71 +++++++++++++++++++ .../optionspanel/Bundle.properties | 1 + .../optionspanel/GlobalSettingsPanel.form | 43 +++++++---- .../optionspanel/GlobalSettingsPanel.java | 34 +++++---- 4 files changed, 122 insertions(+), 27 deletions(-) create mode 100755 Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettings.java diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettings.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettings.java new file mode 100755 index 0000000000..32ab9e9f2d --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettings.java @@ -0,0 +1,71 @@ +/* + * Central Repository + * + * Copyright 2018 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.centralrepository.ingestmodule; + +import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettings; + +/** + * Ingest job settings for the Correlation Engine module. + */ +final class IngestSettings implements IngestModuleIngestJobSettings { + + private static final long serialVersionUID = 1L; + + private boolean flagTaggedNotableItems; + + /** + * Instantiate the ingest job settings with default values. + */ + IngestSettings() { + this.flagTaggedNotableItems = IngestModule.DEFAULT_FLAG_TAGGED_NOTABLE_ITEMS; + } + + /** + * Instantiate the ingest job settings. + * + * @param flagTaggedNotableItems Flag previously tagged notable items. + */ + IngestSettings(boolean flagTaggedNotableItems) { + this.flagTaggedNotableItems = flagTaggedNotableItems; + } + + @Override + public long getVersionNumber() { + return serialVersionUID; + } + + /** + * Are previously tagged notable items to be flagged? + * + * @return True if flagging; otherwise false. + */ + boolean isFlagTaggedNotableItems() { + return flagTaggedNotableItems; + } + + /** + * Flag or ignore previously identified notable items. + * + * @param ignorePreviousNotableItems Are previously tagged notable items to + * be flagged? + */ + void setFlagTaggedNotableItems(boolean flagTaggedNotableItems) { + this.flagTaggedNotableItems = flagTaggedNotableItems; + } +} diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/Bundle.properties b/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/Bundle.properties index a5bbe04992..88a2150bd6 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/Bundle.properties @@ -77,3 +77,4 @@ GlobalSettingsPanel.casesTextArea.text=Display table that lists central reposito GlobalSettingsPanel.ingestRunningWarningLabel.text=Cannot make changes to central repository settings when ingest is running! GlobalSettingsPanel.flagTaggedNotableItemsCheckbox.text=Flag items previously tagged as notable GlobalSettingsPanel.correlationPropertiesTextArea.text=Choose which file and result properties to store in the central repository for later correlation.\n +GlobalSettingsPanel.generateInterestingItemsLabel.text=Generate interesting items to: diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/GlobalSettingsPanel.form b/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/GlobalSettingsPanel.form index cb9f3c1b5a..24efccdc99 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/GlobalSettingsPanel.form +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/GlobalSettingsPanel.form @@ -57,7 +57,7 @@ - + @@ -243,33 +243,39 @@ - + + + + + - - - + + + + + + - - + - - - - - - - + + + + + + + @@ -327,6 +333,13 @@ + + + + + + + @@ -347,7 +360,7 @@ - + diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/GlobalSettingsPanel.java b/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/GlobalSettingsPanel.java index 356c5475e6..ed7fa1b1d0 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/GlobalSettingsPanel.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/GlobalSettingsPanel.java @@ -121,6 +121,7 @@ public final class GlobalSettingsPanel extends IngestModuleGlobalSettingsPanel i correlationPropertiesScrollPane = new javax.swing.JScrollPane(); correlationPropertiesTextArea = new javax.swing.JTextArea(); flagTaggedNotableItemsCheckbox = new javax.swing.JCheckBox(); + generateInterestingItemsLabel = new javax.swing.JLabel(); organizationPanel = new javax.swing.JPanel(); manageOrganizationButton = new javax.swing.JButton(); organizationScrollPane = new javax.swing.JScrollPane(); @@ -232,31 +233,38 @@ public final class GlobalSettingsPanel extends IngestModuleGlobalSettingsPanel i org.openide.awt.Mnemonics.setLocalizedText(flagTaggedNotableItemsCheckbox, org.openide.util.NbBundle.getMessage(GlobalSettingsPanel.class, "GlobalSettingsPanel.flagTaggedNotableItemsCheckbox.text")); // NOI18N + org.openide.awt.Mnemonics.setLocalizedText(generateInterestingItemsLabel, org.openide.util.NbBundle.getMessage(GlobalSettingsPanel.class, "GlobalSettingsPanel.generateInterestingItemsLabel.text")); // NOI18N + javax.swing.GroupLayout pnCorrelationPropertiesLayout = new javax.swing.GroupLayout(pnCorrelationProperties); pnCorrelationProperties.setLayout(pnCorrelationPropertiesLayout); pnCorrelationPropertiesLayout.setHorizontalGroup( pnCorrelationPropertiesLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addGroup(pnCorrelationPropertiesLayout.createSequentialGroup() .addContainerGap() + .addGroup(pnCorrelationPropertiesLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addComponent(bnManageTypes) + .addComponent(correlationPropertiesScrollPane, javax.swing.GroupLayout.PREFERRED_SIZE, 468, javax.swing.GroupLayout.PREFERRED_SIZE)) .addGroup(pnCorrelationPropertiesLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addGroup(pnCorrelationPropertiesLayout.createSequentialGroup() - .addComponent(correlationPropertiesScrollPane, javax.swing.GroupLayout.PREFERRED_SIZE, 497, javax.swing.GroupLayout.PREFERRED_SIZE) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) - .addComponent(flagTaggedNotableItemsCheckbox)) - .addComponent(bnManageTypes)) - .addContainerGap(javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)) + .addGap(29, 29, 29) + .addComponent(flagTaggedNotableItemsCheckbox, javax.swing.GroupLayout.PREFERRED_SIZE, 246, javax.swing.GroupLayout.PREFERRED_SIZE)) + .addGroup(pnCorrelationPropertiesLayout.createSequentialGroup() + .addGap(18, 18, 18) + .addComponent(generateInterestingItemsLabel, javax.swing.GroupLayout.PREFERRED_SIZE, 187, javax.swing.GroupLayout.PREFERRED_SIZE))) + .addContainerGap(247, Short.MAX_VALUE)) ); pnCorrelationPropertiesLayout.setVerticalGroup( pnCorrelationPropertiesLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addGroup(javax.swing.GroupLayout.Alignment.TRAILING, pnCorrelationPropertiesLayout.createSequentialGroup() - .addGroup(pnCorrelationPropertiesLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addGroup(pnCorrelationPropertiesLayout.createSequentialGroup() - .addContainerGap() - .addComponent(flagTaggedNotableItemsCheckbox)) - .addComponent(correlationPropertiesScrollPane, javax.swing.GroupLayout.DEFAULT_SIZE, 32, Short.MAX_VALUE)) + .addComponent(correlationPropertiesScrollPane, javax.swing.GroupLayout.DEFAULT_SIZE, 32, Short.MAX_VALUE) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) .addComponent(bnManageTypes) .addGap(8, 8, 8)) + .addGroup(pnCorrelationPropertiesLayout.createSequentialGroup() + .addComponent(generateInterestingItemsLabel) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addComponent(flagTaggedNotableItemsCheckbox) + .addGap(0, 0, 0)) ); organizationPanel.setBorder(javax.swing.BorderFactory.createTitledBorder(null, org.openide.util.NbBundle.getMessage(GlobalSettingsPanel.class, "GlobalSettingsPanel.organizationPanel.border.title"), javax.swing.border.TitledBorder.DEFAULT_JUSTIFICATION, javax.swing.border.TitledBorder.DEFAULT_POSITION, new java.awt.Font("Tahoma", 0, 12))); // NOI18N @@ -288,7 +296,7 @@ public final class GlobalSettingsPanel extends IngestModuleGlobalSettingsPanel i .addGroup(organizationPanelLayout.createSequentialGroup() .addContainerGap() .addGroup(organizationPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addComponent(organizationScrollPane, javax.swing.GroupLayout.DEFAULT_SIZE, 980, Short.MAX_VALUE) + .addComponent(organizationScrollPane) .addGroup(organizationPanelLayout.createSequentialGroup() .addComponent(manageOrganizationButton) .addGap(0, 0, Short.MAX_VALUE))) @@ -363,7 +371,7 @@ public final class GlobalSettingsPanel extends IngestModuleGlobalSettingsPanel i jPanel1.setLayout(jPanel1Layout); jPanel1Layout.setHorizontalGroup( jPanel1Layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addComponent(lbCentralRepository, javax.swing.GroupLayout.DEFAULT_SIZE, 1022, Short.MAX_VALUE) + .addComponent(lbCentralRepository, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) .addGroup(jPanel1Layout.createSequentialGroup() .addGroup(jPanel1Layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addComponent(pnDatabaseConfiguration, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) @@ -635,6 +643,7 @@ public final class GlobalSettingsPanel extends IngestModuleGlobalSettingsPanel i casesPanel.setEnabled(enable && !ingestRunning); casesTextArea.setEnabled(enable && !ingestRunning); flagTaggedNotableItemsCheckbox.setEnabled(enable && !ingestRunning); + generateInterestingItemsLabel.setEnabled(enable & !ingestRunning); return true; } @@ -648,6 +657,7 @@ public final class GlobalSettingsPanel extends IngestModuleGlobalSettingsPanel i private javax.swing.JScrollPane correlationPropertiesScrollPane; private javax.swing.JTextArea correlationPropertiesTextArea; private javax.swing.JCheckBox flagTaggedNotableItemsCheckbox; + private javax.swing.JLabel generateInterestingItemsLabel; private javax.swing.JLabel ingestRunningWarningLabel; private javax.swing.JPanel jPanel1; private javax.swing.JScrollPane jScrollPane1; From 39a499c96e51165410aa9d9b20a52105f5da03c3 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Wed, 31 Oct 2018 16:41:40 -0400 Subject: [PATCH 26/44] 4305 always create interesting items if CR is enabled and ingest is not running --- .../eventlisteners/IngestEventsListener.java | 5 +- .../ingestmodule/IngestModule.java | 9 +- .../ingestmodule/IngestSettingsPanel.form | 63 ++++++++++++ .../ingestmodule/IngestSettingsPanel.java | 98 +++++++++++++++++++ .../optionspanel/Bundle.properties | 16 ++- .../optionspanel/EamDbSettingsDialog.java | 2 +- .../optionspanel/GlobalSettingsPanel.form | 37 +------ .../optionspanel/GlobalSettingsPanel.java | 32 +----- 8 files changed, 186 insertions(+), 76 deletions(-) create mode 100755 Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.form create mode 100755 Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.java diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java index 7cdf06f82c..40dcd8338a 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java @@ -47,7 +47,6 @@ import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardAttribute; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.autopsy.centralrepository.datamodel.EamDb; -import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbUtil; import org.sleuthkit.autopsy.coreutils.ThreadUtils; /** @@ -197,8 +196,8 @@ public class IngestEventsListener { } switch (IngestManager.IngestModuleEvent.valueOf(evt.getPropertyName())) { case DATA_ADDED: { - //if ingest isn't running read the flag notable items global setting otherwise use the value that existed when ingest was started - boolean flagNotable = IngestManager.getInstance().isIngestRunning() ? isFlagNotableItems() : EamDbUtil.flagNotableItems(); + //if ingest isn't running create the interesting items + boolean flagNotable = IngestManager.getInstance().isIngestRunning() ? isFlagNotableItems() : true; jobProcessingExecutor.submit(new DataAddedTask(dbManager, evt, flagNotable)); break; } diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java index e091902b85..1ee6dd8a68 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java @@ -79,8 +79,8 @@ final class IngestModule implements FileIngestModule { * * @param settings The ingest settings for the module instance. */ - IngestModule() { - flagTaggedNotableItems = EamDbUtil.flagNotableItems(); + IngestModule(IngestSettings settings) { + flagTaggedNotableItems = settings.isFlagTaggedNotableItems(); } @Override @@ -214,8 +214,9 @@ final class IngestModule implements FileIngestModule { /* * Tell the IngestEventsListener to flag notable items based on the - * global setting. This is a work around for the lack of - * an artifacts pipeline. All modules are affected by the value. While not + * current module's configuration. This is a work around for the lack of + * an artifacts pipeline. Note that this can be changed by another + * module instance. All modules are affected by the value. While not * ideal, this will be good enough until a better solution can be * posited. * diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.form b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.form new file mode 100755 index 0000000000..564031cb72 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.form @@ -0,0 +1,63 @@ + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.java new file mode 100755 index 0000000000..ed36c71287 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.java @@ -0,0 +1,98 @@ +/* + * Central Repository + * + * Copyright 2018 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.centralrepository.ingestmodule; + +import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettings; +import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettingsPanel; + +/** + * Ingest job settings panel for the Correlation Engine module. + */ +@SuppressWarnings("PMD.SingularField") // UI widgets cause lots of false positives +final class IngestSettingsPanel extends IngestModuleIngestJobSettingsPanel { + + /** + * Creates new form IngestSettingsPanel + */ + public IngestSettingsPanel(IngestSettings settings) { + initComponents(); + customizeComponents(settings); + } + + /** + * Update components with values from the ingest job settings. + * + * @param settings The ingest job settings. + */ + private void customizeComponents(IngestSettings settings) { + flagTaggedNotableItemsCheckbox.setSelected(settings.isFlagTaggedNotableItems()); + } + + @Override + public IngestModuleIngestJobSettings getSettings() { + return new IngestSettings(flagTaggedNotableItemsCheckbox.isSelected()); + } + + /** + * This method is called from within the constructor to initialize the form. + * WARNING: Do NOT modify this code. The content of this method is always + * regenerated by the Form Editor. + */ + @SuppressWarnings("unchecked") + // //GEN-BEGIN:initComponents + private void initComponents() { + + ingestSettingsLabel = new javax.swing.JLabel(); + flagTaggedNotableItemsCheckbox = new javax.swing.JCheckBox(); + + ingestSettingsLabel.setFont(new java.awt.Font("Tahoma", 1, 11)); // NOI18N + org.openide.awt.Mnemonics.setLocalizedText(ingestSettingsLabel, org.openide.util.NbBundle.getMessage(IngestSettingsPanel.class, "IngestSettingsPanel.ingestSettingsLabel.text")); // NOI18N + + org.openide.awt.Mnemonics.setLocalizedText(flagTaggedNotableItemsCheckbox, org.openide.util.NbBundle.getMessage(IngestSettingsPanel.class, "IngestSettingsPanel.flagTaggedNotableItemsCheckbox.text")); // NOI18N + + javax.swing.GroupLayout layout = new javax.swing.GroupLayout(this); + this.setLayout(layout); + layout.setHorizontalGroup( + layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addGroup(layout.createSequentialGroup() + .addContainerGap() + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addGroup(layout.createSequentialGroup() + .addGap(10, 10, 10) + .addComponent(flagTaggedNotableItemsCheckbox)) + .addComponent(ingestSettingsLabel)) + .addContainerGap(65, Short.MAX_VALUE)) + ); + layout.setVerticalGroup( + layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addGroup(layout.createSequentialGroup() + .addContainerGap() + .addComponent(ingestSettingsLabel) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) + .addComponent(flagTaggedNotableItemsCheckbox) + .addContainerGap(245, Short.MAX_VALUE)) + ); + }// //GEN-END:initComponents + + // Variables declaration - do not modify//GEN-BEGIN:variables + private javax.swing.JCheckBox flagTaggedNotableItemsCheckbox; + private javax.swing.JLabel ingestSettingsLabel; + // End of variables declaration//GEN-END:variables + +} diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/Bundle.properties b/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/Bundle.properties index 88a2150bd6..cbafa5602c 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/Bundle.properties @@ -34,7 +34,7 @@ AddNewOrganizationDialog.bnOK.text=OK AddNewOrganizationDialog.tfName.tooltip=POC Name ManageTagsDialog.okButton.text=OK ManageTagsDialog.cancelButton.text=Cancel -ManageArtifactTypesDialog.taInstructionsMsg.text=Enable one or more correlation properties to use for correlation during ingest. Note, these properties are global and impact all users of the central repository. +ManageArtifactTypesDialog.taInstructionsMsg.text=Enable one or more correlation properties to use for correlation during ingest. Note, these properties are global and impact all users of the Central Repository. EamSqliteSettingsDialog.bnOk.text=OK EamPostgresSettingsDialog.bnSave.text=Save EamDbSettingsDialog.bnDatabasePathFileOpen.text=Browse... @@ -58,10 +58,10 @@ ManageCorrelationPropertiesDialog.okButton.text=OK GlobalSettingsPanel.bnManageProperties.text=Manage Correlation Properties EamDbSettingsDialog.lbDatabaseDesc.text=Database File: EamDbSettingsDialog.lbFullDbPath.text= -GlobalSettingsPanel.cbUseCentralRepo.text=Use a central repository -GlobalSettingsPanel.organizationTextArea.text=Organization information can be tracked in the central repository. +GlobalSettingsPanel.cbUseCentralRepo.text=Use a Central Repository +GlobalSettingsPanel.organizationTextArea.text=Organization information can be tracked in the Central Repository. GlobalSettingsPanel.manageOrganizationButton.text=Manage Organizations -GlobalSettingsPanel.lbCentralRepository.text=A central repository allows you to correlate files and results between cases. +GlobalSettingsPanel.lbCentralRepository.text=A Central Repository allows you to correlate files and results between cases. GlobalSettingsPanel.pnCorrelationProperties.border.title=Correlation Properties GlobalSettingsPanel.organizationPanel.border.title=Organizations GlobalSettingsPanel.casesPanel.border.title=Case Details @@ -73,8 +73,6 @@ ShowCasesDialog.caseDetailsTable.toolTipText=Click column name to sort. Right-cl ShowCasesDialog.title=Case Details GlobalSettingsPanel.Case\ Details.AccessibleContext.accessibleName=Cases Details ShowCasesDialog.caseDetailsTable.AccessibleContext.accessibleDescription=Click column name to sort. -GlobalSettingsPanel.casesTextArea.text=Display table that lists central repository case details. -GlobalSettingsPanel.ingestRunningWarningLabel.text=Cannot make changes to central repository settings when ingest is running! -GlobalSettingsPanel.flagTaggedNotableItemsCheckbox.text=Flag items previously tagged as notable -GlobalSettingsPanel.correlationPropertiesTextArea.text=Choose which file and result properties to store in the central repository for later correlation.\n -GlobalSettingsPanel.generateInterestingItemsLabel.text=Generate interesting items to: +GlobalSettingsPanel.casesTextArea.text=Display table that lists Central Repository case details. +GlobalSettingsPanel.ingestRunningWarningLabel.text=Cannot make changes to Central Repository settings when ingest is running! +GlobalSettingsPanel.correlationPropertiesTextArea.text=Choose which file and result properties to store in the Central Repository for later correlation.\n diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/EamDbSettingsDialog.java b/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/EamDbSettingsDialog.java index c504f666e4..826e66ecbc 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/EamDbSettingsDialog.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/EamDbSettingsDialog.java @@ -102,7 +102,7 @@ public class EamDbSettingsDialog extends JDialog { @Override public String getDescription() { - return "Directories and central repository databases"; + return "Directories and Central Repository databases"; } }); cbDatabaseType.setSelectedItem(selectedPlatform); diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/GlobalSettingsPanel.form b/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/GlobalSettingsPanel.form index 24efccdc99..0a8c7dcc64 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/GlobalSettingsPanel.form +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/GlobalSettingsPanel.form @@ -67,7 +67,7 @@ - + @@ -244,21 +244,14 @@ - - - - - - - - - - + + + - +
@@ -270,12 +263,6 @@ - - - - - - @@ -326,20 +313,6 @@ - - - - - - - - - - - - - - diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/GlobalSettingsPanel.java b/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/GlobalSettingsPanel.java index ed7fa1b1d0..c8560fce10 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/GlobalSettingsPanel.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/GlobalSettingsPanel.java @@ -120,8 +120,6 @@ public final class GlobalSettingsPanel extends IngestModuleGlobalSettingsPanel i bnManageTypes = new javax.swing.JButton(); correlationPropertiesScrollPane = new javax.swing.JScrollPane(); correlationPropertiesTextArea = new javax.swing.JTextArea(); - flagTaggedNotableItemsCheckbox = new javax.swing.JCheckBox(); - generateInterestingItemsLabel = new javax.swing.JLabel(); organizationPanel = new javax.swing.JPanel(); manageOrganizationButton = new javax.swing.JButton(); organizationScrollPane = new javax.swing.JScrollPane(); @@ -231,27 +229,18 @@ public final class GlobalSettingsPanel extends IngestModuleGlobalSettingsPanel i correlationPropertiesTextArea.setBorder(null); correlationPropertiesScrollPane.setViewportView(correlationPropertiesTextArea); - org.openide.awt.Mnemonics.setLocalizedText(flagTaggedNotableItemsCheckbox, org.openide.util.NbBundle.getMessage(GlobalSettingsPanel.class, "GlobalSettingsPanel.flagTaggedNotableItemsCheckbox.text")); // NOI18N - - org.openide.awt.Mnemonics.setLocalizedText(generateInterestingItemsLabel, org.openide.util.NbBundle.getMessage(GlobalSettingsPanel.class, "GlobalSettingsPanel.generateInterestingItemsLabel.text")); // NOI18N - javax.swing.GroupLayout pnCorrelationPropertiesLayout = new javax.swing.GroupLayout(pnCorrelationProperties); pnCorrelationProperties.setLayout(pnCorrelationPropertiesLayout); pnCorrelationPropertiesLayout.setHorizontalGroup( pnCorrelationPropertiesLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addGroup(pnCorrelationPropertiesLayout.createSequentialGroup() .addContainerGap() - .addGroup(pnCorrelationPropertiesLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addComponent(bnManageTypes) - .addComponent(correlationPropertiesScrollPane, javax.swing.GroupLayout.PREFERRED_SIZE, 468, javax.swing.GroupLayout.PREFERRED_SIZE)) .addGroup(pnCorrelationPropertiesLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addGroup(pnCorrelationPropertiesLayout.createSequentialGroup() - .addGap(29, 29, 29) - .addComponent(flagTaggedNotableItemsCheckbox, javax.swing.GroupLayout.PREFERRED_SIZE, 246, javax.swing.GroupLayout.PREFERRED_SIZE)) - .addGroup(pnCorrelationPropertiesLayout.createSequentialGroup() - .addGap(18, 18, 18) - .addComponent(generateInterestingItemsLabel, javax.swing.GroupLayout.PREFERRED_SIZE, 187, javax.swing.GroupLayout.PREFERRED_SIZE))) - .addContainerGap(247, Short.MAX_VALUE)) + .addComponent(bnManageTypes) + .addGap(0, 0, Short.MAX_VALUE)) + .addComponent(correlationPropertiesScrollPane)) + .addContainerGap()) ); pnCorrelationPropertiesLayout.setVerticalGroup( pnCorrelationPropertiesLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) @@ -260,11 +249,6 @@ public final class GlobalSettingsPanel extends IngestModuleGlobalSettingsPanel i .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) .addComponent(bnManageTypes) .addGap(8, 8, 8)) - .addGroup(pnCorrelationPropertiesLayout.createSequentialGroup() - .addComponent(generateInterestingItemsLabel) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) - .addComponent(flagTaggedNotableItemsCheckbox) - .addGap(0, 0, 0)) ); organizationPanel.setBorder(javax.swing.BorderFactory.createTitledBorder(null, org.openide.util.NbBundle.getMessage(GlobalSettingsPanel.class, "GlobalSettingsPanel.organizationPanel.border.title"), javax.swing.border.TitledBorder.DEFAULT_JUSTIFICATION, javax.swing.border.TitledBorder.DEFAULT_POSITION, new java.awt.Font("Tahoma", 0, 12))); // NOI18N @@ -381,7 +365,7 @@ public final class GlobalSettingsPanel extends IngestModuleGlobalSettingsPanel i .addGroup(jPanel1Layout.createSequentialGroup() .addComponent(cbUseCentralRepo, javax.swing.GroupLayout.PREFERRED_SIZE, 162, javax.swing.GroupLayout.PREFERRED_SIZE) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) - .addComponent(ingestRunningWarningLabel, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)) + .addComponent(ingestRunningWarningLabel, javax.swing.GroupLayout.DEFAULT_SIZE, 844, Short.MAX_VALUE)) .addGroup(jPanel1Layout.createSequentialGroup() .addContainerGap() .addComponent(tbOops, javax.swing.GroupLayout.PREFERRED_SIZE, 974, javax.swing.GroupLayout.PREFERRED_SIZE))) @@ -466,7 +450,6 @@ public final class GlobalSettingsPanel extends IngestModuleGlobalSettingsPanel i enableAllSubComponents(false); EamDbPlatformEnum selectedPlatform = EamDbPlatformEnum.getSelectedPlatform(); cbUseCentralRepo.setSelected(EamDbUtil.useCentralRepo()); // NON-NLS - flagTaggedNotableItemsCheckbox.setSelected(EamDbUtil.flagNotableItems()); switch (selectedPlatform) { case POSTGRESQL: PostgresEamDbSettings dbSettingsPg = new PostgresEamDbSettings(); @@ -496,7 +479,6 @@ public final class GlobalSettingsPanel extends IngestModuleGlobalSettingsPanel i @Override public void store() { // Click OK or Apply on Options Panel EamDbUtil.setUseCentralRepo(cbUseCentralRepo.isSelected()); - EamDbUtil.setFlagNotableItems(flagTaggedNotableItemsCheckbox.isSelected()); } /** @@ -642,8 +624,6 @@ public final class GlobalSettingsPanel extends IngestModuleGlobalSettingsPanel i showCasesButton.setEnabled(enable && !ingestRunning); casesPanel.setEnabled(enable && !ingestRunning); casesTextArea.setEnabled(enable && !ingestRunning); - flagTaggedNotableItemsCheckbox.setEnabled(enable && !ingestRunning); - generateInterestingItemsLabel.setEnabled(enable & !ingestRunning); return true; } @@ -656,8 +636,6 @@ public final class GlobalSettingsPanel extends IngestModuleGlobalSettingsPanel i private javax.swing.JCheckBox cbUseCentralRepo; private javax.swing.JScrollPane correlationPropertiesScrollPane; private javax.swing.JTextArea correlationPropertiesTextArea; - private javax.swing.JCheckBox flagTaggedNotableItemsCheckbox; - private javax.swing.JLabel generateInterestingItemsLabel; private javax.swing.JLabel ingestRunningWarningLabel; private javax.swing.JPanel jPanel1; private javax.swing.JScrollPane jScrollPane1; From 46d3e32d6da9d0f8ea719adcda6c38e076bfde5e Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Wed, 31 Oct 2018 16:42:58 -0400 Subject: [PATCH 27/44] 4305 restore previous version of cr ingestmodulefactory --- .../ingestmodule/IngestModuleFactory.java | 39 ++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModuleFactory.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModuleFactory.java index 8d6068a408..fc5dcd9c81 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModuleFactory.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModuleFactory.java @@ -26,6 +26,8 @@ import org.sleuthkit.autopsy.ingest.IngestModuleGlobalSettingsPanel; import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettings; import org.sleuthkit.autopsy.centralrepository.optionspanel.GlobalSettingsPanel; import org.sleuthkit.autopsy.coreutils.Version; +import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettingsPanel; +import org.sleuthkit.autopsy.ingest.NoIngestModuleIngestJobSettings; /** * Factory for Central Repository ingest modules @@ -66,7 +68,17 @@ public class IngestModuleFactory extends IngestModuleFactoryAdapter { @Override public FileIngestModule createFileIngestModule(IngestModuleIngestJobSettings settings) { - return new IngestModule(); + if (settings instanceof IngestSettings) { + return new IngestModule((IngestSettings) settings); + } + /* + * Compatibility check for older versions. + */ + if (settings instanceof NoIngestModuleIngestJobSettings) { + return new IngestModule(new IngestSettings()); + } + + throw new IllegalArgumentException("Expected settings argument to be an instance of IngestSettings"); } @Override @@ -80,5 +92,30 @@ public class IngestModuleFactory extends IngestModuleFactoryAdapter { globalOptionsPanel.load(); return globalOptionsPanel; } + + @Override + public IngestModuleIngestJobSettings getDefaultIngestJobSettings() { + return new IngestSettings(); + } + + @Override + public boolean hasIngestJobSettingsPanel() { + return true; + } + + @Override + public IngestModuleIngestJobSettingsPanel getIngestJobSettingsPanel(IngestModuleIngestJobSettings settings) { + if (settings instanceof IngestSettings) { + return new IngestSettingsPanel((IngestSettings) settings); + } + /* + * Compatibility check for older versions. + */ + if (settings instanceof NoIngestModuleIngestJobSettings) { + return new IngestSettingsPanel(new IngestSettings()); + } + + throw new IllegalArgumentException("Expected settings argument to be an instance of IngestSettings"); + } } From 077d95456703300a054801f8f77655988d02cf04 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Thu, 1 Nov 2018 12:14:41 -0400 Subject: [PATCH 28/44] 4305 fix codacy issues --- .../eventlisteners/IngestEventsListener.java | 4 ++-- .../autopsy/centralrepository/ingestmodule/IngestModule.java | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java index 40dcd8338a..2f2a35ab2a 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java @@ -196,8 +196,8 @@ public class IngestEventsListener { } switch (IngestManager.IngestModuleEvent.valueOf(evt.getPropertyName())) { case DATA_ADDED: { - //if ingest isn't running create the interesting items - boolean flagNotable = IngestManager.getInstance().isIngestRunning() ? isFlagNotableItems() : true; + //if ingest isn't running create the interesting items otherwise use the ingest module setting to determine if we create interesting items + boolean flagNotable = !IngestManager.getInstance().isIngestRunning() || isFlagNotableItems(); jobProcessingExecutor.submit(new DataAddedTask(dbManager, evt, flagNotable)); break; } diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java index 1ee6dd8a68..7469e38bd0 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java @@ -42,7 +42,6 @@ import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationDataSource; import org.sleuthkit.autopsy.centralrepository.datamodel.EamDb; import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbPlatformEnum; import org.sleuthkit.autopsy.centralrepository.datamodel.EamArtifactUtil; -import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbUtil; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardAttribute; From 2272bba4df60adfcf5896b104d6fa4587a3ad933 Mon Sep 17 00:00:00 2001 From: millmanorama Date: Fri, 2 Nov 2018 09:11:27 +0100 Subject: [PATCH 29/44] only show regrouping progress in IG status bar ( not main autopsy one) --- .../imagegallery/ImageGalleryController.java | 51 +++++++++---------- .../datamodel/grouping/GroupManager.java | 43 +++++++--------- .../autopsy/imagegallery/gui/StatusBar.fxml | 51 +++++++++---------- .../autopsy/imagegallery/gui/StatusBar.java | 34 ++++++------- 4 files changed, 86 insertions(+), 93 deletions(-) diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java index 646affbad7..666a2cd02d 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java @@ -67,7 +67,6 @@ import org.sleuthkit.autopsy.imagegallery.datamodel.HashSetManager; import org.sleuthkit.autopsy.imagegallery.datamodel.grouping.GroupManager; import org.sleuthkit.autopsy.imagegallery.datamodel.grouping.GroupViewState; import org.sleuthkit.autopsy.ingest.IngestManager; -import org.sleuthkit.autopsy.modules.filetypeid.FileTypeDetector; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.DataSource; import org.sleuthkit.datamodel.SleuthkitCase; @@ -232,19 +231,19 @@ public final class ImageGalleryController { dbTaskQueueSize.addListener(obs -> this.updateRegroupDisabled()); } - + /** * @return Currently displayed group or null if nothing is being displayed */ public GroupViewState getViewState() { return historyManager.getCurrentState(); } - + /** * Get observable property of the current group. The UI currently changes - * based on this property changing, which happens when other actions and + * based on this property changing, which happens when other actions and * threads call advance(). - * + * * @return Currently displayed group (as a property that can be observed) */ public ReadOnlyObjectProperty viewStateProperty() { @@ -253,7 +252,8 @@ public final class ImageGalleryController { /** * Should the "forward" button on the history be enabled? - * @return + * + * @return */ public ReadOnlyBooleanProperty getCanAdvance() { return historyManager.getCanAdvance(); @@ -261,19 +261,19 @@ public final class ImageGalleryController { /** * Should the "Back" button on the history be enabled? - * @return + * + * @return */ public ReadOnlyBooleanProperty getCanRetreat() { return historyManager.getCanRetreat(); } /** - * Display the passed in group. Causes this group to - * get recorded in the history queue and observers of the - * current state will be notified and update their panels/widgets - * appropriately. - * - * @param newState + * Display the passed in group. Causes this group to get recorded in the + * history queue and observers of the current state will be notified and + * update their panels/widgets appropriately. + * + * @param newState */ @ThreadConfined(type = ThreadConfined.ThreadType.ANY) public void advance(GroupViewState newState) { @@ -282,7 +282,8 @@ public final class ImageGalleryController { /** * Display the next group in the "forward" history stack - * @return + * + * @return */ public GroupViewState advance() { return historyManager.advance(); @@ -290,7 +291,8 @@ public final class ImageGalleryController { /** * Display the previous group in the "back" history stack - * @return + * + * @return */ public GroupViewState retreat() { return historyManager.retreat(); @@ -433,10 +435,6 @@ public final class ImageGalleryController { return drawableDB.getFileFromID(fileID); } - public ReadOnlyDoubleProperty regroupProgress() { - return groupManager.regroupProgress(); - } - public HashSetManager getHashSetManager() { return hashSetManager; } @@ -696,16 +694,17 @@ public final class ImageGalleryController { // Cycle through all of the files returned and call processFile on each //do in transaction drawableDbTransaction = taskDB.beginTransaction(); - - /* We are going to periodically commit the CaseDB transaction and sleep so - * that the user can have Autopsy do other stuff while these bulk tasks are ongoing. + + /* We are going to periodically commit the CaseDB transaction + * and sleep so that the user can have Autopsy do other stuff + * while these bulk tasks are ongoing. */ int caseDbCounter = 0; for (final AbstractFile f : files) { if (caseDbTransaction == null) { caseDbTransaction = tskCase.beginTransaction(); } - + if (isCancelled() || Thread.interrupted()) { logger.log(Level.WARNING, "Task cancelled or interrupted: not all contents may be transfered to drawable database."); //NON-NLS taskCompletionStatus = false; @@ -720,7 +719,7 @@ public final class ImageGalleryController { progressHandle.progress(f.getName(), workDone); updateProgress(workDone - 1 / (double) files.size()); updateMessage(f.getName()); - + // Periodically, commit the transaction (which frees the lock) and sleep // to allow other threads to get some work done in CaseDB if ((++caseDbCounter % 200) == 0) { @@ -740,12 +739,12 @@ public final class ImageGalleryController { caseDbTransaction.commit(); caseDbTransaction = null; } - + // pass true so that groupmanager is notified of the changes taskDB.commitTransaction(drawableDbTransaction, true); drawableDbTransaction = null; - } catch (TskCoreException | InterruptedException ex) { + } catch (TskCoreException | InterruptedException ex) { progressHandle.progress(Bundle.BulkTask_stopCopy_status()); logger.log(Level.WARNING, "Stopping copy to drawable db task. Failed to transfer all database contents", ex); //NON-NLS MessageNotifyUtil.Notify.warn(Bundle.BulkTask_errPopulating_errMsg(), ex.getMessage()); diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupManager.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupManager.java index f25142b406..bfd0fab496 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupManager.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupManager.java @@ -50,6 +50,7 @@ import javafx.beans.property.ReadOnlyBooleanWrapper; import javafx.beans.property.ReadOnlyDoubleProperty; import javafx.beans.property.ReadOnlyObjectProperty; import javafx.beans.property.ReadOnlyObjectWrapper; +import javafx.beans.property.ReadOnlyStringProperty; import javafx.collections.FXCollections; import javafx.collections.ObservableList; import javafx.concurrent.Service; @@ -278,10 +279,10 @@ public class GroupManager { } /** - * Update unseenGroups list accordingly based on the current status - * of 'group'. Removes it if it is seen or adds it if it is unseen. - * - * @param group + * Update unseenGroups list accordingly based on the current status of + * 'group'. Removes it if it is seen or adds it if it is unseen. + * + * @param group */ synchronized private void updateUnSeenGroups(DrawableGroup group) { if (group.isSeen()) { @@ -505,6 +506,10 @@ public class GroupManager { return regrouper.progressProperty(); } + public ReadOnlyStringProperty regroupMessage() { + return regrouper.messageProperty(); + } + @Subscribe synchronized public void handleTagAdded(ContentTagAddedEvent evt) { GroupKey newGroupKey = null; @@ -619,7 +624,7 @@ public class GroupManager { * If the group is analyzed (or other criteria based on grouping) and should * be shown to the user, then add it to the appropriate data structures so * that it can be viewed. - * + * * @returns null if Group is not ready to be viewed */ synchronized private DrawableGroup popuplateIfAnalyzed(GroupKey groupKey, ReGroupTask task) { @@ -730,12 +735,7 @@ public class GroupManager { */ @SuppressWarnings({"unchecked", "rawtypes"}) @NbBundle.Messages({"# {0} - groupBy attribute Name", - "# {1} - sortBy name", - "# {2} - sort Order", - "ReGroupTask.displayTitle=regrouping files by {0} sorted by {1} in {2} order", - "# {0} - groupBy attribute Name", - "# {1} - atribute value", - "ReGroupTask.progressUpdate=regrouping files by {0} : {1}"}) + "ReGroupTask.displayTitle=regrouping by {0}: " }) class ReGroupTask> extends LoggedTask { private final DataSource dataSource; @@ -743,16 +743,14 @@ public class GroupManager { private final GroupSortBy sortBy; private final SortOrder sortOrder; - private final ProgressHandle groupProgress; - ReGroupTask(DataSource dataSource, DrawableAttribute groupBy, GroupSortBy sortBy, SortOrder sortOrder) { - super(Bundle.ReGroupTask_displayTitle(groupBy.attrName.toString(), sortBy.getDisplayName(), sortOrder.toString()), true); + super(Bundle.ReGroupTask_displayTitle(groupBy.attrName.toString() ), true); this.dataSource = dataSource; this.groupBy = groupBy; this.sortBy = sortBy; this.sortOrder = sortOrder; - groupProgress = ProgressHandle.createHandle(Bundle.ReGroupTask_displayTitle(groupBy.attrName.toString(), sortBy.getDisplayName(), sortOrder.toString()), this); + updateTitle(Bundle.ReGroupTask_displayTitle(groupBy.attrName.toString() )); } @Override @@ -761,7 +759,8 @@ public class GroupManager { if (isCancelled()) { return null; } - groupProgress.start(); + + updateProgress(-1, 1); analyzedGroups.clear(); unSeenGroups.clear(); @@ -769,7 +768,7 @@ public class GroupManager { // Get the list of group keys Multimap valsByDataSource = findValuesForAttribute(); - groupProgress.switchToDeterminate(valsByDataSource.entries().size()); + updateProgress(0, valsByDataSource.entries().size()); int p = 0; // For each key value, partially create the group and add it to the list. for (final Map.Entry valForDataSource : valsByDataSource.entries()) { @@ -777,9 +776,8 @@ public class GroupManager { return null; } p++; - updateMessage(Bundle.ReGroupTask_progressUpdate(groupBy.attrName.toString(), valForDataSource.getValue())); + updateMessage(Bundle.ReGroupTask_displayTitle(groupBy.attrName.toString()) + valForDataSource.getValue()); updateProgress(p, valsByDataSource.size()); - groupProgress.progress(Bundle.ReGroupTask_progressUpdate(groupBy.attrName.toString(), valForDataSource), p); popuplateIfAnalyzed(new GroupKey<>(groupBy, valForDataSource.getValue(), valForDataSource.getKey()), this); } @@ -808,8 +806,8 @@ public class GroupManager { } } } finally { - groupProgress.finish(); updateProgress(1, 1); + updateMessage(""); } return null; } @@ -827,12 +825,9 @@ public class GroupManager { } /** - * find the distinct values for the given column (DrawableAttribute) - * + * Find the distinct values for the given column (DrawableAttribute). * These values represent the groups of files. * - * @param groupBy - * * @return map of data source (or null if group by attribute ignores * data sources) to list of unique group values */ diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/StatusBar.fxml b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/StatusBar.fxml index b02e2e2496..c0ee4488b1 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/StatusBar.fxml +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/StatusBar.fxml @@ -10,12 +10,21 @@ - + + @@ -31,37 +40,27 @@ - - - - - - - - - - - + + + + + + + + diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/StatusBar.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/StatusBar.java index 2355d6d2a0..d7224c4a58 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/StatusBar.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/StatusBar.java @@ -28,28 +28,26 @@ import javafx.scene.control.Tooltip; import javafx.scene.layout.AnchorPane; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.imagegallery.ImageGalleryController; +import org.sleuthkit.autopsy.imagegallery.datamodel.grouping.GroupManager; /** * */ public class StatusBar extends AnchorPane { - private final ImageGalleryController controller; - @FXML private ProgressBar fileTaskProgresBar; - @FXML private Label fileUpdateTaskLabel; - @FXML - private Label bgTaskLabel; - + private Label regroupLabel; @FXML private Label staleLabel; - @FXML - private ProgressBar bgTaskProgressBar; + private ProgressBar regroupProgressBar; + + private final ImageGalleryController controller; + private final GroupManager groupManager; @FXML @NbBundle.Messages({"StatusBar.fileUpdateTaskLabel.text= File Update Tasks", @@ -58,23 +56,25 @@ public class StatusBar extends AnchorPane { void initialize() { assert fileTaskProgresBar != null : "fx:id=\"fileTaskProgresBar\" was not injected: check your FXML file 'StatusBar.fxml'."; assert fileUpdateTaskLabel != null : "fx:id=\"fileUpdateTaskLabel\" was not injected: check your FXML file 'StatusBar.fxml'."; - assert bgTaskLabel != null : "fx:id=\"bgTaskLabel\" was not injected: check your FXML file 'StatusBar.fxml'."; - assert bgTaskProgressBar != null : "fx:id=\"bgTaskProgressBar\" was not injected: check your FXML file 'StatusBar.fxml'."; + assert regroupLabel != null : "fx:id=\"regroupLabel\" was not injected: check your FXML file 'StatusBar.fxml'."; + assert regroupProgressBar != null : "fx:id=\"regroupProgressBar\" was not injected: check your FXML file 'StatusBar.fxml'."; fileUpdateTaskLabel.textProperty().bind(controller.getDBTasksQueueSizeProperty().asString().concat(Bundle.StatusBar_fileUpdateTaskLabel_text())); fileTaskProgresBar.progressProperty().bind(controller.getDBTasksQueueSizeProperty().negate()); - controller.regroupProgress().addListener((ov, oldSize, newSize) -> { + groupManager.regroupProgress().addListener((ov, oldSize, newSize) -> { Platform.runLater(() -> { - if (controller.regroupProgress().lessThan(1.0).get()) { + if (groupManager.regroupProgress().lessThan(1.0).get()) { // Regrouping in progress - bgTaskProgressBar.progressProperty().setValue(-1.0); - bgTaskLabel.setText(Bundle.StatusBar_bgTaskLabel_text()); + regroupProgressBar.progressProperty().setValue(groupManager.regroupProgress().doubleValue()); + regroupLabel.setText(groupManager.regroupMessage().get()); + } else { // Clear the progress bar - bgTaskProgressBar.progressProperty().setValue(0.0); - bgTaskLabel.setText(""); + regroupProgressBar.progressProperty().setValue(0.0); + regroupLabel.setText(""); } + regroupLabel.setTooltip(new Tooltip(regroupLabel.getText())); }); }); @@ -84,6 +84,7 @@ public class StatusBar extends AnchorPane { public StatusBar(ImageGalleryController controller) { this.controller = controller; + this.groupManager = controller.getGroupManager(); FXMLLoader fxmlLoader = new FXMLLoader(getClass().getResource("StatusBar.fxml")); //NON-NLS fxmlLoader.setRoot(this); fxmlLoader.setController(this); @@ -93,6 +94,5 @@ public class StatusBar extends AnchorPane { } catch (IOException exception) { throw new RuntimeException(exception); } - } } From cb8021892b16f2680f57d8f550b17048017afd15 Mon Sep 17 00:00:00 2001 From: millmanorama Date: Fri, 2 Nov 2018 13:08:28 +0100 Subject: [PATCH 30/44] push isImageGalleryOpen() onto the swing EDT. --- .../imagegallery/ImageGalleryController.java | 1 - .../imagegallery/ImageGalleryModule.java | 42 ++++++++++--------- 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java index 646affbad7..1a7cc096f5 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java @@ -67,7 +67,6 @@ import org.sleuthkit.autopsy.imagegallery.datamodel.HashSetManager; import org.sleuthkit.autopsy.imagegallery.datamodel.grouping.GroupManager; import org.sleuthkit.autopsy.imagegallery.datamodel.grouping.GroupViewState; import org.sleuthkit.autopsy.ingest.IngestManager; -import org.sleuthkit.autopsy.modules.filetypeid.FileTypeDetector; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.DataSource; import org.sleuthkit.datamodel.SleuthkitCase; diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryModule.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryModule.java index 228b3765e5..c0336b76b1 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryModule.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryModule.java @@ -154,14 +154,14 @@ public class ImageGalleryModule { IngestManager.getInstance().removeIngestModuleEventListener(this); return; } - + /* only process individual files in realtime on the node that is * running the ingest. on a remote node, image files are processed * enblock when ingest is complete */ if (((AutopsyEvent) evt).getSourceType() != AutopsyEvent.SourceType.LOCAL) { return; } - + // Bail out if the case is closed try { if (controller == null || Case.getCurrentCaseThrows() == null) { @@ -208,8 +208,8 @@ public class ImageGalleryModule { } } else if (IngestManager.IngestModuleEvent.valueOf(evt.getPropertyName()) == DATA_ADDED) { - ModuleDataEvent mde = (ModuleDataEvent)evt.getOldValue(); - + ModuleDataEvent mde = (ModuleDataEvent) evt.getOldValue(); + if (mde.getBlackboardArtifactType().getTypeID() == ARTIFACT_TYPE.TSK_METADATA_EXIF.getTypeID()) { DrawableDB drawableDB = controller.getDatabase(); if (mde.getArtifacts() != null) { @@ -288,13 +288,13 @@ public class ImageGalleryModule { break; case CONTENT_TAG_ADDED: final ContentTagAddedEvent tagAddedEvent = (ContentTagAddedEvent) evt; - + long objId = tagAddedEvent.getAddedTag().getContent().getId(); - + // update the cache DrawableDB drawableDB = controller.getDatabase(); drawableDB.addTagCache(objId); - + if (con.getDatabase().isInDB(objId)) { con.getTagsManager().fireTagAddedEvent(tagAddedEvent); } @@ -336,21 +336,23 @@ public class ImageGalleryModule { try { ImageGalleryController con = getController(); con.setStale(true); - if (con.isListeningEnabled() && ImageGalleryTopComponent.isImageGalleryOpen()) { + if (con.isListeningEnabled()) { SwingUtilities.invokeLater(() -> { - int showAnswer = JOptionPane.showConfirmDialog(ImageGalleryTopComponent.getTopComponent(), - Bundle.ImageGalleryController_dataSourceAnalyzed_confDlg_msg(), - Bundle.ImageGalleryController_dataSourceAnalyzed_confDlg_title(), - JOptionPane.YES_NO_CANCEL_OPTION, JOptionPane.WARNING_MESSAGE); + if (ImageGalleryTopComponent.isImageGalleryOpen()) { + int showAnswer = JOptionPane.showConfirmDialog(ImageGalleryTopComponent.getTopComponent(), + Bundle.ImageGalleryController_dataSourceAnalyzed_confDlg_msg(), + Bundle.ImageGalleryController_dataSourceAnalyzed_confDlg_title(), + JOptionPane.YES_NO_CANCEL_OPTION, JOptionPane.WARNING_MESSAGE); - switch (showAnswer) { - case JOptionPane.YES_OPTION: - con.rebuildDB(); - break; - case JOptionPane.NO_OPTION: - case JOptionPane.CANCEL_OPTION: - default: - break; //do nothing + switch (showAnswer) { + case JOptionPane.YES_OPTION: + con.rebuildDB(); + break; + case JOptionPane.NO_OPTION: + case JOptionPane.CANCEL_OPTION: + default: + break; //do nothing + } } }); } From 559a963a0bbbc51aa74984dce249d03ca13facf6 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Mon, 5 Nov 2018 02:41:50 -0500 Subject: [PATCH 31/44] Added handling of NSArray data. --- .../autopsy/contentviewers/PListViewer.java | 31 +++++++++++++------ 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/contentviewers/PListViewer.java b/Core/src/org/sleuthkit/autopsy/contentviewers/PListViewer.java index 506084f0c3..dd73b6f6ca 100644 --- a/Core/src/org/sleuthkit/autopsy/contentviewers/PListViewer.java +++ b/Core/src/org/sleuthkit/autopsy/contentviewers/PListViewer.java @@ -74,7 +74,7 @@ class PListViewer extends javax.swing.JPanel implements FileTypeViewer, Explorer private final Outline outline; private ExplorerManager explorerManager; - private NSDictionary rootDict; + private NSObject rootDict; /** * Creates new form PListViewer @@ -415,22 +415,35 @@ class PListViewer extends javax.swing.JPanel implements FileTypeViewer, Explorer } /** - * Parses given binary stream and extracts Plist key/value + * Parses given binary stream and extracts Plist key/value. * - * @param plistbytes + * @param plistbytes The byte array containing the Plist data. * * @return list of PropKeyValue */ private List parsePList(final byte[] plistbytes) throws IOException, PropertyListFormatException, ParseException, ParserConfigurationException, SAXException { final List plist = new ArrayList<>(); - rootDict = (NSDictionary) PropertyListParser.parse(plistbytes); + rootDict = PropertyListParser.parse(plistbytes); - final String[] keys = rootDict.allKeys(); - for (final String key : keys) { - final PropKeyValue pkv = parseProperty(key, rootDict.objectForKey(key)); - if (null != pkv) { - plist.add(pkv); + /* + * Parse the data if the root is an NSArray or NSDictionary. Anything + * else is unexpected and will be ignored. + */ + if (rootDict instanceof NSArray) { + for (int i=0; i < ((NSArray)rootDict).count(); i++) { + final PropKeyValue pkv = parseProperty("", ((NSArray)rootDict).objectAtIndex(i)); + if (null != pkv) { + plist.add(pkv); + } + } + } else if (rootDict instanceof NSDictionary) { + final String[] keys = ((NSDictionary)rootDict).allKeys(); + for (final String key : keys) { + final PropKeyValue pkv = parseProperty(key, ((NSDictionary)rootDict).objectForKey(key)); + if (null != pkv) { + plist.add(pkv); + } } } From 1c925b65ec22f2c638b1e34ede50d7ff9a582766 Mon Sep 17 00:00:00 2001 From: Ann Priestman Date: Mon, 5 Nov 2018 09:40:59 -0500 Subject: [PATCH 32/44] Remove RecentActivity from pubilc packages --- RecentActivity/nbproject/project.xml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/RecentActivity/nbproject/project.xml b/RecentActivity/nbproject/project.xml index 7b7ae18347..020377beec 100644 --- a/RecentActivity/nbproject/project.xml +++ b/RecentActivity/nbproject/project.xml @@ -64,9 +64,7 @@ - - org.sleuthkit.autopsy.recentactivity - + ext/gson-2.1.jar release/modules/ext/gson-2.1.jar From e5313083d817f68d3c5adb3668fdf016e8317f4b Mon Sep 17 00:00:00 2001 From: Ann Priestman Date: Mon, 5 Nov 2018 10:21:03 -0500 Subject: [PATCH 33/44] Moved extractDomain from RecentActivity into NetworkUtils --- .../autopsy/coreutils/NetworkUtils.java | 84 ++++++++++++++++++- .../autopsy/recentactivity/Chrome.java | 9 +- .../autopsy/recentactivity/ExtractIE.java | 3 +- .../autopsy/recentactivity/Firefox.java | 3 +- .../autopsy/recentactivity/Util.java | 77 ----------------- 5 files changed, 92 insertions(+), 84 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/coreutils/NetworkUtils.java b/Core/src/org/sleuthkit/autopsy/coreutils/NetworkUtils.java index af41bc1980..78547f8370 100644 --- a/Core/src/org/sleuthkit/autopsy/coreutils/NetworkUtils.java +++ b/Core/src/org/sleuthkit/autopsy/coreutils/NetworkUtils.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2012-2015 Basis Technology Corp. + * Copyright 2012-2018 Basis Technology Corp. * Contact: carrier sleuthkit org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,14 +18,22 @@ */ package org.sleuthkit.autopsy.coreutils; +import java.net.MalformedURLException; +import java.net.URL; import java.net.UnknownHostException; +import java.util.StringTokenizer; public class NetworkUtils { + + private NetworkUtils() { + } /** * Set the host name variable. Sometimes the network can be finicky, so the * answer returned by getHostName() could throw an exception or be null. * Have it read the environment variable if getHostName() is unsuccessful. + * + * @return the local host name */ public static String getLocalHostName() { String hostName = ""; @@ -41,4 +49,78 @@ public class NetworkUtils { } return hostName; } + + /** + * Attempt to manually extract the domain from a URL. + * + * @param url + * @return empty string if no domain could be found + */ + private static String getBaseDomain(String url) { + String host = null; + + //strip protocol + String cleanUrl = url.replaceFirst(".*:\\/\\/", ""); + + //strip after slashes + String dirToks[] = cleanUrl.split("\\/"); + if (dirToks.length > 0) { + host = dirToks[0]; + } else { + host = cleanUrl; + } + + //get the domain part from host (last 2) + StringTokenizer tok = new StringTokenizer(host, "."); + StringBuilder hostB = new StringBuilder(); + int toks = tok.countTokens(); + + for (int count = 0; count < toks; ++count) { + String part = tok.nextToken(); + int diff = toks - count; + if (diff < 3) { + hostB.append(part); + } + if (diff == 2) { + hostB.append("."); + } + } + + + String base = hostB.toString(); + // verify there are no special characters in there + if (base.matches(".*[~`!@#$%^&\\*\\(\\)\\+={}\\[\\];:\\?<>,/ ].*")) { + return ""; + } + return base; + } + + /** + * Attempt to extract the domain from a URL. + * Will start by using the built-in URL class, and if that fails will + * try to extract it manually. + * + * @param urlString The URL to extract the domain from + * @return empty string if no domain name was found + */ + public static String extractDomain(String urlString) { + if (urlString == null) { + return ""; + } + String result = ""; + + try { + URL url = new URL(urlString); + result = url.getHost(); + } catch (MalformedURLException ex) { + //do not log if not a valid URL - we will try to extract it ourselves + } + + //was not a valid URL, try a less picky method + if (result == null || result.trim().isEmpty()) { + return getBaseDomain(urlString); + } + return result; + } + } diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Chrome.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Chrome.java index 0f9a98cd88..69381d8a13 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Chrome.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Chrome.java @@ -39,6 +39,7 @@ import java.io.FileReader; import java.io.IOException; import org.sleuthkit.autopsy.casemodule.services.FileManager; import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.autopsy.coreutils.NetworkUtils; import org.sleuthkit.autopsy.ingest.IngestJobContext; import org.sleuthkit.autopsy.ingest.ModuleDataEvent; import org.sleuthkit.datamodel.AbstractFile; @@ -163,7 +164,7 @@ class Chrome extends Extract { NbBundle.getMessage(this.getClass(), "Chrome.moduleName"))); bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DOMAIN, NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"), - (Util.extractDomain((result.get("url").toString() != null) ? result.get("url").toString() : "")))); //NON-NLS + (NetworkUtils.extractDomain((result.get("url").toString() != null) ? result.get("url").toString() : "")))); //NON-NLS BlackboardArtifact bbart = this.addArtifact(ARTIFACT_TYPE.TSK_WEB_HISTORY, historyFile, bbattributes); if (bbart != null) { @@ -286,7 +287,7 @@ class Chrome extends Extract { } else { date = Long.valueOf(0); } - String domain = Util.extractDomain(url); + String domain = NetworkUtils.extractDomain(url); try { BlackboardArtifact bbart = bookmarkFile.newArtifact(ARTIFACT_TYPE.TSK_WEB_BOOKMARK); Collection bbattributes = new ArrayList<>(); @@ -496,7 +497,7 @@ class Chrome extends Extract { //bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_LAST_ACCESSED.getTypeID(), "Recent Activity", "Last Visited", time)); bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_ACCESSED, NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"), time)); - String domain = Util.extractDomain((result.get("url").toString() != null) ? result.get("url").toString() : ""); //NON-NLS + String domain = NetworkUtils.extractDomain((result.get("url").toString() != null) ? result.get("url").toString() : ""); //NON-NLS bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DOMAIN, NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"), domain)); bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PROG_NAME, @@ -590,7 +591,7 @@ class Chrome extends Extract { NbBundle.getMessage(this.getClass(), "Chrome.moduleName"))); bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL_DECODED, NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"), - (Util.extractDomain((result.get("origin_url").toString() != null) ? result.get("url").toString() : "")))); //NON-NLS + (NetworkUtils.extractDomain((result.get("origin_url").toString() != null) ? result.get("url").toString() : "")))); //NON-NLS bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_USER_NAME, NbBundle.getMessage(this.getClass(), "Chrome.parentModuleName"), ((result.get("username_value").toString() != null) ? result.get("username_value").toString().replaceAll("'", "''") : ""))); //NON-NLS diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractIE.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractIE.java index 90a0a4d7fa..29844b0827 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractIE.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractIE.java @@ -26,6 +26,7 @@ import java.io.BufferedReader; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.coreutils.ExecUtil; +import org.sleuthkit.autopsy.coreutils.NetworkUtils; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; @@ -609,6 +610,6 @@ class ExtractIE extends Extract { return null; } - return Util.extractDomain(url); + return NetworkUtils.extractDomain(url); } } diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Firefox.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Firefox.java index d1874d4010..6cb51795f3 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Firefox.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Firefox.java @@ -35,6 +35,7 @@ import java.util.logging.Level; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.casemodule.services.FileManager; import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.autopsy.coreutils.NetworkUtils; import org.sleuthkit.autopsy.datamodel.ContentUtils; import org.sleuthkit.autopsy.ingest.IngestJobContext; import org.sleuthkit.autopsy.ingest.IngestServices; @@ -669,6 +670,6 @@ class Firefox extends Extract { return null; } - return Util.extractDomain(url); + return NetworkUtils.extractDomain(url); } } diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Util.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Util.java index 9bff067394..fd9630cebd 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Util.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/Util.java @@ -84,83 +84,6 @@ class Util { } } - /** - * - * @param url - * @return empty string if no domain could be found - */ - private static String getBaseDomain(String url) { - String host = null; - - //strip protocol - String cleanUrl = url.replaceFirst(".*:\\/\\/", ""); - - //strip after slashes - String dirToks[] = cleanUrl.split("\\/"); - if (dirToks.length > 0) { - host = dirToks[0]; - } else { - host = cleanUrl; - } - - //get the domain part from host (last 2) - StringTokenizer tok = new StringTokenizer(host, "."); - StringBuilder hostB = new StringBuilder(); - int toks = tok.countTokens(); - - for (int count = 0; count < toks; ++count) { - String part = tok.nextToken(); - int diff = toks - count; - if (diff < 3) { - hostB.append(part); - } - if (diff == 2) { - hostB.append("."); - } - } - - - String base = hostB.toString(); - // verify there are no special characters in there - if (base.matches(".*[~`!@#$%^&\\*\\(\\)\\+={}\\[\\];:\\?<>,/ ].*")) { - return ""; - } - return base; - } - - /** - * - * @param value - * @return empty string if no domain name was found - */ - public static String extractDomain(String value) { - if (value == null) { - return ""; - - } - String result = ""; - // String domainPattern = "(\\w+)\\.(AC|AD|AE|AERO|AF|AG|AI|AL|AM|AN|AO|AQ|AR|ARPA|AS|ASIA|AT|AU|AW|AX|AZ|BA|BB|BD|BE|BF|BG|BH|BI|BIZ|BJ|BM|BN|BO|BR|BS|BT|BV|BW|BY|BZ|CA|CAT|CC|CD|CF|CG|CH|CI|CK|CL|CM|CN|CO|COM|COOP|CR|CU|CV|CW|CX|CY|CZ|DE|DJ|DK|DM|DO|DZ|EC|EDU|EE|EG|ER|ES|ET|EU|FI|FJ|FK|FM|FO|FR|GA|GB|GD|GE|GF|GG|GH|GI|GL|GM|GN|GOV|GP|GQ|GR|GS|GT|GU|GW|GY|HK|HM|HN|HR|HT|HU|ID|IE|IL|IM|IN|INFO|INT|IO|IQ|IR|IS|IT|JE|JM|JO|JOBS|JP|KE|KG|KH|KI|KM|KN|KP|KR|KW|KY|KZ|LA|LB|LC|LI|LK|LR|LS|LT|LU|LV|LY|MA|MC|MD|ME|MG|MH|MIL|MK|ML|MM|MN|MO|MOBI|MP|MQ|MR|MS|MT|MU|MUSEUM|MV|MW|MX|MY|MZ|NA|NAME|NC|NE|NET|NF|NG|NI|NL|NO|NP|NR|NU|NZ|OM|ORG|PA|PE|PF|PG|PH|PK|PL|PM|PN|PR|PRO|PS|PT|PW|PY|QA|RE|RO|RS|RU|RW|SA|SB|SC|SD|SE|SG|SH|SI|SJ|SK|SL|SM|SN|SO|SR|ST|SU|SV|SX|SY|SZ|TC|TD|TEL|TF|TG|TH|TJ|TK|TL|TM|TN|TO|TP|TR|TRAVEL|TT|TV|TW|TZ|UA|UG|UK|US|UY|UZ|VA|VC|VE|VG|VI|VN|VU|WF|WS|XXX|YE|YT|ZA|ZM|ZW(co\\.[a-z].))"; - // Pattern p = Pattern.compile(domainPattern,Pattern.CASE_INSENSITIVE); - // Matcher m = p.matcher(value); - // while (m.find()) { - // result = value.substring(m.start(0),m.end(0)); - // } - - try { - URL url = new URL(value); - result = url.getHost(); - } catch (MalformedURLException ex) { - //do not log if not a valid URL, and handle later - //Logger.getLogger(Util.class.getName()).log(Level.SEVERE, null, ex); - } - - //was not a valid URL, try a less picky method - if (result == null || result.trim().isEmpty()) { - return getBaseDomain(value); - } - return result; - } - public static String getFileName(String value) { String filename = ""; String filematch = "^([a-zA-Z]\\:)(\\\\[^\\\\/:*?<>\"|]*(? Date: Tue, 6 Nov 2018 14:17:54 -0500 Subject: [PATCH 34/44] Add stopwatch logging for case deletion --- .../sleuthkit/autopsy/casemodule/Case.java | 51 ++++++++++++++++--- .../autoingest/AutoIngestMonitor.java | 49 ++++++++++++++---- 2 files changed, 83 insertions(+), 17 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/Case.java b/Core/src/org/sleuthkit/autopsy/casemodule/Case.java index dcb2af3eef..eaaa8f9926 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/Case.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/Case.java @@ -116,6 +116,7 @@ import org.sleuthkit.datamodel.Report; import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskUnsupportedSchemaVersionException; +import org.sleuthkit.autopsy.coreutils.StopWatch; /** * An Autopsy case. Currently, only one case at a time may be open. @@ -707,11 +708,15 @@ public class Case { "Case.exceptionMessage.cannotGetLockToDeleteCase=Cannot delete case because it is open for another user or there is a problem with the coordination service." }) public static void deleteCase(CaseMetadata metadata) throws CaseActionException { + StopWatch stopWatch = new StopWatch(); + stopWatch.start(); synchronized (caseActionSerializationLock) { if (null != currentCase) { throw new CaseActionException(Bundle.Case_exceptionMessage_cannotDeleteCurrentCase()); } } + stopWatch.stop(); + logger.log(Level.INFO, String.format("Used %d s to acquire caseActionSerializationLock (Java monitor in Case class) for %s (%s) in %s", stopWatch.getElapsedTimeSecs(), metadata.getCaseDisplayName(), metadata.getCaseName(), metadata.getCaseDirectory())); /* * Set up either a GUI progress indicator without a cancel button (can't @@ -733,10 +738,19 @@ public class Case { * cannot be deleted if another node has it open. */ progressIndicator.progress(Bundle.Case_progressMessage_checkingForOtherUser()); + stopWatch.reset(); + stopWatch.start(); try (CoordinationService.Lock dirLock = CoordinationService.getInstance().tryGetExclusiveLock(CategoryNode.CASES, metadata.getCaseDirectory())) { - assert (null != dirLock); - deleteCase(metadata, progressIndicator); + stopWatch.stop(); + logger.log(Level.INFO, String.format("Used %d s to acquire case directory coordination service lock for %s (%s) in %s", stopWatch.getElapsedTimeSecs(), metadata.getCaseDisplayName(), metadata.getCaseName(), metadata.getCaseDirectory())); + if (dirLock != null) { + deleteCase(metadata, progressIndicator); + } else { + throw new CaseActionException(Bundle.Case_creationException_couldNotAcquireDirLock()); + } } catch (CoordinationServiceException ex) { + stopWatch.stop(); + logger.log(Level.INFO, String.format("Used %d s to fail to acquire case directory coordination service lock for %s (%s) in %s", stopWatch.getElapsedTimeSecs(), metadata.getCaseDisplayName(), metadata.getCaseName(), metadata.getCaseDirectory())); throw new CaseActionException(Bundle.Case_exceptionMessage_cannotGetLockToDeleteCase(), ex); } } @@ -946,11 +960,13 @@ public class Case { "Case.exceptionMessage.errorsDeletingCase=Errors occured while deleting the case. See the application log for details" }) private static void deleteCase(CaseMetadata metadata, ProgressIndicator progressIndicator) throws CaseActionException { + StopWatch stopWatch = new StopWatch(); boolean errorsOccurred = false; if (CaseType.MULTI_USER_CASE == metadata.getCaseType()) { /* * Delete the case database from the database server. */ + stopWatch.start(); try { progressIndicator.progress(Bundle.Case_progressMessage_deletingCaseDatabase()); CaseDbConnectionInfo db; @@ -960,10 +976,14 @@ public class Case { Statement statement = connection.createStatement();) { String deleteCommand = "DROP DATABASE \"" + metadata.getCaseDatabaseName() + "\""; //NON-NLS statement.execute(deleteCommand); + stopWatch.stop(); + logger.log(Level.INFO, String.format("Used %d s to delete case database for %s (%s) in %s", stopWatch.getElapsedTimeSecs(), metadata.getCaseDisplayName(), metadata.getCaseName(), metadata.getCaseDirectory())); } } catch (UserPreferencesException | ClassNotFoundException | SQLException ex) { logger.log(Level.SEVERE, String.format("Failed to delete case database %s for %s (%s) in %s", metadata.getCaseDatabaseName(), metadata.getCaseDisplayName(), metadata.getCaseName(), metadata.getCaseDirectory()), ex); errorsOccurred = true; + stopWatch.stop(); + logger.log(Level.INFO, String.format("Used %d s to fail delete case database for %s (%s) in %s", stopWatch.getElapsedTimeSecs(), metadata.getCaseDisplayName(), metadata.getCaseName(), metadata.getCaseDirectory())); } } @@ -973,10 +993,16 @@ public class Case { progressIndicator.progress(Bundle.Case_progressMessage_deletingTextIndex()); for (KeywordSearchService searchService : Lookup.getDefault().lookupAll(KeywordSearchService.class)) { try { + stopWatch.reset(); + stopWatch.start(); searchService.deleteTextIndex(metadata); + stopWatch.stop(); + logger.log(Level.INFO, String.format("Used %d s to delete text index for %s (%s) in %s", stopWatch.getElapsedTimeSecs(), metadata.getCaseDisplayName(), metadata.getCaseName(), metadata.getCaseDirectory())); } catch (KeywordSearchServiceException ex) { logger.log(Level.SEVERE, String.format("Failed to delete text index for %s (%s) in %s", metadata.getCaseDisplayName(), metadata.getCaseName(), metadata.getCaseDirectory()), ex); errorsOccurred = true; + stopWatch.stop(); + logger.log(Level.INFO, String.format("Used %d s to fail to delete text index for %s (%s) in %s", stopWatch.getElapsedTimeSecs(), metadata.getCaseDisplayName(), metadata.getCaseName(), metadata.getCaseDirectory())); } } @@ -984,9 +1010,16 @@ public class Case { * Delete the case directory. */ progressIndicator.progress(Bundle.Case_progressMessage_deletingCaseDirectory()); + stopWatch.reset(); + stopWatch.start(); if (!FileUtil.deleteDir(new File(metadata.getCaseDirectory()))) { logger.log(Level.SEVERE, String.format("Failed to delete case directory for %s (%s) in %s", metadata.getCaseDisplayName(), metadata.getCaseName(), metadata.getCaseDirectory())); errorsOccurred = true; + stopWatch.stop(); + logger.log(Level.INFO, String.format("Used %d s to fail to delete case directory for %s (%s) in %s", stopWatch.getElapsedTimeSecs(), metadata.getCaseDisplayName(), metadata.getCaseName(), metadata.getCaseDirectory())); + } else { + stopWatch.stop(); + logger.log(Level.INFO, String.format("Used %d s to delete case directory for %s (%s) in %s", stopWatch.getElapsedTimeSecs(), metadata.getCaseDisplayName(), metadata.getCaseName(), metadata.getCaseDirectory())); } /* @@ -1540,11 +1573,13 @@ public class Case { } /** - * Notifies case event subscribers that a central repository comment has been changed. - * + * Notifies case event subscribers that a central repository comment has + * been changed. + * * This should not be called from the event dispatch thread (EDT) - * - * @param contentId the objectId for the Content which has had its central repo comment changed + * + * @param contentId the objectId for the Content which has had its central + * repo comment changed * @param newComment the new value of the comment */ public void notifyCentralRepoCommentChanged(long contentId, String newComment) { @@ -1800,7 +1835,7 @@ public class Case { progressIndicator.progress(Bundle.Case_progressMessage_preparingToOpenCaseResources()); acquireSharedCaseDirLock(metadata.getCaseDirectory()); try (CoordinationService.Lock resourcesLock = acquireExclusiveCaseResourcesLock(metadata.getCaseDirectory())) { - assert (null != resourcesLock); + assert(resourcesLock != null); // Use reference to avoid compile time warning. open(isNewCase, progressIndicator); } catch (CaseActionException ex) { releaseSharedCaseDirLock(getMetadata().getCaseDirectory()); @@ -2375,7 +2410,7 @@ public class Case { * @throws CaseActionException with a user-friendly message if the lock * cannot be acquired. */ - @Messages({"Case.creationException.couldNotAcquireDirLock=Failed to get lock on case directory."}) + @Messages({"Case.creationException.couldNotAcquireDirLock=Failed to get lock on case directory"}) private void acquireSharedCaseDirLock(String caseDir) throws CaseActionException { try { caseDirLock = CoordinationService.getInstance().tryGetSharedLock(CategoryNode.CASES, caseDir, DIR_LOCK_TIMOUT_HOURS, TimeUnit.HOURS); diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestMonitor.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestMonitor.java index cfdf6cf973..7f80201f26 100644 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestMonitor.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestMonitor.java @@ -45,6 +45,7 @@ import org.sleuthkit.autopsy.coordinationservice.CoordinationService; import org.sleuthkit.autopsy.coordinationservice.CoordinationService.CoordinationServiceException; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.NetworkUtils; +import org.sleuthkit.autopsy.coreutils.StopWatch; import org.sleuthkit.autopsy.events.AutopsyEventException; import org.sleuthkit.autopsy.events.AutopsyEventPublisher; import org.sleuthkit.autopsy.experimental.autoingest.AutoIngestJob.ProcessingStatus; @@ -666,43 +667,73 @@ final class AutoIngestMonitor extends Observable implements PropertyChangeListen * @return A result code indicating success, partial success, or failure. */ CaseDeletionResult deleteCase(AutoIngestJob job) { + String caseName = job.getManifest().getCaseName(); + Path caseDirectoryPath = job.getCaseDirectoryPath(); + Path metadataFilePath = caseDirectoryPath.resolve(caseName + CaseMetadata.getFileExtension()); + StopWatch stopWatch = new StopWatch(); + stopWatch.start(); synchronized (jobsLock) { - String caseName = job.getManifest().getCaseName(); - Path metadataFilePath = job.getCaseDirectoryPath().resolve(caseName + CaseMetadata.getFileExtension()); - + stopWatch.stop(); + LOGGER.log(Level.INFO, String.format("Used %d s to acquire jobsLock (Java monitor in AutoIngestMonitor class) for case %s at %s", stopWatch.getElapsedTimeSecs(), caseName, caseDirectoryPath)); + stopWatch.reset(); + stopWatch.start(); try { CaseMetadata metadata = new CaseMetadata(metadataFilePath); + stopWatch.stop(); + LOGGER.log(Level.INFO, String.format("Used %d s to read case metadata for case %s at %s", stopWatch.getElapsedTimeSecs(), caseName, caseDirectoryPath)); + stopWatch.reset(); + stopWatch.start(); Case.deleteCase(metadata); - } catch (CaseMetadata.CaseMetadataException ex) { - LOGGER.log(Level.SEVERE, String.format("Failed to get case metadata file %s for case %s at %s", metadataFilePath.toString(), caseName, job.getCaseDirectoryPath().toString()), ex); + LOGGER.log(Level.SEVERE, String.format("Failed to read case metadata file %s for case %s at %s", metadataFilePath, caseName, caseDirectoryPath), ex); + stopWatch.stop(); + LOGGER.log(Level.INFO, String.format("Used %d s to fail to read case metadata file %s for case %s at %s", stopWatch.getElapsedTimeSecs(), metadataFilePath, caseName, caseDirectoryPath)); return CaseDeletionResult.FAILED; } catch (CaseActionException ex) { - LOGGER.log(Level.SEVERE, String.format("Failed to physically delete case %s at %s", caseName, job.getCaseDirectoryPath().toString()), ex); + LOGGER.log(Level.SEVERE, String.format("Failed to delete case %s at %s", caseName, caseDirectoryPath), ex); return CaseDeletionResult.FAILED; } // Update the state of completed jobs associated with this case to indicate // that the case has been deleted - for (AutoIngestJob completedJob : getCompletedJobs()) { + stopWatch.reset(); + stopWatch.start(); + List completedJobs = getCompletedJobs(); + stopWatch.stop(); + LOGGER.log(Level.INFO, String.format("Used %d s to get completed jobs listing for case %s at %s", stopWatch.getElapsedTimeSecs(), caseName, caseDirectoryPath)); + stopWatch.reset(); + stopWatch.start(); + for (AutoIngestJob completedJob : completedJobs) { if (caseName.equals(completedJob.getManifest().getCaseName())) { try { completedJob.setProcessingStatus(DELETED); AutoIngestJobNodeData nodeData = new AutoIngestJobNodeData(completedJob); coordinationService.setNodeData(CoordinationService.CategoryNode.MANIFESTS, completedJob.getManifest().getFilePath().toString(), nodeData.toArray()); } catch (CoordinationServiceException | InterruptedException ex) { - LOGGER.log(Level.SEVERE, String.format("Failed to update completed job node data for %s when deleting case %s", completedJob.getManifest().getFilePath().toString(), caseName), ex); + LOGGER.log(Level.SEVERE, String.format("Failed to update completed job node data for %s when deleting case %s at %s", completedJob.getManifest().getFilePath(), caseName, caseDirectoryPath), ex); + stopWatch.stop(); + LOGGER.log(Level.INFO, String.format("Used %d s to fail to update job node data for completed jobs for case %s at %s", stopWatch.getElapsedTimeSecs(), caseName, caseDirectoryPath)); return CaseDeletionResult.PARTIALLY_DELETED; } } } + stopWatch.stop(); + LOGGER.log(Level.INFO, String.format("Used %d s to update job node data for completed jobs for case %s at %s", stopWatch.getElapsedTimeSecs(), caseName, caseDirectoryPath)); // Remove jobs associated with this case from the completed jobs collection. - jobsSnapshot.completedJobs.removeIf((AutoIngestJob completedJob) + stopWatch.reset(); + stopWatch.start(); + completedJobs.removeIf((AutoIngestJob completedJob) -> completedJob.getManifest().getCaseName().equals(caseName)); + stopWatch.stop(); + LOGGER.log(Level.INFO, String.format("Used %d s to remove completed jobs for case %s at %s from current jobs snapshot", stopWatch.getElapsedTimeSecs(), caseName, caseDirectoryPath)); // Publish a message to update auto ingest nodes. + stopWatch.reset(); + stopWatch.start(); eventPublisher.publishRemotely(new AutoIngestCaseDeletedEvent(caseName, LOCAL_HOST_NAME, AutoIngestManager.getSystemUserNameProperty())); + stopWatch.stop(); + LOGGER.log(Level.INFO, String.format("Used %d s to publish job deletion event for case %s at %s", stopWatch.getElapsedTimeSecs(), caseName,caseDirectoryPath)); } return CaseDeletionResult.FULLY_DELETED; From 80ba826d1333d71941b34b66e0e6747def9a53ba Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Tue, 6 Nov 2018 14:54:07 -0500 Subject: [PATCH 35/44] Update build type to RELEASE for 4.9.1 release --- nbproject/project.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nbproject/project.properties b/nbproject/project.properties index 96658e548c..8f3f577177 100644 --- a/nbproject/project.properties +++ b/nbproject/project.properties @@ -6,8 +6,8 @@ app.name=${branding.token} ### if left unset, version will default to today's date app.version=4.9.1 ### build.type must be one of: DEVELOPMENT, RELEASE -#build.type=RELEASE -build.type=DEVELOPMENT +build.type=RELEASE +#build.type=DEVELOPMENT project.org.netbeans.progress=org-netbeans-api-progress project.org.sleuthkit.autopsy.experimental=Experimental From 5964efb5b53c826065151145b42637cdeb108c73 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Tue, 6 Nov 2018 14:58:27 -0500 Subject: [PATCH 36/44] Fix version number in 4.9.1 release notes in NEWS.txt --- NEWS.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/NEWS.txt b/NEWS.txt index f93db10f9c..6130ca50da 100644 --- a/NEWS.txt +++ b/NEWS.txt @@ -1,8 +1,8 @@ ----------------- VERSION 4.9.0 -------------- +---------------- VERSION 4.9.1 -------------- Bug Fixes: -- Fixed possible ingest deadlock from Image Gallery database inserts -- Image Gallery does not need lock on Case DB during pre-population, which makes UI more responsive -- Other misc Image Gallery fixes +- Fixed possible ingest deadlock from Image Gallery database inserts. +- Image Gallery does not need lock on Case DB during pre-population, which makes UI more responsive. +- Other misc Image Gallery fixes. ---------------- VERSION 4.9.0 -------------- From f78ac991970d4257575d1df3671185b30dcd4821 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Wed, 7 Nov 2018 15:04:04 -0500 Subject: [PATCH 37/44] 4305 remove changes for flag items global setting --- .../datamodel/EamDbUtil.java | 59 +++++-------------- 1 file changed, 14 insertions(+), 45 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDbUtil.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDbUtil.java index db6687222a..e30ed174eb 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDbUtil.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDbUtil.java @@ -39,7 +39,6 @@ public class EamDbUtil { private final static Logger LOGGER = Logger.getLogger(EamDbUtil.class.getName()); private static final String CENTRAL_REPO_NAME = "CentralRepository"; private static final String CENTRAL_REPO_USE_KEY = "db.useCentralRepo"; - private static final String CENTRAL_REPO_FLAG_ITEMS_KEY = "db.flagNotableItems"; private static final String DEFAULT_ORG_NAME = "Not Specified"; /** @@ -193,9 +192,9 @@ public class EamDbUtil { } /** - * Upgrade the current central reposity to the newest version. If the - * upgrade fails, the central repository will be disabled and the current - * settings will be cleared. + * Upgrade the current central reposity to the newest version. If the upgrade + * fails, the central repository will be disabled and the current settings + * will be cleared. * * @return true if the upgrade succeeds, false otherwise. */ @@ -203,7 +202,7 @@ public class EamDbUtil { if (!EamDb.isEnabled()) { return true; } - + CoordinationService.Lock lock = null; try { EamDb db = EamDb.getInstance(); @@ -219,23 +218,23 @@ public class EamDbUtil { LOGGER.log(Level.SEVERE, "Error updating central repository", ex); // Disable the central repo and clear the current settings. - try { + try{ if (null != EamDb.getInstance()) { EamDb.getInstance().shutdownConnections(); } - } catch (EamDbException ex2) { + } catch (EamDbException ex2){ LOGGER.log(Level.SEVERE, "Error shutting down central repo connection pool", ex); - } + } setUseCentralRepo(false); EamDbPlatformEnum.setSelectedPlatform(EamDbPlatformEnum.DISABLED.name()); EamDbPlatformEnum.saveSelectedPlatform(); - + return false; } finally { - if (lock != null) { - try { + if(lock != null){ + try{ lock.release(); - } catch (CoordinationServiceException ex) { + } catch (CoordinationServiceException ex){ LOGGER.log(Level.SEVERE, "Error releasing database lock", ex); } } @@ -256,7 +255,6 @@ public class EamDbUtil { * Check whether the given org is the default organization. * * @param org - * * @return true if it is the default org, false otherwise */ public static boolean isDefaultOrg(EamOrganization org) { @@ -267,7 +265,6 @@ public class EamDbUtil { * Add the default organization to the database * * @param conn - * * @return true if successful, false otherwise */ static boolean insertDefaultOrganization(Connection conn) { @@ -298,7 +295,7 @@ public class EamDbUtil { * If the Central Repos use has been enabled. * * @return true if the Central Repo may be configured, false if it should - * not be able to be + * not be able to be */ public static boolean useCentralRepo() { return Boolean.parseBoolean(ModuleSettings.getConfigSetting(CENTRAL_REPO_NAME, CENTRAL_REPO_USE_KEY)); @@ -309,40 +306,12 @@ public class EamDbUtil { * configured. * * @param centralRepoCheckBoxIsSelected - true if the central repo can be - * used + * used */ public static void setUseCentralRepo(boolean centralRepoCheckBoxIsSelected) { ModuleSettings.setConfigSetting(CENTRAL_REPO_NAME, CENTRAL_REPO_USE_KEY, Boolean.toString(centralRepoCheckBoxIsSelected)); } - /** - * If items previously tagged as notable should be flagged after adding them - * to the Central Repo. - * - * @return true if the previously tagged as notable items should be flagged, - * false if previously tagged as notable items should not be flagged - */ - public static boolean flagNotableItems() { - String flagNotableItems = ModuleSettings.getConfigSetting(CENTRAL_REPO_NAME, CENTRAL_REPO_FLAG_ITEMS_KEY); - if (flagNotableItems == null){ //getConfigSetting can return null if the setting did not exist - setFlagNotableItems(true); - return true; - } - return Boolean.parseBoolean(flagNotableItems); - } - - /** - * Saves the setting for whether items previously tagged as notable should - * be flagged after adding them to the Central Repo. - * - * @param flagNotableItems - true if the previously tagged as notable items - * should be flagged, false if previously tagged as - * notable items should not be flagged - */ - public static void setFlagNotableItems(boolean flagNotableItems) { - ModuleSettings.setConfigSetting(CENTRAL_REPO_NAME, CENTRAL_REPO_FLAG_ITEMS_KEY, Boolean.toString(flagNotableItems)); - } - /** * Use the current settings and the validation query to test the connection * to the database. @@ -396,7 +365,7 @@ public class EamDbUtil { * Close the prepared statement. * * @param preparedStatement The prepared statement to be closed. - * + * * @deprecated Use closeStatement() instead. * * @throws EamDbException From a694e7870185300294348f5847a3cce9c426bd67 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Wed, 7 Nov 2018 17:50:53 -0500 Subject: [PATCH 38/44] 4305 allow version to be updated if schema changes already present --- .../centralrepository/datamodel/AbstractSqlEamDb.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java index e239f0b987..e1e87024f0 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java @@ -2960,11 +2960,10 @@ abstract class AbstractSqlEamDb implements EamDb { resultSet.getString("poc_phone")); } - CorrelationCase eamCase = new CorrelationCase(resultSet.getInt("case_id"), resultSet.getString("case_uid"), eamOrg, resultSet.getString("case_name"), - resultSet.getString("creation_date"), resultSet.getString("case_number"), resultSet.getString("examiner_name"), + CorrelationCase eamCase = new CorrelationCase(resultSet.getInt("case_id"), resultSet.getString("case_uid"), eamOrg, resultSet.getString("case_name"), + resultSet.getString("creation_date"), resultSet.getString("case_number"), resultSet.getString("examiner_name"), resultSet.getString("examiner_email"), resultSet.getString("examiner_phone"), resultSet.getString("notes")); - return eamCase; } @@ -3119,7 +3118,7 @@ abstract class AbstractSqlEamDb implements EamDb { logger.log(Level.INFO, "Central Repository is of newer version than software creates"); return; } - + // Update from 1.0 to 1.1 if (dbSchemaVersion.compareTo(new CaseDbSchemaVersionNumber(1, 1)) < 0) { statement.execute("ALTER TABLE reference_sets ADD COLUMN known_status INTEGER;"); //NON-NLS @@ -3140,9 +3139,11 @@ abstract class AbstractSqlEamDb implements EamDb { final String addDataSourceIdIndexTemplate; final String addValueIndexTemplate; final String addKnownStatusIndexTemplate; + final String addAttributeSql; //get the data base specific code for creating a new _instance table switch (selectedPlatform) { case POSTGRESQL: + addAttributeSql = "INSERT INTO correlation_types(id, display_name, db_table_name, supported, enabled) VALUES (?, ?, ?, ?, ?) " + getConflictClause(); addSsidTableTemplate = PostgresEamDbSettings.getCreateArtifactInstancesTableTemplate(); addCaseIdIndexTemplate = PostgresEamDbSettings.getAddCaseIdIndexTemplate(); addDataSourceIdIndexTemplate = PostgresEamDbSettings.getAddDataSourceIdIndexTemplate(); @@ -3150,6 +3151,7 @@ abstract class AbstractSqlEamDb implements EamDb { addKnownStatusIndexTemplate = PostgresEamDbSettings.getAddKnownStatusIndexTemplate(); break; case SQLITE: + addAttributeSql = "INSERT OR IGNORE INTO correlation_types(id, display_name, db_table_name, supported, enabled) VALUES (?, ?, ?, ?, ?)"; addSsidTableTemplate = SqliteEamDbSettings.getCreateArtifactInstancesTableTemplate(); addCaseIdIndexTemplate = SqliteEamDbSettings.getAddCaseIdIndexTemplate(); addDataSourceIdIndexTemplate = SqliteEamDbSettings.getAddDataSourceIdIndexTemplate(); @@ -3162,7 +3164,6 @@ abstract class AbstractSqlEamDb implements EamDb { final String wirelessNetworsDbTableName = "wireless_networks"; final String wirelessNetworksTableInstanceName = wirelessNetworsDbTableName + "_instances"; //add the wireless_networks attribute to the correlation_types table - final String addAttributeSql = "INSERT INTO correlation_types(id, display_name, db_table_name, supported, enabled) VALUES (?, ?, ?, ?, ?)"; preparedStatement = conn.prepareStatement(addAttributeSql); preparedStatement.setInt(1, CorrelationAttributeInstance.SSID_TYPE_ID); preparedStatement.setString(2, Bundle.CorrelationType_SSID_displayName()); From b907d6db0eaf09b7fbbd8de02ee8855495e67682 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Thu, 8 Nov 2018 10:25:32 -0500 Subject: [PATCH 39/44] Change the build type back to DEVELOPMENT --- nbproject/project.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nbproject/project.properties b/nbproject/project.properties index 8f3f577177..96658e548c 100644 --- a/nbproject/project.properties +++ b/nbproject/project.properties @@ -6,8 +6,8 @@ app.name=${branding.token} ### if left unset, version will default to today's date app.version=4.9.1 ### build.type must be one of: DEVELOPMENT, RELEASE -build.type=RELEASE -#build.type=DEVELOPMENT +#build.type=RELEASE +build.type=DEVELOPMENT project.org.netbeans.progress=org-netbeans-api-progress project.org.sleuthkit.autopsy.experimental=Experimental From 3f2a748d9d48868c9774ad3ba328e72136710487 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Thu, 8 Nov 2018 15:43:32 -0500 Subject: [PATCH 40/44] Fixing unncessary class indirection --- .../SevenZipExtractor.java | 53 ++----------------- 1 file changed, 4 insertions(+), 49 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java index 0deea2baf0..d3cc8f159f 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java +++ b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java @@ -785,51 +785,6 @@ class SevenZipExtractor { .mapToInt(Integer::intValue) .toArray(); } - - /** - * Stream used to unpack the archive item to local file. This will be used when - * the 7ZIP bindings call getStream() on the StandardIArchiveExtractCallback. - */ - private final static class MutableEncodedFileOutputStream extends EncodedFileOutputStream - implements AutoCloseable { - - private static final int HEADER_LENGTH = 32; - private static final String HEADER = "TSK_CONTAINER_XOR1_xxxxxxxxxxxxx"; - - MutableEncodedFileOutputStream(String localAbsPath) throws IOException { - super(new FileOutputStream(localAbsPath), TskData.EncodingType.XOR1); - } - - /** - * Update the OutputStream to point to a new File. This method mutates the state so - * that there is no overhead of creating a new object and buffer. Additionally, the - * 7zip binding has a memory leak, so this prevents multiple streams from being created - * and never GC'd since the 7zip lib never frees a reference to them. - * - * @param localAbsPath Path to local file - * @throws IOException - */ - public void setNewOutputStream(String localAbsPath) throws IOException { - this.out.close(); - this.out = new FileOutputStream(localAbsPath); - writeHeader(); - } - - private byte[] getEncodedHeader() { - byte [] encHeader = new byte[HEADER_LENGTH]; - byte [] plainHeader = HEADER.getBytes(); - - for(int i = 0; i < HEADER_LENGTH; i++){ - encHeader[i] = ((byte)(plainHeader[i] ^ 0xca)); - } - return encHeader; - } - - private void writeHeader() throws IOException { - // We get the encoded header here so it will be in plaintext after encoding - write(getEncodedHeader(), 0, HEADER_LENGTH); - } - } /** * UnpackStream used by the SevenZipBindings to do archive extraction. A memory @@ -841,18 +796,18 @@ class SevenZipExtractor { */ private final static class UnpackStream implements ISequentialOutStream { - private final MutableEncodedFileOutputStream output; + private EncodedFileOutputStream output; private String localAbsPath; private int bytesWritten; UnpackStream(String localAbsPath) throws IOException { - this.output = new MutableEncodedFileOutputStream(localAbsPath); + this.output = new EncodedFileOutputStream(localAbsPath); this.localAbsPath = localAbsPath; this.bytesWritten = 0; } public void setNewOutputStream(String localAbsPath) throws IOException { - this.output.setNewOutputStream(localAbsPath); + this.output = new EncodedFileOutputStream(localAbsPath); this.localAbsPath = localAbsPath; this.bytesWritten = 0; } @@ -876,7 +831,7 @@ class SevenZipExtractor { } public void close() throws IOException { - try(MutableEncodedFileOutputStream out = output) { + try(EncodedFileOutputStream out = output) { out.flush(); } } From 3040976ab81645ab670f0e49110a4c75d9fdf2c2 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Thu, 8 Nov 2018 15:55:11 -0500 Subject: [PATCH 41/44] Made only unpackstream wrap an encodedfileoutputstream --- .../modules/embeddedfileextractor/SevenZipExtractor.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java index dba3e82e81..c9cdf59d74 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java +++ b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java @@ -809,13 +809,14 @@ class SevenZipExtractor { private int bytesWritten; UnpackStream(String localAbsPath) throws IOException { - this.output = new EncodedFileOutputStream(localAbsPath); + this.output = new EncodedFileOutputStream(new FileOutputStream(localAbsPath), TskData.EncodingType.XOR1); this.localAbsPath = localAbsPath; this.bytesWritten = 0; } public void setNewOutputStream(String localAbsPath) throws IOException { - this.output = new EncodedFileOutputStream(localAbsPath); + this.output.close(); + this.output = new EncodedFileOutputStream(new FileOutputStream(localAbsPath), TskData.EncodingType.XOR1); this.localAbsPath = localAbsPath; this.bytesWritten = 0; } From e77faa4d2d011822c66a1951c760a775b1b9cc37 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Thu, 8 Nov 2018 15:57:16 -0500 Subject: [PATCH 42/44] Fixed a log message --- .../modules/embeddedfileextractor/SevenZipExtractor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java index c9cdf59d74..45963d7cdd 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java +++ b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java @@ -955,7 +955,7 @@ class SevenZipExtractor { } } catch (IOException ex) { logger.log(Level.WARNING, String.format("Error opening or setting new stream " //NON-NLS - + "for archive file at %s", localAbsPath), ex); //NON-NLS + + "for archive file at %s", localAbsPath), ex.getMessage()); //NON-NLS return null; } From 061a5debb1edbabdcc217c0d60e1ede7f09a0b15 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Fri, 9 Nov 2018 17:46:55 -0500 Subject: [PATCH 43/44] Add missing break to case statement for HTML Report generation --- Core/src/org/sleuthkit/autopsy/report/ReportHTML.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/report/ReportHTML.java b/Core/src/org/sleuthkit/autopsy/report/ReportHTML.java index 0d9210df27..13a5078658 100644 --- a/Core/src/org/sleuthkit/autopsy/report/ReportHTML.java +++ b/Core/src/org/sleuthkit/autopsy/report/ReportHTML.java @@ -272,7 +272,8 @@ class ReportHTML implements TableReportModule { in = getClass().getResourceAsStream("/org/sleuthkit/autopsy/report/images/accounts.png"); //NON-NLS break; case TSK_WIFI_NETWORK: - in = getClass().getResourceAsStream("/org/sleuthkit/autopsy/report/images/network-wifi.png"); //NON-NLS + in = getClass().getResourceAsStream("/org/sleuthkit/autopsy/report/images/network-wifi.png"); //NON-NLS + break; default: logger.log(Level.WARNING, "useDataTypeIcon: unhandled artifact type = {0}", dataType); //NON-NLS in = getClass().getResourceAsStream("/org/sleuthkit/autopsy/report/images/star.png"); //NON-NLS From ac54dee5a2d843cc24088016b009a3625d1ccae3 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Mon, 12 Nov 2018 13:09:29 -0500 Subject: [PATCH 44/44] Updated method header --- .../modules/embeddedfileextractor/SevenZipExtractor.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java index 45963d7cdd..c1aec6c81b 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java +++ b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java @@ -796,11 +796,10 @@ class SevenZipExtractor { /** * UnpackStream used by the SevenZipBindings to do archive extraction. A memory - * leak exists in the SevenZip library that will not let go of the Streams until - * the entire archive extraction is complete. So, to compensate this UnpackStream - * uses a MutableEncodedFileOutputStream, which supports setting a new disk - * location for the contents to be written to (as opposed to creating a new steam - * for each file). + * leak exists in the SevenZip library that will not let go of the streams until + * the entire archive extraction is complete. Instead of creating a new UnpackStream + * for every file in the archive, instead we just rebase our EncodedFileOutputStream pointer + * for every new file. */ private final static class UnpackStream implements ISequentialOutStream {