From c9913f24e8b54119aa38335e6eccb8f40e60b147 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Tue, 9 Oct 2018 15:05:42 -0400 Subject: [PATCH 001/129] Fixed the source of the memory issue and did some light refactoring --- .../SevenZipExtractor.java | 224 ++++++------------ 1 file changed, 74 insertions(+), 150 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java index 704bdac4f8..df208b14c6 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java +++ b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java @@ -19,6 +19,7 @@ package org.sleuthkit.autopsy.modules.embeddedfileextractor; import java.io.File; +import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStream; @@ -46,6 +47,7 @@ import net.sf.sevenzipjbinding.IArchiveExtractCallback; import net.sf.sevenzipjbinding.ICryptoGetTextPassword; import net.sf.sevenzipjbinding.PropID; import org.netbeans.api.progress.ProgressHandle; +import org.openide.util.Exceptions; import org.openide.util.NbBundle; import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.casemodule.Case; @@ -93,7 +95,7 @@ class SevenZipExtractor { private String moduleDirAbsolute; private Blackboard blackboard; - + private ProgressHandle progress; private int numItems; private String currentArchiveName; @@ -164,18 +166,19 @@ class SevenZipExtractor { * * More heuristics to be added here * - * @param archiveFile the AbstractFile for the parent archive which - * which we are checking - * @param inArchive The SevenZip archive currently open for extraction - * - * @param inArchiveItemIndex Index of item inside the SevenZip archive. Each - * file inside an archive is associated with a unique - * integer - * - * @param depthMap a concurrent hashmap which keeps track of the - * depth of all nested archives, key of objectID - * @param escapedFilePath the path to the archiveFileItem which has been - * escaped + * @param archiveFile the AbstractFile for the parent archive which + * which we are checking + * @param inArchive The SevenZip archive currently open for + * extraction + * + * @param inArchiveItemIndex Index of item inside the SevenZip archive. Each + * file inside an archive is associated with a + * unique integer + * + * @param depthMap a concurrent hashmap which keeps track of the + * depth of all nested archives, key of objectID + * @param escapedFilePath the path to the archiveFileItem which has been + * escaped * * @return true if potential zip bomb, false otherwise */ @@ -543,7 +546,7 @@ class SevenZipExtractor { logger.log(Level.INFO, "Count of items in archive: {0}: {1}", new Object[]{escapedArchiveFilePath, numItems}); //NON-NLS progress.start(numItems); progressStarted = true; - + //setup the archive local root folder final String uniqueArchiveFileName = FileUtil.escapeFileName(EmbeddedFileExtractorIngestModule.getUniqueName(archiveFile)); try { @@ -597,7 +600,7 @@ class SevenZipExtractor { inArchiveItemIndex, PropID.SIZE); if (freeDiskSpace != IngestMonitor.DISK_FREE_SPACE_UNKNOWN && archiveItemSize != null && archiveItemSize > 0) { //if free space is known and file is not empty. String archiveItemPath = (String) inArchive.getProperty( - inArchiveItemIndex, PropID.PATH); + inArchiveItemIndex, PropID.PATH); long newDiskSpace = freeDiskSpace - archiveItemSize; if (newDiskSpace < MIN_FREE_DISK_SPACE) { String msg = NbBundle.getMessage(SevenZipExtractor.class, @@ -669,7 +672,7 @@ class SevenZipExtractor { inArchive.extract(extractionIndices, false, archiveCallBack); unpackSuccessful = unpackSuccessful & archiveCallBack.wasSuccessful(); - + archiveDetailsMap = null; // add them to the DB. We wait until the end so that we have the metadata on all of the @@ -788,62 +791,22 @@ class SevenZipExtractor { .toArray(); } - /** - * Stream used to unpack the archive to local file - */ - private abstract static class UnpackStream implements ISequentialOutStream { - - private OutputStream output; - private String localAbsPath; - - UnpackStream(String localAbsPath) { - this.localAbsPath = localAbsPath; - try { - output = new EncodedFileOutputStream(new FileOutputStream(localAbsPath), TskData.EncodingType.XOR1); - } catch (IOException ex) { - logger.log(Level.SEVERE, "Error writing extracted file: " + localAbsPath, ex); //NON-NLS - } - - } - - public abstract long getSize(); - - OutputStream getOutput() { - return output; - } - - String getLocalAbsPath() { - return localAbsPath; - } - - public void close() { - if (output != null) { - try { - output.flush(); - output.close(); - output = null; - } catch (IOException e) { - logger.log(Level.SEVERE, "Error closing unpack stream for file: {0}", localAbsPath); //NON-NLS - } - } - } - } - /** * Stream used to unpack the archive of unknown size to local file */ - private static class UnknownSizeUnpackStream extends UnpackStream { + private static class UnpackStream implements ISequentialOutStream, AutoCloseable { - private long freeDiskSpace; - private boolean outOfSpace = false; - private long bytesWritten = 0; + private EncodedFileOutputStream output; + private String localAbsPath; - UnknownSizeUnpackStream(String localAbsPath, long freeDiskSpace) { - super(localAbsPath); - this.freeDiskSpace = freeDiskSpace; + private long bytesWritten; + + UnpackStream(String localAbsPath) throws IOException { + output = new EncodedFileOutputStream(new FileOutputStream(localAbsPath), TskData.EncodingType.XOR1); + this.localAbsPath = localAbsPath; + this.bytesWritten = 0; } - @Override public long getSize() { return this.bytesWritten; } @@ -851,78 +814,36 @@ class SevenZipExtractor { @Override public int write(byte[] bytes) throws SevenZipException { try { - // If the content size is unknown, cautiously write to disk. - // Write only if byte array is less than 80% of the current - // free disk space. - if (freeDiskSpace == IngestMonitor.DISK_FREE_SPACE_UNKNOWN || bytes.length < 0.8 * freeDiskSpace) { - getOutput().write(bytes); - // NOTE: this method is called multiple times for a - // single extractSlow() call. Update bytesWritten and - // freeDiskSpace after every write operation. - this.bytesWritten += bytes.length; - this.freeDiskSpace -= bytes.length; - } else { - this.outOfSpace = true; - logger.log(Level.INFO, NbBundle.getMessage( - SevenZipExtractor.class, - "EmbeddedFileExtractorIngestModule.ArchiveExtractor.UnpackStream.write.noSpace.msg")); - throw new SevenZipException( - NbBundle.getMessage(SevenZipExtractor.class, "EmbeddedFileExtractorIngestModule.ArchiveExtractor.UnpackStream.write.noSpace.msg")); - } + output.write(bytes); + // Update bytesWritten + this.bytesWritten += bytes.length; } catch (IOException ex) { throw new SevenZipException( NbBundle.getMessage(SevenZipExtractor.class, "EmbeddedFileExtractorIngestModule.ArchiveExtractor.UnpackStream.write.exception.msg", - getLocalAbsPath()), ex); + localAbsPath), ex); } return bytes.length; } + public void setNewOutputStream(String localAbsPath) throws IOException { + this.output.setOutputStream(new FileOutputStream(localAbsPath), TskData.EncodingType.XOR1); + this.localAbsPath = localAbsPath; + this.bytesWritten = 0; + } + @Override public void close() { - if (getOutput() != null) { + if (output != null) { try { - getOutput().flush(); - getOutput().close(); - if (this.outOfSpace) { - Files.delete(Paths.get(getLocalAbsPath())); - } + output.flush(); + output.close(); } catch (IOException e) { - logger.log(Level.SEVERE, "Error closing unpack stream for file: {0}", getLocalAbsPath()); //NON-NLS + logger.log(Level.WARNING, "Error closing unpack stream for file: {0}", localAbsPath); //NON-NLS } } } } - /** - * Stream used to unpack the archive of known size to local file - */ - private static class KnownSizeUnpackStream extends UnpackStream { - - private long size; - - KnownSizeUnpackStream(String localAbsPath, long size) { - super(localAbsPath); - this.size = size; - } - - @Override - public long getSize() { - return this.size; - } - - @Override - public int write(byte[] bytes) throws SevenZipException { - try { - getOutput().write(bytes); - } catch (IOException ex) { - throw new SevenZipException( - NbBundle.getMessage(SevenZipExtractor.class, "EmbeddedFileExtractorIngestModule.ArchiveExtractor.UnpackStream.write.exception.msg", - getLocalAbsPath()), ex); - } - return bytes.length; - } - } - /** * Wrapper for necessary details used in StandardIArchiveExtractCallback */ @@ -965,9 +886,8 @@ class SevenZipExtractor { private UnpackStream unpackStream = null; private final Map archiveDetailsMap; private final ProgressHandle progressHandle; - + private int inArchiveItemIndex; - private final long freeDiskSpace; private long createTimeInSeconds; private long modTimeInSeconds; @@ -984,7 +904,6 @@ class SevenZipExtractor { String password, long freeDiskSpace) { this.inArchive = inArchive; - this.freeDiskSpace = freeDiskSpace; this.progressHandle = progressHandle; this.archiveFile = archiveFile; this.archiveDetailsMap = archiveDetailsMap; @@ -992,19 +911,21 @@ class SevenZipExtractor { } /** - * Get stream is called by the internal framework as it traverses - * the archive structure. The ISequentialOutStream is where the - * archive file contents will be expanded and written to the local disk. - * + * Get stream is called by the internal framework as it traverses the + * archive structure. The ISequentialOutStream is where the archive file + * contents will be expanded and written to the local disk. + * * Skips folders, as there is nothing to extract. - * - * @param inArchiveItemIndex current location of the - * @param mode Will always be EXTRACT + * + * @param inArchiveItemIndex current location of the + * @param mode Will always be EXTRACT + * * @return - * @throws SevenZipException + * + * @throws SevenZipException */ @Override - public ISequentialOutStream getStream(int inArchiveItemIndex, + public ISequentialOutStream getStream(int inArchiveItemIndex, ExtractAskMode mode) throws SevenZipException { this.inArchiveItemIndex = inArchiveItemIndex; @@ -1015,28 +936,31 @@ class SevenZipExtractor { return null; } - final Long archiveItemSize = (Long) inArchive.getProperty( - inArchiveItemIndex, PropID.SIZE); final String localAbsPath = archiveDetailsMap.get( inArchiveItemIndex).getLocalAbsPath(); - - if (archiveItemSize != null) { - unpackStream = new SevenZipExtractor.KnownSizeUnpackStream( - localAbsPath, archiveItemSize); - } else { - unpackStream = new SevenZipExtractor.UnknownSizeUnpackStream( - localAbsPath, freeDiskSpace); + + try { + if (unpackStream != null) { + unpackStream.setNewOutputStream(localAbsPath); + } else { + unpackStream = new UnpackStream(localAbsPath); + } + } catch (IOException ex) { + logger.log(Level.WARNING, String.format("Error opening or setting new stream " //NON-NLS + + "for archive file at %s", localAbsPath), ex); //NON-NLS + return null; } return unpackStream; } /** - * Retrieves the file metadata from the archive before extraction. + * Retrieves the file metadata from the archive before extraction. * Called after getStream. - * + * * @param mode Will always be EXTRACT. - * @throws SevenZipException + * + * @throws SevenZipException */ @Override public void prepareOperation(ExtractAskMode mode) throws SevenZipException { @@ -1053,18 +977,18 @@ class SevenZipExtractor { : writeTime.getTime() / 1000; accessTimeInSeconds = accessTime == null ? 0L : accessTime.getTime() / 1000; - + progressHandle.progress(archiveFile.getName() + ": " + (String) inArchive.getProperty(inArchiveItemIndex, PropID.PATH), inArchiveItemIndex); - + } /** * Updates the unpackedNode data in the tree after the archive has been - * expanded to local disk. + * expanded to local disk. * - * @param result - ExtractOperationResult + * @param result - ExtractOperationResult * * @throws SevenZipException */ @@ -1081,7 +1005,7 @@ class SevenZipExtractor { localRelPath); return; } - + final String localAbsPath = archiveDetailsMap.get( inArchiveItemIndex).getLocalAbsPath(); if (result != ExtractOperationResult.OK) { From c88ea15bbcedfd340a23ffb5672295a92f18cba0 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Tue, 9 Oct 2018 16:22:50 -0400 Subject: [PATCH 002/129] Added some comments and did some testing, everything looks good --- .../SevenZipExtractor.java | 23 +++++++++++++------ 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java index df208b14c6..977340600b 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java +++ b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java @@ -19,12 +19,8 @@ package org.sleuthkit.autopsy.modules.embeddedfileextractor; import java.io.File; -import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; -import java.io.OutputStream; -import java.nio.file.Files; -import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -47,7 +43,6 @@ import net.sf.sevenzipjbinding.IArchiveExtractCallback; import net.sf.sevenzipjbinding.ICryptoGetTextPassword; import net.sf.sevenzipjbinding.PropID; import org.netbeans.api.progress.ProgressHandle; -import org.openide.util.Exceptions; import org.openide.util.NbBundle; import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.casemodule.Case; @@ -792,7 +787,8 @@ class SevenZipExtractor { } /** - * Stream used to unpack the archive of unknown size to local file + * Stream used to unpack the archive item to local file. This will be used when + * the 7ZIP bindings call getStream() on the StandardIArchiveExtractCallback. */ private static class UnpackStream implements ISequentialOutStream, AutoCloseable { @@ -815,7 +811,6 @@ class SevenZipExtractor { public int write(byte[] bytes) throws SevenZipException { try { output.write(bytes); - // Update bytesWritten this.bytesWritten += bytes.length; } catch (IOException ex) { throw new SevenZipException( @@ -825,6 +820,15 @@ class SevenZipExtractor { return bytes.length; } + /** + * Update the OutputStream to point to a new File. This method mutates the state so + * that there is no overhead of creating a new object and buffer. Additionally, the + * 7zip binding has a memory leak, so this prevents multiple streams from being created + * and never GC'd since the 7zip lib never frees a reference to them. + * + * @param localAbsPath Path to local file + * @throws IOException + */ public void setNewOutputStream(String localAbsPath) throws IOException { this.output.setOutputStream(new FileOutputStream(localAbsPath), TskData.EncodingType.XOR1); this.localAbsPath = localAbsPath; @@ -939,6 +943,11 @@ class SevenZipExtractor { final String localAbsPath = archiveDetailsMap.get( inArchiveItemIndex).getLocalAbsPath(); + //If the Unpackstream has been allocated, then set the Outputstream + //to another file rather than creating a new unpack stream. The 7Zip + //binding has a memory leak, so creating new unpack streams will not be + //dereferenced. As a fix, we create one UnpackStream, and mutate its state, + //so that there only exists one 8192 byte buffer in memory per archive. try { if (unpackStream != null) { unpackStream.setNewOutputStream(localAbsPath); From 611353d8f1132755f00e925757cc38451be1061a Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Thu, 11 Oct 2018 16:36:13 -0400 Subject: [PATCH 003/129] Created a solution entirely within the EFE module and not changing any public API in datamodel --- .../SevenZipExtractor.java | 123 ++++++++++++------ 1 file changed, 80 insertions(+), 43 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java index 977340600b..c8f0fce062 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java +++ b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java @@ -43,6 +43,7 @@ import net.sf.sevenzipjbinding.IArchiveExtractCallback; import net.sf.sevenzipjbinding.ICryptoGetTextPassword; import net.sf.sevenzipjbinding.PropID; import org.netbeans.api.progress.ProgressHandle; +import org.openide.util.Exceptions; import org.openide.util.NbBundle; import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.casemodule.Case; @@ -790,34 +791,19 @@ class SevenZipExtractor { * Stream used to unpack the archive item to local file. This will be used when * the 7ZIP bindings call getStream() on the StandardIArchiveExtractCallback. */ - private static class UnpackStream implements ISequentialOutStream, AutoCloseable { + private final static class MutableEncodedFileOutputStream extends EncodedFileOutputStream + implements AutoCloseable { + + private static final int HEADER_LENGTH = 32; + private static final String HEADER = "TSK_CONTAINER_XOR1_xxxxxxxxxxxxx"; - private EncodedFileOutputStream output; - private String localAbsPath; - - private long bytesWritten; - - UnpackStream(String localAbsPath) throws IOException { - output = new EncodedFileOutputStream(new FileOutputStream(localAbsPath), TskData.EncodingType.XOR1); - this.localAbsPath = localAbsPath; - this.bytesWritten = 0; - } - - public long getSize() { - return this.bytesWritten; + MutableEncodedFileOutputStream(String localAbsPath) throws IOException { + super(new FileOutputStream(localAbsPath), TskData.EncodingType.XOR1); } @Override - public int write(byte[] bytes) throws SevenZipException { - try { - output.write(bytes); - this.bytesWritten += bytes.length; - } catch (IOException ex) { - throw new SevenZipException( - NbBundle.getMessage(SevenZipExtractor.class, "EmbeddedFileExtractorIngestModule.ArchiveExtractor.UnpackStream.write.exception.msg", - localAbsPath), ex); - } - return bytes.length; + public void write(byte[] bytes) throws IOException { + super.write(bytes); } /** @@ -830,22 +816,69 @@ class SevenZipExtractor { * @throws IOException */ public void setNewOutputStream(String localAbsPath) throws IOException { - this.output.setOutputStream(new FileOutputStream(localAbsPath), TskData.EncodingType.XOR1); + this.out.close(); + this.out = new FileOutputStream(localAbsPath); + writeHeader(); + } + + private byte[] getEncodedHeader() { + byte [] encHeader = new byte[HEADER_LENGTH]; + byte [] plainHeader = HEADER.getBytes(); + + for(int i = 0; i < HEADER_LENGTH; i++){ + encHeader[i] = ((byte)(plainHeader[i] ^ 0xca)); + } + return encHeader; + } + + private void writeHeader() throws IOException { + // We get the encoded header here so it will be in plaintext after encoding + write(getEncodedHeader(), 0, HEADER_LENGTH); + } + } + + private final static class UnpackStream implements ISequentialOutStream { + + private final MutableEncodedFileOutputStream output; + private String localAbsPath; + private int bytesWritten; + + UnpackStream(String localAbsPath) throws IOException { + this.output = new MutableEncodedFileOutputStream(localAbsPath); + this.localAbsPath = localAbsPath; + this.bytesWritten = 0; + } + + public void setNewOutputStream(String localAbsPath) throws IOException { + this.output.setNewOutputStream(localAbsPath); this.localAbsPath = localAbsPath; this.bytesWritten = 0; } - - @Override - public void close() { - if (output != null) { - try { - output.flush(); - output.close(); - } catch (IOException e) { - logger.log(Level.WARNING, "Error closing unpack stream for file: {0}", localAbsPath); //NON-NLS - } - } + + public int getSize() { + return bytesWritten; } + + @Override + public int write(byte[] bytes) throws SevenZipException { + try { + output.write(bytes); + this.bytesWritten += bytes.length; + } catch (IOException ex) { + throw new SevenZipException( + NbBundle.getMessage(SevenZipExtractor.class, + "EmbeddedFileExtractorIngestModule.ArchiveExtractor.UnpackStream.write.exception.msg", + localAbsPath), ex); + } + return bytes.length; + } + + public void close() throws IOException { + try(MutableEncodedFileOutputStream out = output) { + out.flush(); + } + } + } /** @@ -1028,7 +1061,11 @@ class SevenZipExtractor { !(Boolean) inArchive.getProperty(inArchiveItemIndex, PropID.IS_FOLDER), 0L, createTimeInSeconds, accessTimeInSeconds, modTimeInSeconds, localRelPath); - unpackStream.close(); + try { + unpackStream.close(); + } catch (IOException e) { + logger.log(Level.WARNING, "Error closing unpack stream for file: {0}", localAbsPath); //NON-NLS + } } @Override @@ -1139,9 +1176,9 @@ class SevenZipExtractor { */ List getRootFileObjects() { List ret = new ArrayList<>(); - for (UnpackedNode child : rootNode.getChildren()) { + rootNode.getChildren().forEach((child) -> { ret.add(child.getFile()); - } + }); return ret; } @@ -1153,17 +1190,17 @@ class SevenZipExtractor { */ List getAllFileObjects() { List ret = new ArrayList<>(); - for (UnpackedNode child : rootNode.getChildren()) { + rootNode.getChildren().forEach((child) -> { getAllFileObjectsRec(ret, child); - } + }); return ret; } private void getAllFileObjectsRec(List list, UnpackedNode parent) { list.add(parent.getFile()); - for (UnpackedNode child : parent.getChildren()) { + parent.getChildren().forEach((child) -> { getAllFileObjectsRec(list, child); - } + }); } /** From d00d67e8546e9782b60c29dd80d5310b86c0749d Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Tue, 16 Oct 2018 13:35:26 -0400 Subject: [PATCH 004/129] Added the framework for text translation services --- Core/nbproject/project.xml | 2 +- .../TextTranslationService.java | 60 +++++++++++++++++++ .../texttranslation/TextTranslator.java | 28 +++++++++ 3 files changed, 89 insertions(+), 1 deletion(-) create mode 100755 Core/src/org/sleuthkit/autopsy/texttranslation/TextTranslationService.java create mode 100755 Core/src/org/sleuthkit/autopsy/texttranslation/TextTranslator.java diff --git a/Core/nbproject/project.xml b/Core/nbproject/project.xml index d142e0b8c9..2c6b0fce9a 100644 --- a/Core/nbproject/project.xml +++ b/Core/nbproject/project.xml @@ -338,7 +338,7 @@ org.sleuthkit.autopsy.modules.vmextractor org.sleuthkit.autopsy.progress org.sleuthkit.autopsy.report - org.sleuthkit.autopsy.tabulardatareader + org.sleuthkit.autopsy.texttranslation org.sleuthkit.datamodel diff --git a/Core/src/org/sleuthkit/autopsy/texttranslation/TextTranslationService.java b/Core/src/org/sleuthkit/autopsy/texttranslation/TextTranslationService.java new file mode 100755 index 0000000000..16bf9b0914 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/texttranslation/TextTranslationService.java @@ -0,0 +1,60 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2018-2018 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.texttranslation; + +import java.util.Optional; +import org.openide.util.Lookup; + +/** + * Service for finding and running TextTranslator implementations + */ +public class TextTranslationService { + + /** + * Performs a lookup for a TextTranslator service provider and if present, + * will use this provider to run translation on the input. + * + * @param input Input string to be translated + * + * @return Translation string + * + * @throws NoServiceProviderException Failed to find a Translation service + * provider + */ + public String translate(String input) throws NoServiceProviderException { + Optional translator = Optional.of(Lookup.getDefault() + .lookup(TextTranslator.class)); + if (translator.isPresent()) { + return translator.get().translate(input); + } + throw new NoServiceProviderException( + "Could not find a TextTranslator service provider"); + } + + /** + * Exception to indicate that no Service Provider could be found during the + * Lookup action. + */ + public class NoServiceProviderException extends Exception { + + public NoServiceProviderException(String msg) { + super(msg); + } + } +} \ No newline at end of file diff --git a/Core/src/org/sleuthkit/autopsy/texttranslation/TextTranslator.java b/Core/src/org/sleuthkit/autopsy/texttranslation/TextTranslator.java new file mode 100755 index 0000000000..3dbb98d50d --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/texttranslation/TextTranslator.java @@ -0,0 +1,28 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2018-2018 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.texttranslation; + +/** + * Interface for creating text translators. Implementing classes will be picked + * up and run by the Text Translation Service. + */ +public interface TextTranslator { + + public String translate(String input); +} From 87a145d74c6141baae1eb2684339ef8ebd0033d2 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Tue, 16 Oct 2018 14:50:01 -0400 Subject: [PATCH 005/129] Created a system exception for translation errors for implementing classes --- .../texttranslation/TextTranslator.java | 2 +- .../texttranslation/TranslationException.java | 51 +++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) create mode 100755 Core/src/org/sleuthkit/autopsy/texttranslation/TranslationException.java diff --git a/Core/src/org/sleuthkit/autopsy/texttranslation/TextTranslator.java b/Core/src/org/sleuthkit/autopsy/texttranslation/TextTranslator.java index 3dbb98d50d..86683d699e 100755 --- a/Core/src/org/sleuthkit/autopsy/texttranslation/TextTranslator.java +++ b/Core/src/org/sleuthkit/autopsy/texttranslation/TextTranslator.java @@ -24,5 +24,5 @@ package org.sleuthkit.autopsy.texttranslation; */ public interface TextTranslator { - public String translate(String input); + public String translate(String input) throws TranslationException; } diff --git a/Core/src/org/sleuthkit/autopsy/texttranslation/TranslationException.java b/Core/src/org/sleuthkit/autopsy/texttranslation/TranslationException.java new file mode 100755 index 0000000000..9c03f322dd --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/texttranslation/TranslationException.java @@ -0,0 +1,51 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2018-2018 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.texttranslation; + +/** + * Provides a system exception for the Text Translation errors + */ +public class TranslationException extends Exception { + + /** + * Constructs a new exception with null as its message. + */ + public TranslationException() { + super(); + } + + /** + * Constructs a new exception with the specified message. + * + * @param message The message. + */ + public TranslationException(String message) { + super(message); + } + + /** + * Constructs a new exception with the specified message and cause. + * + * @param message The message. + * @param cause The cause. + */ + public TranslationException(String message, Throwable cause) { + super(message, cause); + } +} From bc9a94ab79249507c58635b3c8b0ded37dd9f27e Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Fri, 19 Oct 2018 08:50:31 -0400 Subject: [PATCH 006/129] comment fix --- .../autopsy/texttranslation/ServiceTest.java | 20 +++++++++++++++++++ .../TextTranslationService.java | 9 ++++++--- 2 files changed, 26 insertions(+), 3 deletions(-) create mode 100755 Core/src/org/sleuthkit/autopsy/texttranslation/ServiceTest.java diff --git a/Core/src/org/sleuthkit/autopsy/texttranslation/ServiceTest.java b/Core/src/org/sleuthkit/autopsy/texttranslation/ServiceTest.java new file mode 100755 index 0000000000..a3a2941989 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/texttranslation/ServiceTest.java @@ -0,0 +1,20 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ +package org.sleuthkit.autopsy.texttranslation; + +/** + * + * @author dsmyda + */ + +public class ServiceTest implements TextTranslator { + + @Override + public String translate(String input) throws TranslationException { + throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates. + } + +} diff --git a/Core/src/org/sleuthkit/autopsy/texttranslation/TextTranslationService.java b/Core/src/org/sleuthkit/autopsy/texttranslation/TextTranslationService.java index 16bf9b0914..d6cb5cdaa2 100755 --- a/Core/src/org/sleuthkit/autopsy/texttranslation/TextTranslationService.java +++ b/Core/src/org/sleuthkit/autopsy/texttranslation/TextTranslationService.java @@ -36,9 +36,12 @@ public class TextTranslationService { * * @throws NoServiceProviderException Failed to find a Translation service * provider + * @throws TranslationException System exception for classes to use + * when specific translation + * implementations fail */ - public String translate(String input) throws NoServiceProviderException { - Optional translator = Optional.of(Lookup.getDefault() + public String translate(String input) throws NoServiceProviderException, TranslationException { + Optional translator = Optional.ofNullable(Lookup.getDefault() .lookup(TextTranslator.class)); if (translator.isPresent()) { return translator.get().translate(input); @@ -57,4 +60,4 @@ public class TextTranslationService { super(msg); } } -} \ No newline at end of file +} From 1b78fa78a3eb48e29cdd62e93f7c4b4528502b97 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Fri, 19 Oct 2018 14:21:52 -0400 Subject: [PATCH 007/129] In progress of making a correct solution to handling properties in the UI... --- .../contentviewers/MessageContentViewer.java | 35 +- .../datamodel/AbstractAbstractFileNode.java | 349 +++++++++++++++--- .../datamodel/AbstractFsContentNode.java | 43 +-- .../autopsy/datamodel/LayoutFileNode.java | 38 -- .../autopsy/datamodel/LocalDirectoryNode.java | 53 --- .../autopsy/datamodel/LocalFileNode.java | 37 +- .../datamodel/VirtualDirectoryNode.java | 28 +- .../texttranslation/CustomFileProperty.java | 32 ++ .../autopsy/texttranslation/FileProperty.java | 46 +++ .../autopsy/texttranslation/ServiceTest.java | 20 - .../texttranslation/TranslationProperty.java | 36 ++ 11 files changed, 427 insertions(+), 290 deletions(-) create mode 100755 Core/src/org/sleuthkit/autopsy/texttranslation/CustomFileProperty.java create mode 100755 Core/src/org/sleuthkit/autopsy/texttranslation/FileProperty.java delete mode 100755 Core/src/org/sleuthkit/autopsy/texttranslation/ServiceTest.java create mode 100755 Core/src/org/sleuthkit/autopsy/texttranslation/TranslationProperty.java diff --git a/Core/src/org/sleuthkit/autopsy/contentviewers/MessageContentViewer.java b/Core/src/org/sleuthkit/autopsy/contentviewers/MessageContentViewer.java index e1565c3b89..be342e78ca 100644 --- a/Core/src/org/sleuthkit/autopsy/contentviewers/MessageContentViewer.java +++ b/Core/src/org/sleuthkit/autopsy/contentviewers/MessageContentViewer.java @@ -20,6 +20,7 @@ package org.sleuthkit.autopsy.contentviewers; import java.awt.Component; import java.util.Arrays; +import java.util.HashSet; import java.util.List; import java.util.Optional; import java.util.Set; @@ -721,33 +722,17 @@ public class MessageContentViewer extends javax.swing.JPanel implements DataCont @Override protected Sheet createSheet() { - Sheet sheet = new Sheet(); - Sheet.Set sheetSet = sheet.get(Sheet.PROPERTIES); - if (sheetSet == null) { - sheetSet = Sheet.createPropertiesSet(); - sheet.put(sheetSet); + Sheet sheetSet = super.createSheet(); + Set keepProps = new HashSet<>(Arrays.asList("Name" , "Size", "Mime Type", "Known")); + + for(PropertySet p : sheetSet.toArray()) { + if(keepProps.contains(p.getName())){ + continue; + } + sheetSet.remove(p.getName()); } - List tags = getContentTagsFromDatabase(); - AbstractFile file = getContent(); - sheetSet.put(new NodeProperty<>("Name", "Name", "Name", file.getName())); - - addScoreProperty(sheetSet, tags); - - CorrelationAttributeInstance correlationAttribute = null; - if (EamDbUtil.useCentralRepo() && UserPreferences.hideCentralRepoCommentsAndOccurrences()== false) { - correlationAttribute = getCorrelationAttributeInstance(); - } - addCommentProperty(sheetSet, tags, correlationAttribute); - - if (EamDbUtil.useCentralRepo() && UserPreferences.hideCentralRepoCommentsAndOccurrences()== false) { - addCountProperty(sheetSet, correlationAttribute); - } - sheetSet.put(new NodeProperty<>("Size", "Size", "Size", file.getSize())); - sheetSet.put(new NodeProperty<>("Mime Type", "Mime Type", "Mime Type", StringUtils.defaultString(file.getMIMEType()))); - sheetSet.put(new NodeProperty<>("Known", "Known", "Known", file.getKnown().getName())); - - return sheet; + return sheetSet; } } } diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java index 5f8ea3d63f..0e59136bdd 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java @@ -21,15 +21,21 @@ package org.sleuthkit.autopsy.datamodel; import java.beans.PropertyChangeEvent; import java.beans.PropertyChangeListener; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.EnumSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.logging.Level; import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Pair; import org.openide.nodes.Children; import org.openide.nodes.Sheet; +import org.openide.util.Lookup; import org.openide.util.NbBundle; import org.openide.util.WeakListeners; import org.sleuthkit.autopsy.casemodule.Case; @@ -43,19 +49,21 @@ import org.sleuthkit.autopsy.centralrepository.datamodel.EamArtifactUtil; import org.sleuthkit.autopsy.centralrepository.datamodel.EamDb; import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbException; import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbUtil; +import org.sleuthkit.autopsy.core.UserPreferences; import org.sleuthkit.autopsy.corecomponents.DataResultViewerTable.Score; import org.sleuthkit.autopsy.coreutils.Logger; -import static org.sleuthkit.autopsy.datamodel.AbstractAbstractFileNode.AbstractFilePropertyType.*; import static org.sleuthkit.autopsy.datamodel.Bundle.*; import org.sleuthkit.autopsy.corecomponents.DataResultViewerTable.HasCommentStatus; import org.sleuthkit.autopsy.ingest.IngestManager; import org.sleuthkit.autopsy.ingest.ModuleContentEvent; +import org.sleuthkit.autopsy.texttranslation.CustomFileProperty; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.ContentTag; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskData; +import org.sleuthkit.autopsy.texttranslation.FileProperty; /** * An abstract node that encapsulates AbstractFile data @@ -177,8 +185,35 @@ public abstract class AbstractAbstractFileNode extends A private void updateSheet() { this.setSheet(createSheet()); } - + + @Override + protected Sheet createSheet() { + Sheet sheet = super.createSheet(); + Sheet.Set sheetSet = sheet.get(Sheet.PROPERTIES); + if (sheetSet == null) { + sheetSet = Sheet.createPropertiesSet(); + sheet.put(sheetSet); + } + + Map map = new LinkedHashMap<>(); + fillPropertyMap(map, getContent()); + + for (Map.Entry entry : map.entrySet()) { + String desc = Bundle.AbstractFsContentNode_noDesc_text(); + FileProperty p = AbstractFilePropertyType.getPropertyFromDisplayName(entry.getKey()); + if(!p.getDescription(content).isEmpty()) { + desc = p.getDescription(content); + } + sheetSet.put(new NodeProperty<>(entry.getKey(), entry.getKey(), desc, entry.getValue())); + } + + return sheet; + } + @NbBundle.Messages({"AbstractAbstractFileNode.nameColLbl=Name", + "AbstractAbstractFileNode.createSheet.score.name=S", + "AbstractAbstractFileNode.createSheet.comment.name=C", + "AbstractAbstractFileNode.createSheet.count.name=O", "AbstractAbstractFileNode.locationColLbl=Location", "AbstractAbstractFileNode.modifiedTimeColLbl=Modified Time", "AbstractAbstractFileNode.changeTimeColLbl=Change Time", @@ -199,30 +234,223 @@ public abstract class AbstractAbstractFileNode extends A "AbstractAbstractFileNode.objectId=Object ID", "AbstractAbstractFileNode.mimeType=MIME Type", "AbstractAbstractFileNode.extensionColLbl=Extension"}) - public enum AbstractFilePropertyType { + public enum AbstractFilePropertyType implements FileProperty { - NAME(AbstractAbstractFileNode_nameColLbl()), - LOCATION(AbstractAbstractFileNode_locationColLbl()), - MOD_TIME(AbstractAbstractFileNode_modifiedTimeColLbl()), - CHANGED_TIME(AbstractAbstractFileNode_changeTimeColLbl()), - ACCESS_TIME(AbstractAbstractFileNode_accessTimeColLbl()), - CREATED_TIME(AbstractAbstractFileNode_createdTimeColLbl()), - SIZE(AbstractAbstractFileNode_sizeColLbl()), - FLAGS_DIR(AbstractAbstractFileNode_flagsDirColLbl()), - FLAGS_META(AbstractAbstractFileNode_flagsMetaColLbl()), - MODE(AbstractAbstractFileNode_modeColLbl()), - USER_ID(AbstractAbstractFileNode_useridColLbl()), - GROUP_ID(AbstractAbstractFileNode_groupidColLbl()), - META_ADDR(AbstractAbstractFileNode_metaAddrColLbl()), - ATTR_ADDR(AbstractAbstractFileNode_attrAddrColLbl()), - TYPE_DIR(AbstractAbstractFileNode_typeDirColLbl()), - TYPE_META(AbstractAbstractFileNode_typeMetaColLbl()), - KNOWN(AbstractAbstractFileNode_knownColLbl()), - MD5HASH(AbstractAbstractFileNode_md5HashColLbl()), - ObjectID(AbstractAbstractFileNode_objectId()), - MIMETYPE(AbstractAbstractFileNode_mimeType()), - EXTENSION(AbstractAbstractFileNode_extensionColLbl()); + NAME(AbstractAbstractFileNode_nameColLbl()) { + @Override + public Object getPropertyValue(AbstractFile content) { + return getContentDisplayName(content); + } + }, + SCORE(AbstractAbstractFileNode_createSheet_score_name()) { + Optional> result; + + private void initResult(AbstractFile content) { + scoreAndCommentTags = getContentTagsFromDatabase(content); + result = Optional.of(getScoreProperty(content, scoreAndCommentTags)); + } + + @Override + public Object getPropertyValue(AbstractFile content) { + if(!this.result.isPresent()){ + initResult(content); + } + return result.get().getRight(); + } + + @Override + public String getDescription(AbstractFile content) { + if(!this.result.isPresent()){ + initResult(content); + } + return result.get().getLeft(); + } + }, + COMMENT(AbstractAbstractFileNode_createSheet_comment_name()) { + Optional> result; + + private void initResult(AbstractFile content) { + scoreAndCommentTags = getContentTagsFromDatabase(content); + correlationAttribute = null; + if(EamDbUtil.useCentralRepo() && + !UserPreferences.hideCentralRepoCommentsAndOccurrences()){ + if (EamDbUtil.useCentralRepo() && UserPreferences.hideCentralRepoCommentsAndOccurrences() == false) { + correlationAttribute = getCorrelationAttributeInstance(content); + } + } + result = Optional.of(getCommentProperty(scoreAndCommentTags, correlationAttribute)); + } + + @Override + public Object getPropertyValue(AbstractFile content) { + if(!this.result.isPresent()){ + initResult(content); + } + return result.get().getRight(); + } + + @Override + public String getDescription(AbstractFile content) { + if(!this.result.isPresent()){ + initResult(content); + } + return result.get().getLeft(); + } + }, + COUNT(AbstractAbstractFileNode_createSheet_count_name()) { + Optional> result; + + private void initResult(AbstractFile content) { + result = Optional.of(getCountProperty(correlationAttribute)); + } + + + @Override + public Object getPropertyValue(AbstractFile content) { + if(!result.isPresent()){ + initResult(content); + } + return result.get().getRight(); + } + @Override + public boolean isDisabled() { + return !EamDbUtil.useCentralRepo() || + UserPreferences.hideCentralRepoCommentsAndOccurrences(); + } + + @Override + public String getDescription(AbstractFile content) { + if(!result.isPresent()){ + initResult(content); + } + return result.get().getLeft(); + } + + }, + LOCATION(AbstractAbstractFileNode_locationColLbl()) { + @Override + public Object getPropertyValue(AbstractFile content) { + return getContentPath(content); + } + }, + MOD_TIME(AbstractAbstractFileNode_modifiedTimeColLbl()) { + @Override + public Object getPropertyValue(AbstractFile content) { + return ContentUtils.getStringTime(content.getMtime(), content); + } + }, + CHANGED_TIME(AbstractAbstractFileNode_changeTimeColLbl()) { + @Override + public Object getPropertyValue(AbstractFile content) { + return ContentUtils.getStringTime(content.getCtime(), content); + } + }, + ACCESS_TIME(AbstractAbstractFileNode_accessTimeColLbl()) { + @Override + public Object getPropertyValue(AbstractFile content) { + return ContentUtils.getStringTime(content.getAtime(), content); + } + }, + CREATED_TIME(AbstractAbstractFileNode_createdTimeColLbl()) { + @Override + public Object getPropertyValue(AbstractFile content) { + return ContentUtils.getStringTime(content.getCrtime(), content); + } + }, + SIZE(AbstractAbstractFileNode_sizeColLbl()) { + @Override + public Object getPropertyValue(AbstractFile content) { + return content.getSize(); + } + }, + FLAGS_DIR(AbstractAbstractFileNode_flagsDirColLbl()) { + @Override + public Object getPropertyValue(AbstractFile content) { + return content.getDirFlagAsString(); + } + }, + FLAGS_META(AbstractAbstractFileNode_flagsMetaColLbl()) { + @Override + public Object getPropertyValue(AbstractFile content) { + return content.getMetaFlagsAsString(); + } + }, + MODE(AbstractAbstractFileNode_modeColLbl()) { + @Override + public Object getPropertyValue(AbstractFile content) { + return content.getModesAsString(); + } + }, + USER_ID(AbstractAbstractFileNode_useridColLbl()) { + @Override + public Object getPropertyValue(AbstractFile content) { + return content.getUid(); + } + }, + GROUP_ID(AbstractAbstractFileNode_groupidColLbl()) { + @Override + public Object getPropertyValue(AbstractFile content) { + return content.getGid(); + } + }, + META_ADDR(AbstractAbstractFileNode_metaAddrColLbl()) { + @Override + public Object getPropertyValue(AbstractFile content) { + return content.getMetaAddr(); + } + }, + ATTR_ADDR(AbstractAbstractFileNode_attrAddrColLbl()){ + @Override + public Object getPropertyValue(AbstractFile content) { + return content.getAttrType().getValue() + "-" + content.getAttributeId(); + } + }, + TYPE_DIR(AbstractAbstractFileNode_typeDirColLbl()){ + @Override + public Object getPropertyValue(AbstractFile content) { + return content.getDirType().getLabel(); + } + }, + TYPE_META(AbstractAbstractFileNode_typeMetaColLbl()){ + @Override + public Object getPropertyValue(AbstractFile content) { + return content.getMetaType().toString(); + } + }, + KNOWN(AbstractAbstractFileNode_knownColLbl()){ + @Override + public Object getPropertyValue(AbstractFile content) { + return content.getKnown().getName(); + } + }, + MD5HASH(AbstractAbstractFileNode_md5HashColLbl()){ + @Override + public Object getPropertyValue(AbstractFile content) { + return StringUtils.defaultString(content.getMd5Hash()); + } + }, + ObjectID(AbstractAbstractFileNode_objectId()){ + @Override + public Object getPropertyValue(AbstractFile content) { + return content.getId(); + } + }, + MIMETYPE(AbstractAbstractFileNode_mimeType()){ + @Override + public Object getPropertyValue(AbstractFile content) { + return StringUtils.defaultString(content.getMIMEType()); + } + }, + EXTENSION(AbstractAbstractFileNode_extensionColLbl()){ + @Override + public Object getPropertyValue(AbstractFile content) { + return content.getNameExtension(); + } + }; + + private static List scoreAndCommentTags; + private static CorrelationAttributeInstance correlationAttribute; final private String displayString; private AbstractFilePropertyType(String displayString) { @@ -233,6 +461,15 @@ public abstract class AbstractAbstractFileNode extends A public String toString() { return displayString; } + + public static FileProperty getPropertyFromDisplayName(String displayName) { + for(FileProperty p : AbstractFilePropertyType.values()) { + if(p.getPropertyName().equals(displayName)) { + return p; + } + } + return null; + } } /** @@ -243,27 +480,25 @@ public abstract class AbstractAbstractFileNode extends A * @param content The content to get properties for. */ static public void fillPropertyMap(Map map, AbstractFile content) { - map.put(NAME.toString(), getContentDisplayName(content)); - map.put(LOCATION.toString(), getContentPath(content)); - map.put(MOD_TIME.toString(), ContentUtils.getStringTime(content.getMtime(), content)); - map.put(CHANGED_TIME.toString(), ContentUtils.getStringTime(content.getCtime(), content)); - map.put(ACCESS_TIME.toString(), ContentUtils.getStringTime(content.getAtime(), content)); - map.put(CREATED_TIME.toString(), ContentUtils.getStringTime(content.getCrtime(), content)); - map.put(SIZE.toString(), content.getSize()); - map.put(FLAGS_DIR.toString(), content.getDirFlagAsString()); - map.put(FLAGS_META.toString(), content.getMetaFlagsAsString()); - map.put(MODE.toString(), content.getModesAsString()); - map.put(USER_ID.toString(), content.getUid()); - map.put(GROUP_ID.toString(), content.getGid()); - map.put(META_ADDR.toString(), content.getMetaAddr()); - map.put(ATTR_ADDR.toString(), content.getAttrType().getValue() + "-" + content.getAttributeId()); - map.put(TYPE_DIR.toString(), content.getDirType().getLabel()); - map.put(TYPE_META.toString(), content.getMetaType().toString()); - map.put(KNOWN.toString(), content.getKnown().getName()); - map.put(MD5HASH.toString(), StringUtils.defaultString(content.getMd5Hash())); - map.put(ObjectID.toString(), content.getId()); - map.put(MIMETYPE.toString(), StringUtils.defaultString(content.getMIMEType())); - map.put(EXTENSION.toString(), content.getNameExtension()); + //Load in our default properties. + ArrayList properties = new ArrayList<>(); + properties.addAll(Arrays.asList(AbstractFilePropertyType.values())); + + /* //Load in our custom properties + Optional> customProperties = + Optional.ofNullable(Lookup.getDefault().lookupAll(CustomFileProperty.class)); + if (customProperties.isPresent()) { + for(CustomFileProperty p : customProperties.get()) { + //Inject custom properties at the desired column positions + //Specified by the custom property itself. + properties.add(p.getColumnPosition(), p); + } + }*/ + + //Skip properties that are disabled, don't add them to the property map! + properties.stream().filter(p -> p.isDisabled()).forEach((p) -> { + map.put(p.getPropertyName(), p.getPropertyValue(content)); + }); } /** @@ -271,7 +506,7 @@ public abstract class AbstractAbstractFileNode extends A * * @return a list of tags that are associated with the file */ - protected final List getContentTagsFromDatabase() { + private static List getContentTagsFromDatabase(AbstractFile content) { List tags = new ArrayList<>(); try { tags.addAll(Case.getCurrentCaseThrows().getServices().getTagsManager().getContentTagsByContent(content)); @@ -281,7 +516,7 @@ public abstract class AbstractAbstractFileNode extends A return tags; } - protected final CorrelationAttributeInstance getCorrelationAttributeInstance() { + private static CorrelationAttributeInstance getCorrelationAttributeInstance(AbstractFile content) { CorrelationAttributeInstance correlationAttribute = null; if (EamDbUtil.useCentralRepo()) { correlationAttribute = EamArtifactUtil.getInstanceFromContent(content); @@ -299,9 +534,9 @@ public abstract class AbstractAbstractFileNode extends A * @param attribute the correlation attribute associated with this file, * null if central repo is not enabled */ - @NbBundle.Messages({"AbstractAbstractFileNode.createSheet.comment.name=C", + @NbBundle.Messages({ "AbstractAbstractFileNode.createSheet.comment.displayName=C"}) - protected final void addCommentProperty(Sheet.Set sheetSet, List tags, CorrelationAttributeInstance attribute) { + private static Pair getCommentProperty(List tags, CorrelationAttributeInstance attribute) { HasCommentStatus status = tags.size() > 0 ? HasCommentStatus.TAG_NO_COMMENT : HasCommentStatus.NO_COMMENT; @@ -319,8 +554,7 @@ public abstract class AbstractAbstractFileNode extends A status = HasCommentStatus.CR_COMMENT; } } - sheetSet.put(new NodeProperty<>(AbstractAbstractFileNode_createSheet_comment_name(), AbstractAbstractFileNode_createSheet_comment_displayName(), NO_DESCR, - status)); + return Pair.of(NO_DESCR, status); } /** @@ -331,14 +565,14 @@ public abstract class AbstractAbstractFileNode extends A * Sheet.get(Sheet.PROPERTIES) * @param tags the list of tags associated with the file */ - @NbBundle.Messages({"AbstractAbstractFileNode.createSheet.score.name=S", + @NbBundle.Messages({ "AbstractAbstractFileNode.createSheet.score.displayName=S", "AbstractAbstractFileNode.createSheet.notableFile.description=File recognized as notable.", "AbstractAbstractFileNode.createSheet.interestingResult.description=File has interesting result associated with it.", "AbstractAbstractFileNode.createSheet.taggedFile.description=File has been tagged.", "AbstractAbstractFileNode.createSheet.notableTaggedFile.description=File tagged with notable tag.", "AbstractAbstractFileNode.createSheet.noScore.description=No score"}) - protected final void addScoreProperty(Sheet.Set sheetSet, List tags) { + private static Pair getScoreProperty(AbstractFile content, List tags) { Score score = Score.NO_SCORE; String description = Bundle.AbstractAbstractFileNode_createSheet_noScore_description(); if (content.getKnown() == TskData.FileKnown.BAD) { @@ -364,16 +598,16 @@ public abstract class AbstractAbstractFileNode extends A } } } - sheetSet.put(new NodeProperty<>(Bundle.AbstractAbstractFileNode_createSheet_score_name(), Bundle.AbstractAbstractFileNode_createSheet_score_displayName(), description, score)); + return Pair.of(description, score); } - @NbBundle.Messages({"AbstractAbstractFileNode.createSheet.count.name=O", + @NbBundle.Messages({ "AbstractAbstractFileNode.createSheet.count.displayName=O", "AbstractAbstractFileNode.createSheet.count.noCentralRepo.description=Central repository was not enabled when this column was populated", "AbstractAbstractFileNode.createSheet.count.hashLookupNotRun.description=Hash lookup had not been run on this file when the column was populated", "# {0} - occuranceCount", "AbstractAbstractFileNode.createSheet.count.description=There were {0} datasource(s) found with occurances of the correlation value"}) - protected final void addCountProperty(Sheet.Set sheetSet, CorrelationAttributeInstance attribute) { + private static Pair getCountProperty(CorrelationAttributeInstance attribute) { Long count = -1L; //The column renderer will not display negative values, negative value used when count unavailble to preserve sorting String description = Bundle.AbstractAbstractFileNode_createSheet_count_noCentralRepo_description(); try { @@ -390,8 +624,7 @@ public abstract class AbstractAbstractFileNode extends A logger.log(Level.WARNING, "Unable to normalize data to get count of datasources with correlation attribute", ex); } - sheetSet.put( - new NodeProperty<>(Bundle.AbstractAbstractFileNode_createSheet_count_name(), Bundle.AbstractAbstractFileNode_createSheet_count_displayName(), description, count)); + return Pair.of(description, count); } /** diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractFsContentNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractFsContentNode.java index ef8e1d024e..7d18f2a0c7 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractFsContentNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractFsContentNode.java @@ -18,17 +18,10 @@ */ package org.sleuthkit.autopsy.datamodel; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; import org.openide.nodes.Sheet; import org.openide.util.NbBundle; -import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeInstance; -import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbUtil; -import org.sleuthkit.autopsy.core.UserPreferences; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.datamodel.AbstractFile; -import org.sleuthkit.datamodel.ContentTag; /** * Abstract class that implements the commonality between File and Directory @@ -69,44 +62,12 @@ public abstract class AbstractFsContentNode extends Abst @Override @NbBundle.Messages("AbstractFsContentNode.noDesc.text=no description") protected Sheet createSheet() { - Sheet sheet = super.createSheet(); - Sheet.Set sheetSet = sheet.get(Sheet.PROPERTIES); - if (sheetSet == null) { - sheetSet = Sheet.createPropertiesSet(); - sheet.put(sheetSet); - } - List tags = getContentTagsFromDatabase(); - Map map = new LinkedHashMap<>(); - fillPropertyMap(map, getContent()); - final String NO_DESCR = Bundle.AbstractFsContentNode_noDesc_text(); - //add the name property before the comment property to ensure it is first column - sheetSet.put(new NodeProperty<>(AbstractFilePropertyType.NAME.toString(), - AbstractFilePropertyType.NAME.toString(), - NO_DESCR, - getName())); - - addScoreProperty(sheetSet, tags); - - //add the comment property before the propertyMap to ensure it is early in column order - CorrelationAttributeInstance correlationAttribute = null; - if (EamDbUtil.useCentralRepo() && UserPreferences.hideCentralRepoCommentsAndOccurrences()== false) { - correlationAttribute = getCorrelationAttributeInstance(); - } - addCommentProperty(sheetSet, tags, correlationAttribute); - - if (EamDbUtil.useCentralRepo() && UserPreferences.hideCentralRepoCommentsAndOccurrences()== false) { - addCountProperty(sheetSet, correlationAttribute); - } - - for (AbstractFilePropertyType propType : AbstractFilePropertyType.values()) { - final String propString = propType.toString(); - sheetSet.put(new NodeProperty<>(propString, propString, NO_DESCR, map.get(propString))); - } + Sheet sheetSet = super.createSheet(); if (directoryBrowseMode) { sheetSet.put(new NodeProperty<>(HIDE_PARENT, HIDE_PARENT, HIDE_PARENT, HIDE_PARENT)); } - return sheet; + return sheetSet; } } diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/LayoutFileNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/LayoutFileNode.java index a33c5024aa..5895ccab6a 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/LayoutFileNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/LayoutFileNode.java @@ -75,44 +75,6 @@ public class LayoutFileNode extends AbstractAbstractFileNode { } } - @Override - protected Sheet createSheet() { - Sheet sheet = super.createSheet(); - Sheet.Set sheetSet = sheet.get(Sheet.PROPERTIES); - if (sheetSet == null) { - sheetSet = Sheet.createPropertiesSet(); - sheet.put(sheetSet); - } - - List tags = getContentTagsFromDatabase(); - - Map map = new LinkedHashMap<>(); - fillPropertyMap(map); - - sheetSet.put(new NodeProperty<>(NbBundle.getMessage(this.getClass(), "LayoutFileNode.createSheet.name.name"), - NbBundle.getMessage(this.getClass(), "LayoutFileNode.createSheet.name.displayName"), - NbBundle.getMessage(this.getClass(), "LayoutFileNode.createSheet.name.desc"), - getName())); - - addScoreProperty(sheetSet, tags); - - CorrelationAttributeInstance correlationAttribute = null; - if (EamDbUtil.useCentralRepo() && UserPreferences.hideCentralRepoCommentsAndOccurrences() == false) { - correlationAttribute = getCorrelationAttributeInstance(); - } - addCommentProperty(sheetSet, tags, correlationAttribute); - - if (EamDbUtil.useCentralRepo() && UserPreferences.hideCentralRepoCommentsAndOccurrences() == false) { - addCountProperty(sheetSet, correlationAttribute); - } - final String NO_DESCR = NbBundle.getMessage(this.getClass(), "LayoutFileNode.createSheet.noDescr.text"); - for (Map.Entry entry : map.entrySet()) { - sheetSet.put(new NodeProperty<>(entry.getKey(), entry.getKey(), NO_DESCR, entry.getValue())); - } - - return sheet; - } - @Override public T accept(ContentNodeVisitor visitor) { return visitor.visit(this); diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/LocalDirectoryNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/LocalDirectoryNode.java index f4146e7a55..411cd464ec 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/LocalDirectoryNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/LocalDirectoryNode.java @@ -18,15 +18,6 @@ */ package org.sleuthkit.autopsy.datamodel; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import org.openide.nodes.Sheet; -import org.openide.util.NbBundle; -import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeInstance; -import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbUtil; -import org.sleuthkit.autopsy.core.UserPreferences; -import org.sleuthkit.datamodel.ContentTag; import org.sleuthkit.datamodel.LocalDirectory; /** @@ -46,50 +37,6 @@ public class LocalDirectoryNode extends SpecialDirectoryNode { } - @Override - @NbBundle.Messages({ - "LocalDirectoryNode.createSheet.name.name=Name", - "LocalDirectoryNode.createSheet.name.displayName=Name", - "LocalDirectoryNode.createSheet.name.desc=no description", - "LocalDirectoryNode.createSheet.noDesc=no description"}) - protected Sheet createSheet() { - Sheet sheet = super.createSheet(); - Sheet.Set sheetSet = sheet.get(Sheet.PROPERTIES); - if (sheetSet == null) { - sheetSet = Sheet.createPropertiesSet(); - sheet.put(sheetSet); - } - - List tags = getContentTagsFromDatabase(); - sheetSet.put(new NodeProperty<>(Bundle.LocalDirectoryNode_createSheet_name_name(), - Bundle.LocalDirectoryNode_createSheet_name_displayName(), - Bundle.LocalDirectoryNode_createSheet_name_desc(), - getName())); - - addScoreProperty(sheetSet, tags); - - CorrelationAttributeInstance correlationAttribute = null; - if (EamDbUtil.useCentralRepo() && UserPreferences.hideCentralRepoCommentsAndOccurrences() == false) { - correlationAttribute = getCorrelationAttributeInstance(); - } - addCommentProperty(sheetSet, tags, correlationAttribute); - - if (EamDbUtil.useCentralRepo() && UserPreferences.hideCentralRepoCommentsAndOccurrences() == false) { - addCountProperty(sheetSet, correlationAttribute); - } - // At present, a LocalDirectory will never be a datasource - the top level of a logical - // file set is a VirtualDirectory - Map map = new LinkedHashMap<>(); - fillPropertyMap(map, getContent()); - - final String NO_DESCR = Bundle.LocalDirectoryNode_createSheet_noDesc(); - for (Map.Entry entry : map.entrySet()) { - sheetSet.put(new NodeProperty<>(entry.getKey(), entry.getKey(), NO_DESCR, entry.getValue())); - } - - return sheet; - } - @Override public T accept(ContentNodeVisitor visitor) { return visitor.visit(this); diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/LocalFileNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/LocalFileNode.java index 0f69666454..a314791fcb 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/LocalFileNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/LocalFileNode.java @@ -42,6 +42,7 @@ import org.sleuthkit.autopsy.directorytree.ExtractAction; import org.sleuthkit.autopsy.directorytree.NewWindowViewAction; import org.sleuthkit.autopsy.directorytree.ViewContextAction; import org.sleuthkit.autopsy.modules.embeddedfileextractor.ExtractArchiveWithPasswordAction; +import org.sleuthkit.autopsy.texttranslation.FileProperty; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.ContentTag; @@ -68,42 +69,6 @@ public class LocalFileNode extends AbstractAbstractFileNode { } - @Override - protected Sheet createSheet() { - Sheet sheet = super.createSheet(); - Sheet.Set sheetSet = sheet.get(Sheet.PROPERTIES); - if (sheetSet == null) { - sheetSet = Sheet.createPropertiesSet(); - sheet.put(sheetSet); - } - List tags = getContentTagsFromDatabase(); - Map map = new LinkedHashMap<>(); - fillPropertyMap(map, getContent()); - - sheetSet.put(new NodeProperty<>(NbBundle.getMessage(this.getClass(), "LocalFileNode.createSheet.name.name"), - NbBundle.getMessage(this.getClass(), "LocalFileNode.createSheet.name.displayName"), - NbBundle.getMessage(this.getClass(), "LocalFileNode.createSheet.name.desc"), - getName())); - - addScoreProperty(sheetSet, tags); - - CorrelationAttributeInstance correlationAttribute = null; - if (EamDbUtil.useCentralRepo() && UserPreferences.hideCentralRepoCommentsAndOccurrences() == false) { - correlationAttribute = getCorrelationAttributeInstance(); - } - addCommentProperty(sheetSet, tags, correlationAttribute); - - if (EamDbUtil.useCentralRepo() && UserPreferences.hideCentralRepoCommentsAndOccurrences() == false) { - addCountProperty(sheetSet, correlationAttribute); - } - final String NO_DESCR = NbBundle.getMessage(this.getClass(), "LocalFileNode.createSheet.noDescr.text"); - for (Map.Entry entry : map.entrySet()) { - sheetSet.put(new NodeProperty<>(entry.getKey(), entry.getKey(), NO_DESCR, entry.getValue())); - } - - return sheet; - } - @Override public Action[] getActions(boolean context) { List actionsList = new ArrayList<>(); diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/VirtualDirectoryNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/VirtualDirectoryNode.java index 719bc3420a..fabcf79152 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/VirtualDirectoryNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/VirtualDirectoryNode.java @@ -21,18 +21,14 @@ package org.sleuthkit.autopsy.datamodel; import java.sql.ResultSet; import java.sql.SQLException; import java.util.LinkedHashMap; -import java.util.List; import java.util.Map; import java.util.logging.Level; import org.openide.nodes.Sheet; import org.openide.util.NbBundle; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; -import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeInstance; -import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbUtil; -import org.sleuthkit.autopsy.core.UserPreferences; import org.sleuthkit.autopsy.coreutils.Logger; -import org.sleuthkit.datamodel.ContentTag; +import org.sleuthkit.autopsy.texttranslation.FileProperty; import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.VirtualDirectory; @@ -84,32 +80,26 @@ public class VirtualDirectoryNode extends SpecialDirectoryNode { sheetSet = Sheet.createPropertiesSet(); sheet.put(sheetSet); } - List tags = getContentTagsFromDatabase(); sheetSet.put(new NodeProperty<>(NbBundle.getMessage(this.getClass(), "VirtualDirectoryNode.createSheet.name.name"), NbBundle.getMessage(this.getClass(), "VirtualDirectoryNode.createSheet.name.displayName"), NbBundle.getMessage(this.getClass(), "VirtualDirectoryNode.createSheet.name.desc"), getName())); + if (!this.content.isDataSource()) { - addScoreProperty(sheetSet, tags); - - CorrelationAttributeInstance correlationAttribute = null; - if (EamDbUtil.useCentralRepo() && UserPreferences.hideCentralRepoCommentsAndOccurrences() == false) { - correlationAttribute = getCorrelationAttributeInstance(); - } - addCommentProperty(sheetSet, tags, correlationAttribute); - - if (EamDbUtil.useCentralRepo() && UserPreferences.hideCentralRepoCommentsAndOccurrences() == false) { - addCountProperty(sheetSet, correlationAttribute); - } Map map = new LinkedHashMap<>(); fillPropertyMap(map, getContent()); - final String NO_DESCR = NbBundle.getMessage(this.getClass(), "VirtualDirectoryNode.createSheet.noDesc"); for (Map.Entry entry : map.entrySet()) { - sheetSet.put(new NodeProperty<>(entry.getKey(), entry.getKey(), NO_DESCR, entry.getValue())); + String desc = NbBundle.getMessage(this.getClass(), "VirtualDirectoryNode.createSheet.noDesc"); + FileProperty p = AbstractFilePropertyType.getPropertyFromDisplayName(entry.getKey()); + if(!p.getDescription(content).isEmpty()) { + desc = p.getDescription(content); + } + sheetSet.put(new NodeProperty<>(entry.getKey(), entry.getKey(), desc, entry.getValue())); } + } else { sheetSet.put(new NodeProperty<>(Bundle.VirtualDirectoryNode_createSheet_type_name(), Bundle.VirtualDirectoryNode_createSheet_type_displayName(), diff --git a/Core/src/org/sleuthkit/autopsy/texttranslation/CustomFileProperty.java b/Core/src/org/sleuthkit/autopsy/texttranslation/CustomFileProperty.java new file mode 100755 index 0000000000..420965fe3e --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/texttranslation/CustomFileProperty.java @@ -0,0 +1,32 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2018-2018 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.texttranslation; + +/** + * + */ +public interface CustomFileProperty extends FileProperty { + + /** + * + * @return + */ + public Integer getColumnPosition(); + +} diff --git a/Core/src/org/sleuthkit/autopsy/texttranslation/FileProperty.java b/Core/src/org/sleuthkit/autopsy/texttranslation/FileProperty.java new file mode 100755 index 0000000000..8bf7a4d89f --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/texttranslation/FileProperty.java @@ -0,0 +1,46 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ +package org.sleuthkit.autopsy.texttranslation; + +import org.sleuthkit.datamodel.AbstractFile; + +/** + * + * @author dsmyda + */ +public interface FileProperty { + /** + * + * @param content + * @return + */ + public Object getPropertyValue(AbstractFile content); + + /** + * + * @return + */ + public default String getPropertyName(){ + return this.toString(); + } + + /** + * + * @return + */ + public default boolean isDisabled() { + return false; + } + + /** + * + * @param content + * @return + */ + public default String getDescription(AbstractFile content) { + return ""; + } +} diff --git a/Core/src/org/sleuthkit/autopsy/texttranslation/ServiceTest.java b/Core/src/org/sleuthkit/autopsy/texttranslation/ServiceTest.java deleted file mode 100755 index a3a2941989..0000000000 --- a/Core/src/org/sleuthkit/autopsy/texttranslation/ServiceTest.java +++ /dev/null @@ -1,20 +0,0 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ -package org.sleuthkit.autopsy.texttranslation; - -/** - * - * @author dsmyda - */ - -public class ServiceTest implements TextTranslator { - - @Override - public String translate(String input) throws TranslationException { - throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates. - } - -} diff --git a/Core/src/org/sleuthkit/autopsy/texttranslation/TranslationProperty.java b/Core/src/org/sleuthkit/autopsy/texttranslation/TranslationProperty.java new file mode 100755 index 0000000000..27be211b71 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/texttranslation/TranslationProperty.java @@ -0,0 +1,36 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ +package org.sleuthkit.autopsy.texttranslation; + +import org.openide.util.lookup.ServiceProvider; +import org.sleuthkit.datamodel.AbstractFile; +/** + * + * @author dsmyda + */ +@ServiceProvider(service = CustomFileProperty.class) +public class TranslationProperty implements CustomFileProperty { + + @Override + public boolean isDisabled() { + return false; + } + + @Override + public String getPropertyName() { + return "Translated Name"; + } + + @Override + public Object getPropertyValue(AbstractFile content) { + return "Foo"; + } + + @Override + public Integer getColumnPosition() { + return 1; + } +} From 46205e1794cc49ca23f3254bfdb90478d4200aa0 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Fri, 19 Oct 2018 14:28:13 -0400 Subject: [PATCH 008/129] Still making progress on the clean up --- .../autopsy/contentviewers/MessageContentViewer.java | 11 +++++++---- .../sleuthkit/autopsy/datamodel/LayoutFileNode.java | 6 ------ .../sleuthkit/autopsy/datamodel/LocalFileNode.java | 8 -------- 3 files changed, 7 insertions(+), 18 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/contentviewers/MessageContentViewer.java b/Core/src/org/sleuthkit/autopsy/contentviewers/MessageContentViewer.java index be342e78ca..06bccaa6f8 100644 --- a/Core/src/org/sleuthkit/autopsy/contentviewers/MessageContentViewer.java +++ b/Core/src/org/sleuthkit/autopsy/contentviewers/MessageContentViewer.java @@ -722,17 +722,20 @@ public class MessageContentViewer extends javax.swing.JPanel implements DataCont @Override protected Sheet createSheet() { - Sheet sheetSet = super.createSheet(); - Set keepProps = new HashSet<>(Arrays.asList("Name" , "Size", "Mime Type", "Known")); + Sheet sheet = super.createSheet(); + Set keepProps = new HashSet<>(Arrays.asList("Name" , "S", + "C", "O", "Size", "Mime Type", "Known")); - for(PropertySet p : sheetSet.toArray()) { + //Remove all other proprs except for + Sheet.Set sheetSet = sheet.get(Sheet.PROPERTIES); + for(Property p : sheetSet.getProperties()) { if(keepProps.contains(p.getName())){ continue; } sheetSet.remove(p.getName()); } - return sheetSet; + return sheet; } } } diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/LayoutFileNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/LayoutFileNode.java index 5895ccab6a..9e72d4dc8c 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/LayoutFileNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/LayoutFileNode.java @@ -22,24 +22,18 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.HashSet; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import javax.swing.Action; -import org.openide.nodes.Sheet; import org.openide.util.NbBundle; import org.openide.util.Utilities; import org.sleuthkit.autopsy.actions.AddContentTagAction; import org.sleuthkit.autopsy.actions.DeleteFileContentTagAction; -import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeInstance; -import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbUtil; -import org.sleuthkit.autopsy.core.UserPreferences; import org.sleuthkit.autopsy.coreutils.ContextMenuExtensionPoint; import org.sleuthkit.autopsy.directorytree.ExternalViewerAction; import org.sleuthkit.autopsy.directorytree.ExtractAction; import org.sleuthkit.autopsy.directorytree.NewWindowViewAction; import org.sleuthkit.datamodel.AbstractFile; -import org.sleuthkit.datamodel.ContentTag; import org.sleuthkit.datamodel.LayoutFile; import org.sleuthkit.datamodel.TskData; diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/LocalFileNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/LocalFileNode.java index a314791fcb..beb624d1af 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/LocalFileNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/LocalFileNode.java @@ -22,19 +22,13 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.HashSet; -import java.util.LinkedHashMap; import java.util.List; -import java.util.Map; import java.util.logging.Level; import javax.swing.Action; -import org.openide.nodes.Sheet; import org.openide.util.NbBundle; import org.openide.util.Utilities; import org.sleuthkit.autopsy.actions.AddContentTagAction; import org.sleuthkit.autopsy.actions.DeleteFileContentTagAction; -import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeInstance; -import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbUtil; -import org.sleuthkit.autopsy.core.UserPreferences; import org.sleuthkit.autopsy.coreutils.ContextMenuExtensionPoint; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.directorytree.ExternalViewerAction; @@ -42,10 +36,8 @@ import org.sleuthkit.autopsy.directorytree.ExtractAction; import org.sleuthkit.autopsy.directorytree.NewWindowViewAction; import org.sleuthkit.autopsy.directorytree.ViewContextAction; import org.sleuthkit.autopsy.modules.embeddedfileextractor.ExtractArchiveWithPasswordAction; -import org.sleuthkit.autopsy.texttranslation.FileProperty; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.BlackboardArtifact; -import org.sleuthkit.datamodel.ContentTag; import org.sleuthkit.datamodel.TskCoreException; /** From 63ac179f916f7c74f9664cc20a9dc17a0e31b6eb Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Fri, 19 Oct 2018 14:47:18 -0400 Subject: [PATCH 009/129] Finished creating a good workable interface now testing --- .../CaseDBCommonAttributeInstanceNode.java | 40 ++++++------------- .../contentviewers/MessageContentViewer.java | 2 +- .../datamodel/AbstractAbstractFileNode.java | 6 +-- .../datamodel/AbstractFsContentNode.java | 5 ++- 4 files changed, 20 insertions(+), 33 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CaseDBCommonAttributeInstanceNode.java b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CaseDBCommonAttributeInstanceNode.java index b4f8ee525e..0b528239a2 100644 --- a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CaseDBCommonAttributeInstanceNode.java +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CaseDBCommonAttributeInstanceNode.java @@ -18,17 +18,14 @@ */ package org.sleuthkit.autopsy.commonfilesearch; -import java.util.List; -import org.apache.commons.lang3.StringUtils; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; import org.openide.nodes.Sheet; -import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeInstance; -import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbUtil; -import org.sleuthkit.autopsy.core.UserPreferences; import org.sleuthkit.autopsy.datamodel.DisplayableItemNodeVisitor; import org.sleuthkit.autopsy.datamodel.FileNode; import org.sleuthkit.autopsy.datamodel.NodeProperty; import org.sleuthkit.datamodel.AbstractFile; -import org.sleuthkit.datamodel.ContentTag; /** * Node that wraps CaseDBCommonAttributeInstance to represent a file instance @@ -75,33 +72,22 @@ public class CaseDBCommonAttributeInstanceNode extends FileNode { @Override protected Sheet createSheet() { - Sheet sheet = new Sheet(); + Sheet sheet = super.createSheet(); Sheet.Set sheetSet = sheet.get(Sheet.PROPERTIES); - if (sheetSet == null) { - sheetSet = Sheet.createPropertiesSet(); - sheet.put(sheetSet); + Set keepProps = new HashSet<>(Arrays.asList("S", + "C", "O", "Mime Type")); + + for(Property p : sheetSet.getProperties()) { + if(keepProps.contains(p.getName())){ + continue; + } + sheetSet.remove(p.getName()); } - List tags = getContentTagsFromDatabase(); - final String NO_DESCR = Bundle.CommonFilesSearchResultsViewerTable_noDescText(); - - sheetSet.put(new NodeProperty<>(Bundle.CommonFilesSearchResultsViewerTable_filesColLbl(), Bundle.CommonFilesSearchResultsViewerTable_filesColLbl(), NO_DESCR, this.getContent().getName())); - addScoreProperty(sheetSet, tags); - - CorrelationAttributeInstance correlationAttribute = null; - if (EamDbUtil.useCentralRepo() && UserPreferences.hideCentralRepoCommentsAndOccurrences()== false) { - correlationAttribute = getCorrelationAttributeInstance(); - } - addCommentProperty(sheetSet, tags, correlationAttribute); - - if (EamDbUtil.useCentralRepo() && UserPreferences.hideCentralRepoCommentsAndOccurrences()== false) { - addCountProperty(sheetSet, correlationAttribute); - } - sheetSet.put(new NodeProperty<>(Bundle.CommonFilesSearchResultsViewerTable_pathColLbl(), Bundle.CommonFilesSearchResultsViewerTable_pathColLbl(), NO_DESCR, this.getContent().getParentPath())); sheetSet.put(new NodeProperty<>(Bundle.CommonFilesSearchResultsViewerTable_dataSourceColLbl(), Bundle.CommonFilesSearchResultsViewerTable_dataSourceColLbl(), NO_DESCR, this.getDataSource())); - sheetSet.put(new NodeProperty<>(Bundle.CommonFilesSearchResultsViewerTable_mimeTypeColLbl(), Bundle.CommonFilesSearchResultsViewerTable_mimeTypeColLbl(), NO_DESCR, StringUtils.defaultString(this.getContent().getMIMEType()))); sheetSet.put(new NodeProperty<>(Bundle.CommonFilesSearchResultsViewerTable_caseColLbl1(), Bundle.CommonFilesSearchResultsViewerTable_caseColLbl1(), NO_DESCR, caseName)); + return sheet; } } \ No newline at end of file diff --git a/Core/src/org/sleuthkit/autopsy/contentviewers/MessageContentViewer.java b/Core/src/org/sleuthkit/autopsy/contentviewers/MessageContentViewer.java index 06bccaa6f8..81f2fb0482 100644 --- a/Core/src/org/sleuthkit/autopsy/contentviewers/MessageContentViewer.java +++ b/Core/src/org/sleuthkit/autopsy/contentviewers/MessageContentViewer.java @@ -728,7 +728,7 @@ public class MessageContentViewer extends javax.swing.JPanel implements DataCont //Remove all other proprs except for Sheet.Set sheetSet = sheet.get(Sheet.PROPERTIES); - for(Property p : sheetSet.getProperties()) { + for(Property p : sheetSet.getProperties()) { if(keepProps.contains(p.getName())){ continue; } diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java index 0e59136bdd..f4b72b9809 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java @@ -243,7 +243,7 @@ public abstract class AbstractAbstractFileNode extends A } }, SCORE(AbstractAbstractFileNode_createSheet_score_name()) { - Optional> result; + Optional> result = Optional.empty(); private void initResult(AbstractFile content) { scoreAndCommentTags = getContentTagsFromDatabase(content); @@ -267,7 +267,7 @@ public abstract class AbstractAbstractFileNode extends A } }, COMMENT(AbstractAbstractFileNode_createSheet_comment_name()) { - Optional> result; + Optional> result = Optional.empty(); private void initResult(AbstractFile content) { scoreAndCommentTags = getContentTagsFromDatabase(content); @@ -298,7 +298,7 @@ public abstract class AbstractAbstractFileNode extends A } }, COUNT(AbstractAbstractFileNode_createSheet_count_name()) { - Optional> result; + Optional> result = Optional.empty(); private void initResult(AbstractFile content) { result = Optional.of(getCountProperty(correlationAttribute)); diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractFsContentNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractFsContentNode.java index 7d18f2a0c7..816502dd8d 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractFsContentNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractFsContentNode.java @@ -62,12 +62,13 @@ public abstract class AbstractFsContentNode extends Abst @Override @NbBundle.Messages("AbstractFsContentNode.noDesc.text=no description") protected Sheet createSheet() { - Sheet sheetSet = super.createSheet(); + Sheet sheet = super.createSheet(); + Sheet.Set sheetSet = sheet.get(Sheet.PROPERTIES); if (directoryBrowseMode) { sheetSet.put(new NodeProperty<>(HIDE_PARENT, HIDE_PARENT, HIDE_PARENT, HIDE_PARENT)); } - return sheetSet; + return sheet; } } From 6db31caf49e8a0ffe82f38829211bd9f2d9ff774 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Fri, 19 Oct 2018 14:56:21 -0400 Subject: [PATCH 010/129] Looks good, I'll do a little harder testing on Monday --- .../sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java index f4b72b9809..14fe8671bf 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java @@ -496,7 +496,7 @@ public abstract class AbstractAbstractFileNode extends A }*/ //Skip properties that are disabled, don't add them to the property map! - properties.stream().filter(p -> p.isDisabled()).forEach((p) -> { + properties.stream().filter(p -> !p.isDisabled()).forEach((p) -> { map.put(p.getPropertyName(), p.getPropertyValue(content)); }); } From 8eb7e250954e521244ec558b0ffe1b4fa7eec2e8 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Fri, 19 Oct 2018 15:05:13 -0400 Subject: [PATCH 011/129] Moved the Property interfaces to the data model package where they belong and refactored imports --- .../sleuthkit/autopsy/contentviewers/MessageContentViewer.java | 2 +- .../sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java | 3 +-- .../{texttranslation => datamodel}/CustomFileProperty.java | 2 +- .../autopsy/{texttranslation => datamodel}/FileProperty.java | 2 +- .../org/sleuthkit/autopsy/datamodel/VirtualDirectoryNode.java | 1 - .../sleuthkit/autopsy/texttranslation/TranslationProperty.java | 2 ++ 6 files changed, 6 insertions(+), 6 deletions(-) rename Core/src/org/sleuthkit/autopsy/{texttranslation => datamodel}/CustomFileProperty.java (94%) rename Core/src/org/sleuthkit/autopsy/{texttranslation => datamodel}/FileProperty.java (94%) diff --git a/Core/src/org/sleuthkit/autopsy/contentviewers/MessageContentViewer.java b/Core/src/org/sleuthkit/autopsy/contentviewers/MessageContentViewer.java index 81f2fb0482..4210d2eb92 100644 --- a/Core/src/org/sleuthkit/autopsy/contentviewers/MessageContentViewer.java +++ b/Core/src/org/sleuthkit/autopsy/contentviewers/MessageContentViewer.java @@ -726,7 +726,7 @@ public class MessageContentViewer extends javax.swing.JPanel implements DataCont Set keepProps = new HashSet<>(Arrays.asList("Name" , "S", "C", "O", "Size", "Mime Type", "Known")); - //Remove all other proprs except for + //Remove all other props except for the ones above Sheet.Set sheetSet = sheet.get(Sheet.PROPERTIES); for(Property p : sheetSet.getProperties()) { if(keepProps.contains(p.getName())){ diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java index 14fe8671bf..a1fd8b64e6 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java @@ -56,14 +56,13 @@ import static org.sleuthkit.autopsy.datamodel.Bundle.*; import org.sleuthkit.autopsy.corecomponents.DataResultViewerTable.HasCommentStatus; import org.sleuthkit.autopsy.ingest.IngestManager; import org.sleuthkit.autopsy.ingest.ModuleContentEvent; -import org.sleuthkit.autopsy.texttranslation.CustomFileProperty; +import org.sleuthkit.autopsy.datamodel.CustomFileProperty; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.ContentTag; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskData; -import org.sleuthkit.autopsy.texttranslation.FileProperty; /** * An abstract node that encapsulates AbstractFile data diff --git a/Core/src/org/sleuthkit/autopsy/texttranslation/CustomFileProperty.java b/Core/src/org/sleuthkit/autopsy/datamodel/CustomFileProperty.java similarity index 94% rename from Core/src/org/sleuthkit/autopsy/texttranslation/CustomFileProperty.java rename to Core/src/org/sleuthkit/autopsy/datamodel/CustomFileProperty.java index 420965fe3e..37b48e3811 100755 --- a/Core/src/org/sleuthkit/autopsy/texttranslation/CustomFileProperty.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/CustomFileProperty.java @@ -16,7 +16,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.sleuthkit.autopsy.texttranslation; +package org.sleuthkit.autopsy.datamodel; /** * diff --git a/Core/src/org/sleuthkit/autopsy/texttranslation/FileProperty.java b/Core/src/org/sleuthkit/autopsy/datamodel/FileProperty.java similarity index 94% rename from Core/src/org/sleuthkit/autopsy/texttranslation/FileProperty.java rename to Core/src/org/sleuthkit/autopsy/datamodel/FileProperty.java index 8bf7a4d89f..49b2df6a32 100755 --- a/Core/src/org/sleuthkit/autopsy/texttranslation/FileProperty.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/FileProperty.java @@ -3,7 +3,7 @@ * To change this template file, choose Tools | Templates * and open the template in the editor. */ -package org.sleuthkit.autopsy.texttranslation; +package org.sleuthkit.autopsy.datamodel; import org.sleuthkit.datamodel.AbstractFile; diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/VirtualDirectoryNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/VirtualDirectoryNode.java index fabcf79152..01479a6082 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/VirtualDirectoryNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/VirtualDirectoryNode.java @@ -28,7 +28,6 @@ import org.openide.util.NbBundle; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; import org.sleuthkit.autopsy.coreutils.Logger; -import org.sleuthkit.autopsy.texttranslation.FileProperty; import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.VirtualDirectory; diff --git a/Core/src/org/sleuthkit/autopsy/texttranslation/TranslationProperty.java b/Core/src/org/sleuthkit/autopsy/texttranslation/TranslationProperty.java index 27be211b71..576e40844c 100755 --- a/Core/src/org/sleuthkit/autopsy/texttranslation/TranslationProperty.java +++ b/Core/src/org/sleuthkit/autopsy/texttranslation/TranslationProperty.java @@ -7,6 +7,8 @@ package org.sleuthkit.autopsy.texttranslation; import org.openide.util.lookup.ServiceProvider; import org.sleuthkit.datamodel.AbstractFile; + +import org.sleuthkit.autopsy.datamodel.CustomFileProperty; /** * * @author dsmyda From 316b2bd24af0bb02da373f3d8b6a38a18b60c636 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Fri, 19 Oct 2018 15:21:46 -0400 Subject: [PATCH 012/129] Got custom properties working, will most likely change implementation later --- .../datamodel/AbstractAbstractFileNode.java | 36 +++++++++++++++---- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java index a1fd8b64e6..25caaa3487 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java @@ -199,7 +199,9 @@ public abstract class AbstractAbstractFileNode extends A for (Map.Entry entry : map.entrySet()) { String desc = Bundle.AbstractFsContentNode_noDesc_text(); - FileProperty p = AbstractFilePropertyType.getPropertyFromDisplayName(entry.getKey()); + + + FileProperty p = getFilePropertyFromDisplayName(entry.getKey()); if(!p.getDescription(content).isEmpty()) { desc = p.getDescription(content); } @@ -209,6 +211,27 @@ public abstract class AbstractAbstractFileNode extends A return sheet; } + private FileProperty getFilePropertyFromDisplayName(String displayName) { + FileProperty p = AbstractFilePropertyType.getPropertyFromDisplayName(displayName); + if(p != null) { + return p; + } else { + Optional> customProperties = getCustomProperties(); + if(customProperties.isPresent()) { + for(CustomFileProperty cp : customProperties.get()) { + if (cp.getPropertyName().equals(displayName)) { + return cp; + } + } + } + return null; + } + } + + private static Optional> getCustomProperties() { + return Optional.ofNullable(Lookup.getDefault().lookupAll(CustomFileProperty.class)); + } + @NbBundle.Messages({"AbstractAbstractFileNode.nameColLbl=Name", "AbstractAbstractFileNode.createSheet.score.name=S", "AbstractAbstractFileNode.createSheet.comment.name=C", @@ -483,16 +506,15 @@ public abstract class AbstractAbstractFileNode extends A ArrayList properties = new ArrayList<>(); properties.addAll(Arrays.asList(AbstractFilePropertyType.values())); - /* //Load in our custom properties - Optional> customProperties = - Optional.ofNullable(Lookup.getDefault().lookupAll(CustomFileProperty.class)); + //Load in our custom properties + Optional> customProperties = getCustomProperties(); if (customProperties.isPresent()) { - for(CustomFileProperty p : customProperties.get()) { + customProperties.get().forEach((p) -> { //Inject custom properties at the desired column positions //Specified by the custom property itself. properties.add(p.getColumnPosition(), p); - } - }*/ + }); + } //Skip properties that are disabled, don't add them to the property map! properties.stream().filter(p -> !p.isDisabled()).forEach((p) -> { From cf9a1040c92cf9f755a26a8553e987db6177f013 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Fri, 19 Oct 2018 15:22:25 -0400 Subject: [PATCH 013/129] Time to move this somewhere special.... --- .../texttranslation/TranslationProperty.java | 38 ------------------- 1 file changed, 38 deletions(-) delete mode 100755 Core/src/org/sleuthkit/autopsy/texttranslation/TranslationProperty.java diff --git a/Core/src/org/sleuthkit/autopsy/texttranslation/TranslationProperty.java b/Core/src/org/sleuthkit/autopsy/texttranslation/TranslationProperty.java deleted file mode 100755 index 576e40844c..0000000000 --- a/Core/src/org/sleuthkit/autopsy/texttranslation/TranslationProperty.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ -package org.sleuthkit.autopsy.texttranslation; - -import org.openide.util.lookup.ServiceProvider; -import org.sleuthkit.datamodel.AbstractFile; - -import org.sleuthkit.autopsy.datamodel.CustomFileProperty; -/** - * - * @author dsmyda - */ -@ServiceProvider(service = CustomFileProperty.class) -public class TranslationProperty implements CustomFileProperty { - - @Override - public boolean isDisabled() { - return false; - } - - @Override - public String getPropertyName() { - return "Translated Name"; - } - - @Override - public Object getPropertyValue(AbstractFile content) { - return "Foo"; - } - - @Override - public Integer getColumnPosition() { - return 1; - } -} From 82ce8bb843ed5b64c9d672246137fc11e26c76e6 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Fri, 19 Oct 2018 15:34:33 -0400 Subject: [PATCH 014/129] Removed unused import --- .../sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java | 1 - 1 file changed, 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java index 25caaa3487..efaf7d2f18 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java @@ -56,7 +56,6 @@ import static org.sleuthkit.autopsy.datamodel.Bundle.*; import org.sleuthkit.autopsy.corecomponents.DataResultViewerTable.HasCommentStatus; import org.sleuthkit.autopsy.ingest.IngestManager; import org.sleuthkit.autopsy.ingest.ModuleContentEvent; -import org.sleuthkit.autopsy.datamodel.CustomFileProperty; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.Content; From 9114867803d5c9a1b1511c6e27e308d9d5cfaac1 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Wed, 24 Oct 2018 09:25:30 -0400 Subject: [PATCH 015/129] Unfortunately, the changes to allow a custom file property are not worth it and frankly very difficult to do since the AAFN fillPropertyMap method is static --- .../datamodel/AbstractAbstractFileNode.java | 163 ++++++++---------- .../autopsy/datamodel/CustomFileProperty.java | 32 ---- .../TextTranslationService.java | 12 +- .../texttranslation/TextTranslator.java | 8 + 4 files changed, 88 insertions(+), 127 deletions(-) delete mode 100755 Core/src/org/sleuthkit/autopsy/datamodel/CustomFileProperty.java diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java index efaf7d2f18..a7bcbf9299 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java @@ -20,6 +20,7 @@ package org.sleuthkit.autopsy.datamodel; import java.beans.PropertyChangeEvent; import java.beans.PropertyChangeListener; +import java.lang.reflect.InvocationTargetException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -29,20 +30,25 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.logging.Level; import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; import org.openide.nodes.Children; import org.openide.nodes.Sheet; +import org.openide.util.Exceptions; import org.openide.util.Lookup; import org.openide.util.NbBundle; import org.openide.util.WeakListeners; +import org.openide.util.lookup.Lookups; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; import org.sleuthkit.autopsy.casemodule.events.CommentChangedEvent; import org.sleuthkit.autopsy.casemodule.events.ContentTagAddedEvent; import org.sleuthkit.autopsy.casemodule.events.ContentTagDeletedEvent; +import org.sleuthkit.autopsy.casemodule.events.TranslationAvailableEvent; import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeInstance; import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeNormalizationException; import org.sleuthkit.autopsy.centralrepository.datamodel.EamArtifactUtil; @@ -69,7 +75,7 @@ import org.sleuthkit.datamodel.TskData; * @param type of the AbstractFile to encapsulate */ public abstract class AbstractAbstractFileNode extends AbstractContentNode { - + private static final Logger logger = Logger.getLogger(AbstractAbstractFileNode.class.getName()); @NbBundle.Messages("AbstractAbstractFileNode.addFileProperty.desc=no description") private static final String NO_DESCR = AbstractAbstractFileNode_addFileProperty_desc(); @@ -183,7 +189,7 @@ public abstract class AbstractAbstractFileNode extends A private void updateSheet() { this.setSheet(createSheet()); } - + @Override protected Sheet createSheet() { Sheet sheet = super.createSheet(); @@ -192,45 +198,23 @@ public abstract class AbstractAbstractFileNode extends A sheetSet = Sheet.createPropertiesSet(); sheet.put(sheetSet); } - + Map map = new LinkedHashMap<>(); fillPropertyMap(map, getContent()); - + for (Map.Entry entry : map.entrySet()) { String desc = Bundle.AbstractFsContentNode_noDesc_text(); - - - FileProperty p = getFilePropertyFromDisplayName(entry.getKey()); - if(!p.getDescription(content).isEmpty()) { + + FileProperty p = AbstractFilePropertyType.getPropertyFromDisplayName(entry.getKey()); + if (!p.getDescription(content).isEmpty()) { desc = p.getDescription(content); } sheetSet.put(new NodeProperty<>(entry.getKey(), entry.getKey(), desc, entry.getValue())); } - + return sheet; } - - private FileProperty getFilePropertyFromDisplayName(String displayName) { - FileProperty p = AbstractFilePropertyType.getPropertyFromDisplayName(displayName); - if(p != null) { - return p; - } else { - Optional> customProperties = getCustomProperties(); - if(customProperties.isPresent()) { - for(CustomFileProperty cp : customProperties.get()) { - if (cp.getPropertyName().equals(displayName)) { - return cp; - } - } - } - return null; - } - } - - private static Optional> getCustomProperties() { - return Optional.ofNullable(Lookup.getDefault().lookupAll(CustomFileProperty.class)); - } - + @NbBundle.Messages({"AbstractAbstractFileNode.nameColLbl=Name", "AbstractAbstractFileNode.createSheet.score.name=S", "AbstractAbstractFileNode.createSheet.comment.name=C", @@ -259,60 +243,64 @@ public abstract class AbstractAbstractFileNode extends A NAME(AbstractAbstractFileNode_nameColLbl()) { @Override - public Object getPropertyValue(AbstractFile content) { + public Object getPropertyValue(AbstractFile content) { return getContentDisplayName(content); } }, SCORE(AbstractAbstractFileNode_createSheet_score_name()) { Optional> result = Optional.empty(); - + private void initResult(AbstractFile content) { scoreAndCommentTags = getContentTagsFromDatabase(content); - result = Optional.of(getScoreProperty(content, scoreAndCommentTags)); + result = Optional.of(getScoreProperty(content, scoreAndCommentTags)); } - - @Override + + @Override public Object getPropertyValue(AbstractFile content) { - if(!this.result.isPresent()){ - initResult(content); + if (!this.result.isPresent()) { + initResult(content); } - return result.get().getRight(); - } - + Score res = result.get().getRight(); + result = Optional.empty(); + return res; + } + @Override public String getDescription(AbstractFile content) { - if(!this.result.isPresent()){ - initResult(content); + if (!this.result.isPresent()) { + initResult(content); } return result.get().getLeft(); } }, COMMENT(AbstractAbstractFileNode_createSheet_comment_name()) { Optional> result = Optional.empty(); - + private void initResult(AbstractFile content) { scoreAndCommentTags = getContentTagsFromDatabase(content); correlationAttribute = null; - if(EamDbUtil.useCentralRepo() && - !UserPreferences.hideCentralRepoCommentsAndOccurrences()){ + if (EamDbUtil.useCentralRepo() + && !UserPreferences.hideCentralRepoCommentsAndOccurrences()) { if (EamDbUtil.useCentralRepo() && UserPreferences.hideCentralRepoCommentsAndOccurrences() == false) { correlationAttribute = getCorrelationAttributeInstance(content); } - } + } result = Optional.of(getCommentProperty(scoreAndCommentTags, correlationAttribute)); } - + @Override public Object getPropertyValue(AbstractFile content) { - if(!this.result.isPresent()){ + if (!this.result.isPresent()) { initResult(content); } - return result.get().getRight(); - } - + HasCommentStatus res = result.get().getRight(); + result = Optional.empty(); + return res; + } + @Override public String getDescription(AbstractFile content) { - if(!this.result.isPresent()){ + if (!this.result.isPresent()) { initResult(content); } return result.get().getLeft(); @@ -320,34 +308,35 @@ public abstract class AbstractAbstractFileNode extends A }, COUNT(AbstractAbstractFileNode_createSheet_count_name()) { Optional> result = Optional.empty(); - + private void initResult(AbstractFile content) { result = Optional.of(getCountProperty(correlationAttribute)); } - - + @Override public Object getPropertyValue(AbstractFile content) { - if(!result.isPresent()){ + if (!result.isPresent()) { initResult(content); } - return result.get().getRight(); + Long res = result.get().getRight(); + result = Optional.empty(); + return res; } @Override public boolean isDisabled() { - return !EamDbUtil.useCentralRepo() || - UserPreferences.hideCentralRepoCommentsAndOccurrences(); + return !EamDbUtil.useCentralRepo() + || UserPreferences.hideCentralRepoCommentsAndOccurrences(); } @Override public String getDescription(AbstractFile content) { - if(!result.isPresent()){ + if (!result.isPresent()) { initResult(content); } return result.get().getLeft(); } - + }, LOCATION(AbstractAbstractFileNode_locationColLbl()) { @Override @@ -421,49 +410,49 @@ public abstract class AbstractAbstractFileNode extends A return content.getMetaAddr(); } }, - ATTR_ADDR(AbstractAbstractFileNode_attrAddrColLbl()){ + ATTR_ADDR(AbstractAbstractFileNode_attrAddrColLbl()) { @Override public Object getPropertyValue(AbstractFile content) { return content.getAttrType().getValue() + "-" + content.getAttributeId(); } }, - TYPE_DIR(AbstractAbstractFileNode_typeDirColLbl()){ + TYPE_DIR(AbstractAbstractFileNode_typeDirColLbl()) { @Override public Object getPropertyValue(AbstractFile content) { return content.getDirType().getLabel(); } }, - TYPE_META(AbstractAbstractFileNode_typeMetaColLbl()){ + TYPE_META(AbstractAbstractFileNode_typeMetaColLbl()) { @Override public Object getPropertyValue(AbstractFile content) { return content.getMetaType().toString(); } }, - KNOWN(AbstractAbstractFileNode_knownColLbl()){ + KNOWN(AbstractAbstractFileNode_knownColLbl()) { @Override public Object getPropertyValue(AbstractFile content) { return content.getKnown().getName(); } }, - MD5HASH(AbstractAbstractFileNode_md5HashColLbl()){ + MD5HASH(AbstractAbstractFileNode_md5HashColLbl()) { @Override public Object getPropertyValue(AbstractFile content) { return StringUtils.defaultString(content.getMd5Hash()); } }, - ObjectID(AbstractAbstractFileNode_objectId()){ + ObjectID(AbstractAbstractFileNode_objectId()) { @Override public Object getPropertyValue(AbstractFile content) { return content.getId(); } }, - MIMETYPE(AbstractAbstractFileNode_mimeType()){ + MIMETYPE(AbstractAbstractFileNode_mimeType()) { @Override public Object getPropertyValue(AbstractFile content) { return StringUtils.defaultString(content.getMIMEType()); } }, - EXTENSION(AbstractAbstractFileNode_extensionColLbl()){ + EXTENSION(AbstractAbstractFileNode_extensionColLbl()) { @Override public Object getPropertyValue(AbstractFile content) { return content.getNameExtension(); @@ -482,17 +471,17 @@ public abstract class AbstractAbstractFileNode extends A public String toString() { return displayString; } - + public static FileProperty getPropertyFromDisplayName(String displayName) { - for(FileProperty p : AbstractFilePropertyType.values()) { - if(p.getPropertyName().equals(displayName)) { + for (FileProperty p : AbstractFilePropertyType.values()) { + if (p.getPropertyName().equals(displayName)) { return p; } } return null; } } - + /** * Fill map with AbstractFile properties * @@ -501,24 +490,10 @@ public abstract class AbstractAbstractFileNode extends A * @param content The content to get properties for. */ static public void fillPropertyMap(Map map, AbstractFile content) { - //Load in our default properties. - ArrayList properties = new ArrayList<>(); - properties.addAll(Arrays.asList(AbstractFilePropertyType.values())); - - //Load in our custom properties - Optional> customProperties = getCustomProperties(); - if (customProperties.isPresent()) { - customProperties.get().forEach((p) -> { - //Inject custom properties at the desired column positions - //Specified by the custom property itself. - properties.add(p.getColumnPosition(), p); - }); - } - - //Skip properties that are disabled, don't add them to the property map! - properties.stream().filter(p -> !p.isDisabled()).forEach((p) -> { - map.put(p.getPropertyName(), p.getPropertyValue(content)); - }); + Arrays.asList(AbstractFilePropertyType.values()) + .stream() + .filter(p -> !p.isDisabled()) + .forEach((p) -> map.put(p.getPropertyName(), p.getPropertyValue(content))); } /** @@ -643,7 +618,7 @@ public abstract class AbstractAbstractFileNode extends A } catch (CorrelationAttributeNormalizationException ex) { logger.log(Level.WARNING, "Unable to normalize data to get count of datasources with correlation attribute", ex); } - + return Pair.of(description, count); } @@ -653,6 +628,7 @@ public abstract class AbstractAbstractFileNode extends A * * @param sheetSet the modifiable Sheet.Set returned by * Sheet.get(Sheet.PROPERTIES) + * * @deprecated */ @NbBundle.Messages("AbstractAbstractFileNode.tagsProperty.displayName=Tags") @@ -699,6 +675,7 @@ public abstract class AbstractAbstractFileNode extends A * @param file The file. * * @return The CSV list of hash set names. + * * @deprecated */ @Deprecated diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/CustomFileProperty.java b/Core/src/org/sleuthkit/autopsy/datamodel/CustomFileProperty.java deleted file mode 100755 index 37b48e3811..0000000000 --- a/Core/src/org/sleuthkit/autopsy/datamodel/CustomFileProperty.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Autopsy Forensic Browser - * - * Copyright 2018-2018 Basis Technology Corp. - * Contact: carrier sleuthkit org - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.sleuthkit.autopsy.datamodel; - -/** - * - */ -public interface CustomFileProperty extends FileProperty { - - /** - * - * @return - */ - public Integer getColumnPosition(); - -} diff --git a/Core/src/org/sleuthkit/autopsy/texttranslation/TextTranslationService.java b/Core/src/org/sleuthkit/autopsy/texttranslation/TextTranslationService.java index d6cb5cdaa2..4f20c748ed 100755 --- a/Core/src/org/sleuthkit/autopsy/texttranslation/TextTranslationService.java +++ b/Core/src/org/sleuthkit/autopsy/texttranslation/TextTranslationService.java @@ -25,6 +25,16 @@ import org.openide.util.Lookup; * Service for finding and running TextTranslator implementations */ public class TextTranslationService { + + private final Optional translator; + + /** + * + */ + public TextTranslationService() { + translator = Optional.ofNullable(Lookup.getDefault() + .lookup(TextTranslator.class)); + } /** * Performs a lookup for a TextTranslator service provider and if present, @@ -41,8 +51,6 @@ public class TextTranslationService { * implementations fail */ public String translate(String input) throws NoServiceProviderException, TranslationException { - Optional translator = Optional.ofNullable(Lookup.getDefault() - .lookup(TextTranslator.class)); if (translator.isPresent()) { return translator.get().translate(input); } diff --git a/Core/src/org/sleuthkit/autopsy/texttranslation/TextTranslator.java b/Core/src/org/sleuthkit/autopsy/texttranslation/TextTranslator.java index 86683d699e..e8f8fa94d2 100755 --- a/Core/src/org/sleuthkit/autopsy/texttranslation/TextTranslator.java +++ b/Core/src/org/sleuthkit/autopsy/texttranslation/TextTranslator.java @@ -25,4 +25,12 @@ package org.sleuthkit.autopsy.texttranslation; public interface TextTranslator { public String translate(String input) throws TranslationException; + + public default String[] translate(String[] inputs) throws TranslationException { + String[] outputs = new String[inputs.length]; + for(int i = 0; i < inputs.length; i++) { + outputs[i] = translate(inputs[i]); + } + return outputs; + } } From 4ce87674c147500dddba6e2917d701dcb0644b82 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Wed, 24 Oct 2018 09:36:46 -0400 Subject: [PATCH 016/129] Fixed imports and remove unused stuff --- .../datamodel/AbstractAbstractFileNode.java | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java index a7bcbf9299..16cdf400d9 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java @@ -20,35 +20,27 @@ package org.sleuthkit.autopsy.datamodel; import java.beans.PropertyChangeEvent; import java.beans.PropertyChangeListener; -import java.lang.reflect.InvocationTargetException; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.EnumSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; import java.util.logging.Level; import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; import org.openide.nodes.Children; import org.openide.nodes.Sheet; -import org.openide.util.Exceptions; -import org.openide.util.Lookup; import org.openide.util.NbBundle; import org.openide.util.WeakListeners; -import org.openide.util.lookup.Lookups; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; import org.sleuthkit.autopsy.casemodule.events.CommentChangedEvent; import org.sleuthkit.autopsy.casemodule.events.ContentTagAddedEvent; import org.sleuthkit.autopsy.casemodule.events.ContentTagDeletedEvent; -import org.sleuthkit.autopsy.casemodule.events.TranslationAvailableEvent; import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeInstance; import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeNormalizationException; import org.sleuthkit.autopsy.centralrepository.datamodel.EamArtifactUtil; @@ -249,6 +241,8 @@ public abstract class AbstractAbstractFileNode extends A }, SCORE(AbstractAbstractFileNode_createSheet_score_name()) { Optional> result = Optional.empty(); + List scoreAndCommentTags; + CorrelationAttributeInstance correlationAttribute; private void initResult(AbstractFile content) { scoreAndCommentTags = getContentTagsFromDatabase(content); @@ -275,6 +269,8 @@ public abstract class AbstractAbstractFileNode extends A }, COMMENT(AbstractAbstractFileNode_createSheet_comment_name()) { Optional> result = Optional.empty(); + List scoreAndCommentTags; + CorrelationAttributeInstance correlationAttribute; private void initResult(AbstractFile content) { scoreAndCommentTags = getContentTagsFromDatabase(content); @@ -308,8 +304,11 @@ public abstract class AbstractAbstractFileNode extends A }, COUNT(AbstractAbstractFileNode_createSheet_count_name()) { Optional> result = Optional.empty(); + List scoreAndCommentTags; + CorrelationAttributeInstance correlationAttribute; private void initResult(AbstractFile content) { + correlationAttribute = getCorrelationAttributeInstance(content); result = Optional.of(getCountProperty(correlationAttribute)); } @@ -459,8 +458,6 @@ public abstract class AbstractAbstractFileNode extends A } }; - private static List scoreAndCommentTags; - private static CorrelationAttributeInstance correlationAttribute; final private String displayString; private AbstractFilePropertyType(String displayString) { From 1dda52aa03e7d4a5fa97dd563b5b6f3c253b9032 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Wed, 24 Oct 2018 14:45:43 -0400 Subject: [PATCH 017/129] 4305 added correlation attribute for wifi networks --- .../CorrelationAttributeInstance.java | 41 ++++++++++++------- .../CorrelationAttributeNormalizer.java | 4 ++ .../datamodel/EamArtifactUtil.java | 26 ++++++++---- .../recentactivity/ExtractRegistry.java | 8 +++- 4 files changed, 55 insertions(+), 24 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeInstance.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeInstance.java index c26134c5b8..eb790f866b 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeInstance.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeInstance.java @@ -171,7 +171,7 @@ public class CorrelationAttributeInstance implements Serializable { * Is this a database instance? * * @return True if the instance ID is greater or equal to zero; otherwise - * false. + * false. */ public boolean isDatabaseInstance() { return (ID >= 0); @@ -234,7 +234,7 @@ public class CorrelationAttributeInstance implements Serializable { * as notable and should never be set to KNOWN. * * @param knownStatus Should be BAD if the item is tagged as notable, - * UNKNOWN otherwise + * UNKNOWN otherwise */ public void setKnownStatus(TskData.FileKnown knownStatus) { this.knownStatus = knownStatus; @@ -246,18 +246,24 @@ public class CorrelationAttributeInstance implements Serializable { public static final int EMAIL_TYPE_ID = 2; public static final int PHONE_TYPE_ID = 3; public static final int USBID_TYPE_ID = 4; + public static final int SSID_TYPE_ID = 5; + public static final int MAC_TYPE_ID = 6; + public static final int IMEI_TYPE_ID = 7; /** * Load the default correlation types * * @throws EamDbException if the Type's dbTableName has invalid - * characters/format + * characters/format */ @Messages({"CorrelationType.FILES.displayName=Files", "CorrelationType.DOMAIN.displayName=Domains", "CorrelationType.EMAIL.displayName=Email Addresses", "CorrelationType.PHONE.displayName=Phone Numbers", - "CorrelationType.USBID.displayName=USB Devices"}) + "CorrelationType.USBID.displayName=USB Devices", + "CorrelationType.SSID.displayName=Wireless Networks", + "CorrelationType.MAC.displayName=MAC Address", + "CorrelationType.IMEI.displayName=IMEI Number"}) public static List getDefaultCorrelationTypes() throws EamDbException { List DEFAULT_CORRELATION_TYPES = new ArrayList<>(); DEFAULT_CORRELATION_TYPES.add(new CorrelationAttributeInstance.Type(FILES_TYPE_ID, Bundle.CorrelationType_FILES_displayName(), "file", true, true)); // NON-NLS @@ -265,6 +271,9 @@ public class CorrelationAttributeInstance implements Serializable { DEFAULT_CORRELATION_TYPES.add(new CorrelationAttributeInstance.Type(EMAIL_TYPE_ID, Bundle.CorrelationType_EMAIL_displayName(), "email_address", true, true)); // NON-NLS DEFAULT_CORRELATION_TYPES.add(new CorrelationAttributeInstance.Type(PHONE_TYPE_ID, Bundle.CorrelationType_PHONE_displayName(), "phone_number", true, true)); // NON-NLS DEFAULT_CORRELATION_TYPES.add(new CorrelationAttributeInstance.Type(USBID_TYPE_ID, Bundle.CorrelationType_USBID_displayName(), "usb_devices", true, true)); // NON-NLS + DEFAULT_CORRELATION_TYPES.add(new CorrelationAttributeInstance.Type(SSID_TYPE_ID, Bundle.CorrelationType_SSID_displayName(), "wireless_networks", true, true)); // NON-NLS + DEFAULT_CORRELATION_TYPES.add(new CorrelationAttributeInstance.Type(MAC_TYPE_ID, Bundle.CorrelationType_MAC_displayName(), "mac_address", true, true)); // NON-NLS + DEFAULT_CORRELATION_TYPES.add(new CorrelationAttributeInstance.Type(IMEI_TYPE_ID, Bundle.CorrelationType_IMEI_displayName(), "imei_number", true, true)); // NON-NLS return DEFAULT_CORRELATION_TYPES; } @@ -283,13 +292,14 @@ public class CorrelationAttributeInstance implements Serializable { /** * - * @param typeId Unique ID for this Correlation Type + * @param typeId Unique ID for this Correlation Type * @param displayName Name of this type displayed in the UI. * @param dbTableName Central repository db table where data of this - * type is stored. Must start with a lowercase letter and only contain - * lowercase letters, numbers, and '_' characters. - * @param supported Is this Type currently supported - * @param enabled Is this Type currently enabled. + * type is stored. Must start with a lowercase letter + * and only contain lowercase letters, numbers, and + * '_' characters. + * @param supported Is this Type currently supported + * @param enabled Is this Type currently enabled. */ public Type(int typeId, String displayName, String dbTableName, Boolean supported, Boolean enabled) throws EamDbException { if (dbTableName == null) { @@ -312,10 +322,11 @@ public class CorrelationAttributeInstance implements Serializable { * * @param displayName Name of this type displayed in the UI. * @param dbTableName Central repository db table where data of this - * type is stored Must start with a lowercase letter and only contain - * lowercase letters, numbers, and '_' characters. - * @param supported Is this Type currently supported - * @param enabled Is this Type currently enabled. + * type is stored Must start with a lowercase letter + * and only contain lowercase letters, numbers, and + * '_' characters. + * @param supported Is this Type currently supported + * @param enabled Is this Type currently enabled. */ public Type(String displayName, String dbTableName, Boolean supported, Boolean enabled) throws EamDbException { this(-1, displayName, dbTableName, supported, enabled); @@ -477,8 +488,8 @@ public class CorrelationAttributeInstance implements Serializable { * custom_instances) * * @param dbTableName the dbTableName to set. Must start with lowercase - * letter and can only contain lowercase letters, numbers, and '_' - * characters. + * letter and can only contain lowercase letters, + * numbers, and '_' characters. * * @throws EamDbException if dbTableName contains invalid characters */ diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeNormalizer.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeNormalizer.java index 772e1c517e..c7787389f4 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeNormalizer.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeNormalizer.java @@ -63,6 +63,10 @@ final public class CorrelationAttributeNormalizer { return normalizePhone(data); case CorrelationAttributeInstance.USBID_TYPE_ID: return normalizeUsbId(data); + case CorrelationAttributeInstance.SSID_TYPE_ID: + case CorrelationAttributeInstance.MAC_TYPE_ID: + case CorrelationAttributeInstance.IMEI_TYPE_ID: + return data; default: final String errorMessage = String.format( "Validator function not found for attribute type: %s", diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactUtil.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactUtil.java index 30d539e87f..c3b18e62f5 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactUtil.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactUtil.java @@ -54,9 +54,9 @@ public class EamArtifactUtil { * EamArtifact with a single EamArtifactInstance within. If not, return * null. * - * @param bbArtifact BlackboardArtifact to examine + * @param bbArtifact BlackboardArtifact to examine * @param checkEnabled If true, only create a CorrelationAttribute if it is - * enabled + * enabled * * @return List of EamArtifacts */ @@ -93,10 +93,10 @@ public class EamArtifactUtil { * based on the data in the blackboard artifact. * * @param correlationType The Central Repository artifact type to create - * @param bbArtifact The blackboard artifact to pull data from + * @param bbArtifact The blackboard artifact to pull data from * * @return the new EamArtifact, or null if one was not created because - * bbArtifact did not contain the needed data + * bbArtifact did not contain the needed data */ private static CorrelationAttributeInstance makeInstanceFromBlackboardArtifact(CorrelationAttributeInstance.Type correlationType, BlackboardArtifact bbArtifact) throws EamDbException { @@ -164,7 +164,16 @@ public class EamArtifactUtil { && BlackboardArtifact.ARTIFACT_TYPE.TSK_DEVICE_ATTACHED.getTypeID() == artifactTypeID) { value = bbArtifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DEVICE_ID)).getValueString(); - } + } + else if (correlationType.getId() == CorrelationAttributeInstance.SSID_TYPE_ID + && BlackboardArtifact.ARTIFACT_TYPE.TSK_WIFI_NETWORK.getTypeID() == artifactTypeID) { + value = bbArtifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SSID)).getValueString(); + } +// else if (correlationType.getId() == CorrelationAttributeInstance.MAC_TYPE_ID) { +// value = bbArtifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_BSSID)).getValueString(); +// } else if (correlationType.getId() == CorrelationAttributeInstance.IMEI_TYPE_ID) { +// value = bbArtifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_BSSID)).getValueString(); +// } } catch (TskCoreException ex) { logger.log(Level.SEVERE, "Error getting attribute while getting type from BlackboardArtifact.", ex); // NON-NLS @@ -185,9 +194,10 @@ public class EamArtifactUtil { * Uses the determined type and vallue, then looks up instance details to * create proper CorrelationAttributeInstance. * - * @param bbArtifact the blackboard artifatc + * @param bbArtifact the blackboard artifatc * @param correlationType the given type - * @param value the artifact value + * @param value the artifact value + * * @return CorrelationAttributeInstance from details */ private static CorrelationAttributeInstance makeCorrelationAttributeInstanceUsingTypeValue(BlackboardArtifact bbArtifact, CorrelationAttributeInstance.Type correlationType, String value) { @@ -340,7 +350,7 @@ public class EamArtifactUtil { * @param file The file to test * * @return true if the file should be added to the central repo, false - * otherwise + * otherwise */ public static boolean isSupportedAbstractFileType(AbstractFile file) { if (file == null) { diff --git a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java index 70607f361c..6612feb60c 100644 --- a/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java +++ b/RecentActivity/src/org/sleuthkit/autopsy/recentactivity/ExtractRegistry.java @@ -374,7 +374,9 @@ class ExtractRegistry extends Extract { // Add all "usb" dataType nodes to collection of BlackboardArtifacts // that we will submit in a ModuleDataEvent for additional processing. Collection usbBBartifacts = new ArrayList<>(); - + // Add all "ssid" dataType nodes to collection of BlackboardArtifacts + // that we will submit in a ModuleDataEvent for additional processing. + Collection wifiBBartifacts = new ArrayList<>(); for (int i = 0; i < len; i++) { Element tempnode = (Element) children.item(i); @@ -734,6 +736,7 @@ class ExtractRegistry extends Extract { bbart.addAttributes(bbattributes); // index the artifact for keyword search this.indexArtifact(bbart); + wifiBBartifacts.add(bbart); } catch (TskCoreException ex) { logger.log(Level.SEVERE, "Error adding SSID artifact to blackboard."); //NON-NLS } @@ -756,6 +759,9 @@ class ExtractRegistry extends Extract { if (!usbBBartifacts.isEmpty()) { IngestServices.getInstance().fireModuleDataEvent(new ModuleDataEvent(moduleName, BlackboardArtifact.ARTIFACT_TYPE.TSK_DEVICE_ATTACHED, usbBBartifacts)); } + if (!wifiBBartifacts.isEmpty()){ + IngestServices.getInstance().fireModuleDataEvent(new ModuleDataEvent(moduleName, BlackboardArtifact.ARTIFACT_TYPE.TSK_WIFI_NETWORK, wifiBBartifacts)); + } return true; } catch (FileNotFoundException ex) { logger.log(Level.SEVERE, "Error finding the registry file."); //NON-NLS From 57db0cd43f8d81f72d66692a86df3eb3000efbfb Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Thu, 25 Oct 2018 11:51:44 -0400 Subject: [PATCH 018/129] Add test translation of file names --- Core/nbproject/project.xml | 1 + .../sleuthkit/autopsy/casemodule/Case.java | 2 +- .../autopsy/core/UserPreferences.java | 11 ++- .../autopsy/corecomponents/Bundle.properties | 2 + .../corecomponents/ViewPreferencesPanel.form | 35 ++++++-- .../corecomponents/ViewPreferencesPanel.java | 52 +++++++++-- .../datamodel/AbstractAbstractFileNode.java | 42 +++++++++ .../datamodel/AbstractFsContentNode.java | 7 ++ .../datamodel/BlackboardArtifactNode.java | 2 +- .../autopsy/datamodel/LayoutFileNode.java | 10 +++ .../autopsy/datamodel/LocalDirectoryNode.java | 9 ++ .../autopsy/datamodel/LocalFileNode.java | 16 ++++ .../datamodel/VirtualDirectoryNode.java | 9 ++ .../DirectoryTreeTopComponent.java | 1 + .../NoServiceProviderException.java | 17 ++++ .../TextTranslationService.java | 89 +++++++++++++++++++ .../texttranslation/TextTranslator.java | 36 ++++++++ .../texttranslation/TranslationCallback.java | 19 ++++ .../texttranslation/TranslationException.java | 51 +++++++++++ 19 files changed, 395 insertions(+), 16 deletions(-) create mode 100755 Core/src/org/sleuthkit/autopsy/texttranslation/NoServiceProviderException.java create mode 100755 Core/src/org/sleuthkit/autopsy/texttranslation/TextTranslationService.java create mode 100755 Core/src/org/sleuthkit/autopsy/texttranslation/TextTranslator.java create mode 100755 Core/src/org/sleuthkit/autopsy/texttranslation/TranslationCallback.java create mode 100755 Core/src/org/sleuthkit/autopsy/texttranslation/TranslationException.java diff --git a/Core/nbproject/project.xml b/Core/nbproject/project.xml index d142e0b8c9..de39ea6f9a 100644 --- a/Core/nbproject/project.xml +++ b/Core/nbproject/project.xml @@ -339,6 +339,7 @@ org.sleuthkit.autopsy.progress org.sleuthkit.autopsy.report org.sleuthkit.autopsy.tabulardatareader + org.sleuthkit.autopsy.texttranslation org.sleuthkit.datamodel diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/Case.java b/Core/src/org/sleuthkit/autopsy/casemodule/Case.java index dcb2af3eef..b5624f8fa2 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/Case.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/Case.java @@ -1475,7 +1475,7 @@ public class Case { public void notifyAddingDataSource(UUID eventId) { eventPublisher.publish(new AddingDataSourceEvent(eventId)); } - + /** * Notifies case event subscribers that a data source failed to be added to * the case. diff --git a/Core/src/org/sleuthkit/autopsy/core/UserPreferences.java b/Core/src/org/sleuthkit/autopsy/core/UserPreferences.java index cb5f797c2b..3dc230cf21 100644 --- a/Core/src/org/sleuthkit/autopsy/core/UserPreferences.java +++ b/Core/src/org/sleuthkit/autopsy/core/UserPreferences.java @@ -72,7 +72,8 @@ public final class UserPreferences { public static final String GROUP_ITEMS_IN_TREE_BY_DATASOURCE = "GroupItemsInTreeByDataSource"; //NON-NLS public static final String SHOW_ONLY_CURRENT_USER_TAGS = "ShowOnlyCurrentUserTags"; public static final String HIDE_CENTRAL_REPO_COMMENTS_AND_OCCURRENCES = "HideCentralRepoCommentsAndOccurrences"; - + public static final String DISPLAY_TRANSLATED_NAMES = "DisplayTranslatedNames"; + // Prevent instantiation. private UserPreferences() { } @@ -244,6 +245,14 @@ public final class UserPreferences { public static void setHideCentralRepoCommentsAndOccurrences(boolean value) { preferences.putBoolean(HIDE_CENTRAL_REPO_COMMENTS_AND_OCCURRENCES, value); } + + public static void setDisplayTranslationFileNames(boolean value) { + preferences.putBoolean(DISPLAY_TRANSLATED_NAMES, value); + } + + public static boolean displayTranslationFileNames() { + return preferences.getBoolean(DISPLAY_TRANSLATED_NAMES, false); + } /** * Reads persisted case database connection info. diff --git a/Core/src/org/sleuthkit/autopsy/corecomponents/Bundle.properties b/Core/src/org/sleuthkit/autopsy/corecomponents/Bundle.properties index 12ba493380..efd861fb24 100644 --- a/Core/src/org/sleuthkit/autopsy/corecomponents/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/corecomponents/Bundle.properties @@ -192,3 +192,5 @@ ViewPreferencesPanel.centralRepoLabel.text=Do not use Central Repository for: ViewPreferencesPanel.commentsOccurencesColumnsCheckbox.text=C(omments) and O(ccurences) columns to reduce loading times ViewPreferencesPanel.deletedFilesLimitCheckbox.text=Limit to 10,000 ViewPreferencesPanel.deletedFilesLimitLabel.text=Limit number of deleted files displayed: +ViewPreferencesPanel.fileDisplayLabel.text=When displaying files: +ViewPreferencesPanel.translatedNamesButton.text=Show translated file names diff --git a/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.form b/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.form index ddeb57bd74..096c24af11 100755 --- a/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.form +++ b/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.form @@ -92,6 +92,8 @@ + + @@ -114,9 +116,11 @@ + + @@ -125,14 +129,12 @@ + - - - @@ -174,9 +176,15 @@ - + + + + - + + + + @@ -356,6 +364,23 @@ + + + + + + + + + + + + + + + + + diff --git a/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.java b/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.java index 51d4120449..2d7cb019c8 100755 --- a/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.java +++ b/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.java @@ -27,6 +27,7 @@ import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbUtil; import org.sleuthkit.autopsy.core.UserPreferences; import org.sleuthkit.autopsy.deletedFiles.DeletedFilePreferences; import org.sleuthkit.autopsy.directorytree.DirectoryTreeTopComponent; +import org.sleuthkit.autopsy.texttranslation.TextTranslationService; /** * Panel for configuring view preferences. @@ -44,6 +45,12 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { public ViewPreferencesPanel(boolean immediateUpdates) { initComponents(); this.immediateUpdates = immediateUpdates; + + TextTranslationService translationService = new TextTranslationService(); + if(!translationService.hasProvider()) { + translatedNamesButton.setVisible(false); + fileDisplayLabel.setVisible(false); + } } @Override @@ -67,6 +74,7 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { commentsOccurencesColumnsCheckbox.setSelected(UserPreferences.hideCentralRepoCommentsAndOccurrences()); deletedFilesLimitCheckbox.setSelected(DeletedFilePreferences.getDefault().getShouldLimitDeletedFiles()); + translatedNamesButton.setSelected(UserPreferences.displayTranslationFileNames()); // Current Case Settings boolean caseIsOpen = Case.isCaseOpen(); @@ -90,6 +98,7 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { UserPreferences.setHideSlackFilesInViewsTree(viewsHideSlackCheckbox.isSelected()); UserPreferences.setShowOnlyCurrentUserTags(hideOtherUsersTagsCheckbox.isSelected()); UserPreferences.setHideCentralRepoCommentsAndOccurrences(commentsOccurencesColumnsCheckbox.isSelected()); + UserPreferences.setDisplayTranslationFileNames(translatedNamesButton.isSelected()); storeGroupItemsInTreeByDataSource(); @@ -142,6 +151,8 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { centralRepoLabel = new javax.swing.JLabel(); deletedFilesLimitCheckbox = new javax.swing.JCheckBox(); deletedFilesLimitLabel = new javax.swing.JLabel(); + translatedNamesButton = new javax.swing.JRadioButton(); + fileDisplayLabel = new javax.swing.JLabel(); currentCaseSettingsPanel = new javax.swing.JPanel(); groupByDataSourceCheckbox = new javax.swing.JCheckBox(); currentSessionSettingsPanel = new javax.swing.JPanel(); @@ -244,6 +255,15 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { org.openide.awt.Mnemonics.setLocalizedText(deletedFilesLimitLabel, org.openide.util.NbBundle.getMessage(ViewPreferencesPanel.class, "ViewPreferencesPanel.deletedFilesLimitLabel.text")); // NOI18N + org.openide.awt.Mnemonics.setLocalizedText(translatedNamesButton, org.openide.util.NbBundle.getMessage(ViewPreferencesPanel.class, "ViewPreferencesPanel.translatedNamesButton.text")); // NOI18N + translatedNamesButton.addActionListener(new java.awt.event.ActionListener() { + public void actionPerformed(java.awt.event.ActionEvent evt) { + translatedNamesButtonActionPerformed(evt); + } + }); + + org.openide.awt.Mnemonics.setLocalizedText(fileDisplayLabel, org.openide.util.NbBundle.getMessage(ViewPreferencesPanel.class, "ViewPreferencesPanel.fileDisplayLabel.text")); // NOI18N + javax.swing.GroupLayout globalSettingsPanelLayout = new javax.swing.GroupLayout(globalSettingsPanel); globalSettingsPanel.setLayout(globalSettingsPanelLayout); globalSettingsPanelLayout.setHorizontalGroup( @@ -259,6 +279,8 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { .addComponent(deletedFilesLimitCheckbox, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE))) .addGroup(globalSettingsPanelLayout.createSequentialGroup() .addGroup(globalSettingsPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addComponent(centralRepoLabel) + .addComponent(deletedFilesLimitLabel, javax.swing.GroupLayout.PREFERRED_SIZE, 215, javax.swing.GroupLayout.PREFERRED_SIZE) .addGroup(globalSettingsPanelLayout.createSequentialGroup() .addGroup(globalSettingsPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addComponent(hideKnownFilesLabel) @@ -274,9 +296,11 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { .addGap(10, 10, 10) .addGroup(globalSettingsPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addComponent(dataSourcesHideKnownCheckbox) - .addComponent(viewsHideKnownCheckbox))))) + .addComponent(viewsHideKnownCheckbox)))) + .addComponent(hideOtherUsersTagsLabel)) .addGap(18, 18, 18) .addGroup(globalSettingsPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addComponent(fileDisplayLabel) .addComponent(displayTimeLabel) .addGroup(globalSettingsPanelLayout.createSequentialGroup() .addGap(10, 10, 10) @@ -284,11 +308,9 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { .addComponent(keepCurrentViewerRadioButton) .addComponent(useBestViewerRadioButton) .addComponent(useGMTTimeRadioButton) - .addComponent(useLocalTimeRadioButton))) - .addComponent(selectFileLabel))) - .addComponent(hideOtherUsersTagsLabel) - .addComponent(centralRepoLabel) - .addComponent(deletedFilesLimitLabel, javax.swing.GroupLayout.PREFERRED_SIZE, 215, javax.swing.GroupLayout.PREFERRED_SIZE)) + .addComponent(useLocalTimeRadioButton) + .addComponent(translatedNamesButton))) + .addComponent(selectFileLabel)))) .addGap(0, 10, Short.MAX_VALUE))) .addContainerGap()) ); @@ -322,9 +344,13 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) .addComponent(useGMTTimeRadioButton))) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) - .addComponent(hideOtherUsersTagsLabel) + .addGroup(globalSettingsPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) + .addComponent(hideOtherUsersTagsLabel) + .addComponent(fileDisplayLabel)) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) - .addComponent(hideOtherUsersTagsCheckbox) + .addGroup(globalSettingsPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) + .addComponent(hideOtherUsersTagsCheckbox) + .addComponent(translatedNamesButton)) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) .addComponent(centralRepoLabel) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) @@ -535,6 +561,14 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { } }//GEN-LAST:event_deletedFilesLimitCheckboxActionPerformed + private void translatedNamesButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_translatedNamesButtonActionPerformed + if (immediateUpdates) { + UserPreferences.setDisplayTranslationFileNames(translatedNamesButton.isSelected()); + } else { + firePropertyChange(OptionsPanelController.PROP_CHANGED, null, null); + } + }//GEN-LAST:event_translatedNamesButtonActionPerformed + // Variables declaration - do not modify//GEN-BEGIN:variables private javax.swing.JLabel centralRepoLabel; @@ -546,6 +580,7 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { private javax.swing.JCheckBox deletedFilesLimitCheckbox; private javax.swing.JLabel deletedFilesLimitLabel; private javax.swing.JLabel displayTimeLabel; + private javax.swing.JLabel fileDisplayLabel; private javax.swing.JPanel globalSettingsPanel; private javax.swing.JCheckBox groupByDataSourceCheckbox; private javax.swing.JLabel hideKnownFilesLabel; @@ -555,6 +590,7 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { private javax.swing.JLabel hideSlackFilesLabel; private javax.swing.JRadioButton keepCurrentViewerRadioButton; private javax.swing.JLabel selectFileLabel; + private javax.swing.JRadioButton translatedNamesButton; private javax.swing.JRadioButton useBestViewerRadioButton; private javax.swing.JRadioButton useGMTTimeRadioButton; private javax.swing.JRadioButton useLocalTimeRadioButton; diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java index 5f8ea3d63f..2045106a69 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java @@ -48,8 +48,13 @@ import org.sleuthkit.autopsy.coreutils.Logger; import static org.sleuthkit.autopsy.datamodel.AbstractAbstractFileNode.AbstractFilePropertyType.*; import static org.sleuthkit.autopsy.datamodel.Bundle.*; import org.sleuthkit.autopsy.corecomponents.DataResultViewerTable.HasCommentStatus; +import org.sleuthkit.autopsy.events.AutopsyEvent; import org.sleuthkit.autopsy.ingest.IngestManager; import org.sleuthkit.autopsy.ingest.ModuleContentEvent; +import org.sleuthkit.autopsy.texttranslation.NoServiceProviderException; +import org.sleuthkit.autopsy.texttranslation.TextTranslationService; +import org.sleuthkit.autopsy.texttranslation.TranslationCallback; +import org.sleuthkit.autopsy.texttranslation.TranslationException; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.Content; @@ -70,6 +75,8 @@ public abstract class AbstractAbstractFileNode extends A private static final Set CASE_EVENTS_OF_INTEREST = EnumSet.of(Case.Events.CURRENT_CASE, Case.Events.CONTENT_TAG_ADDED, Case.Events.CONTENT_TAG_DELETED, Case.Events.CR_COMMENT_CHANGED); + + private String translateFileName = null; /** * @param abstractFile file to wrap @@ -161,6 +168,9 @@ public abstract class AbstractAbstractFileNode extends A if (event.getContentID() == content.getId()) { updateSheet(); } + } else if (eventType.equals("TRANSLATION_AVAILABLE")) { + setTranslatedSourceName((String) evt.getNewValue()); + updateSheet(); } }; @@ -234,7 +244,39 @@ public abstract class AbstractAbstractFileNode extends A return displayString; } } + + public void setTranslatedSourceName(String translation) { + translateFileName = translation; + } + + public String getTranslatedSourceName(TextTranslationService tts) { + if(translateFileName == null) { + if(tts.hasProvider()) { + tts.translate(this.content.getName(), new TranslationCallback() { + @Override + public void onTranslation(String translation) { + //Talk directy to the nodes PCL, to update when ready. + weakPcl.propertyChange(new PropertyChangeEvent( + AutopsyEvent.SourceType.LOCAL.toString(), + "TRANSLATION_AVAILABLE", "", translation)); + } + @Override + public void onTranslationException(TranslationException ex) { + + } + + @Override + public void onNoServiceProviderException(NoServiceProviderException ex) { + //Do nothing. + } + }); + } + return ""; + } + return translateFileName; + } + /** * Fill map with AbstractFile properties * diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractFsContentNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractFsContentNode.java index ef8e1d024e..660957570a 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractFsContentNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractFsContentNode.java @@ -27,6 +27,7 @@ import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeIns import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbUtil; import org.sleuthkit.autopsy.core.UserPreferences; import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.autopsy.texttranslation.TextTranslationService; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.ContentTag; @@ -85,6 +86,12 @@ public abstract class AbstractFsContentNode extends Abst NO_DESCR, getName())); + TextTranslationService tts = new TextTranslationService(); + if(UserPreferences.displayTranslationFileNames()) { + String translation = getTranslatedSourceName(tts); + sheetSet.put(new NodeProperty<>("Translated Name", "Translated Name", "", translation)); + } + addScoreProperty(sheetSet, tags); //add the comment property before the propertyMap to ensure it is early in column order diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/BlackboardArtifactNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/BlackboardArtifactNode.java index 9a322446bc..7269a09e2c 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/BlackboardArtifactNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/BlackboardArtifactNode.java @@ -320,7 +320,7 @@ public class BlackboardArtifactNode extends AbstractContentNode { NbBundle.getMessage(this.getClass(), "LayoutFileNode.createSheet.name.desc"), getName())); + TextTranslationService tts = new TextTranslationService(); + if(tts.hasProvider() && UserPreferences.displayTranslationFileNames()) { + String translation = getTranslatedSourceName(tts); + sheetSet.put(new NodeProperty<>("Translated Name", "Translated Name", "", translation)); + } + addScoreProperty(sheetSet, tags); CorrelationAttributeInstance correlationAttribute = null; diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/LocalDirectoryNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/LocalDirectoryNode.java index f4146e7a55..53ba3aedbc 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/LocalDirectoryNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/LocalDirectoryNode.java @@ -26,6 +26,9 @@ import org.openide.util.NbBundle; import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeInstance; import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbUtil; import org.sleuthkit.autopsy.core.UserPreferences; +import org.sleuthkit.autopsy.texttranslation.NoServiceProviderException; +import org.sleuthkit.autopsy.texttranslation.TextTranslationService; +import org.sleuthkit.autopsy.texttranslation.TranslationException; import org.sleuthkit.datamodel.ContentTag; import org.sleuthkit.datamodel.LocalDirectory; @@ -66,6 +69,12 @@ public class LocalDirectoryNode extends SpecialDirectoryNode { Bundle.LocalDirectoryNode_createSheet_name_desc(), getName())); + TextTranslationService tts = new TextTranslationService(); + if(tts.hasProvider() && UserPreferences.displayTranslationFileNames()) { + String translation = getTranslatedSourceName(tts); + sheetSet.put(new NodeProperty<>("Translated Name", "Translated Name", "", translation)); + } + addScoreProperty(sheetSet, tags); CorrelationAttributeInstance correlationAttribute = null; diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/LocalFileNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/LocalFileNode.java index 0f69666454..3a75a8a3fb 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/LocalFileNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/LocalFileNode.java @@ -42,6 +42,9 @@ import org.sleuthkit.autopsy.directorytree.ExtractAction; import org.sleuthkit.autopsy.directorytree.NewWindowViewAction; import org.sleuthkit.autopsy.directorytree.ViewContextAction; import org.sleuthkit.autopsy.modules.embeddedfileextractor.ExtractArchiveWithPasswordAction; +import org.sleuthkit.autopsy.texttranslation.NoServiceProviderException; +import org.sleuthkit.autopsy.texttranslation.TextTranslationService; +import org.sleuthkit.autopsy.texttranslation.TranslationException; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.ContentTag; @@ -85,6 +88,19 @@ public class LocalFileNode extends AbstractAbstractFileNode { NbBundle.getMessage(this.getClass(), "LocalFileNode.createSheet.name.desc"), getName())); + TextTranslationService tts = new TextTranslationService(); + if(tts.hasProvider() && UserPreferences.displayTranslationFileNames()) { + String translation = ""; + try { + translation = tts.translate(this.content.getName()); + } catch (NoServiceProviderException ex) { + //Do nothing, can't happen due to call for hasProvider(). + } catch (TranslationException ex) { + System.out.println(ex); + } + sheetSet.put(new NodeProperty<>("Translated Name", "Translated Name", "", translation)); + } + addScoreProperty(sheetSet, tags); CorrelationAttributeInstance correlationAttribute = null; diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/VirtualDirectoryNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/VirtualDirectoryNode.java index 719bc3420a..1b9cdf31a9 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/VirtualDirectoryNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/VirtualDirectoryNode.java @@ -32,6 +32,9 @@ import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeIns import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbUtil; import org.sleuthkit.autopsy.core.UserPreferences; import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.autopsy.texttranslation.NoServiceProviderException; +import org.sleuthkit.autopsy.texttranslation.TextTranslationService; +import org.sleuthkit.autopsy.texttranslation.TranslationException; import org.sleuthkit.datamodel.ContentTag; import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.TskCoreException; @@ -92,6 +95,12 @@ public class VirtualDirectoryNode extends SpecialDirectoryNode { NbBundle.getMessage(this.getClass(), "VirtualDirectoryNode.createSheet.name.desc"), getName())); if (!this.content.isDataSource()) { + TextTranslationService tts = new TextTranslationService(); + if(tts.hasProvider() && UserPreferences.displayTranslationFileNames()) { + String translation = getTranslatedSourceName(tts); + sheetSet.put(new NodeProperty<>("Translated Name", "Translated Name", "", translation)); + } + addScoreProperty(sheetSet, tags); CorrelationAttributeInstance correlationAttribute = null; diff --git a/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java b/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java index 17ee1d078f..17d70834bb 100644 --- a/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java +++ b/Core/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java @@ -170,6 +170,7 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat case UserPreferences.HIDE_KNOWN_FILES_IN_DATA_SRCS_TREE: case UserPreferences.HIDE_SLACK_FILES_IN_DATA_SRCS_TREE: case UserPreferences.HIDE_CENTRAL_REPO_COMMENTS_AND_OCCURRENCES: + case UserPreferences.DISPLAY_TRANSLATED_NAMES: case UserPreferences.KEEP_PREFERRED_VIEWER: refreshContentTreeSafe(); break; diff --git a/Core/src/org/sleuthkit/autopsy/texttranslation/NoServiceProviderException.java b/Core/src/org/sleuthkit/autopsy/texttranslation/NoServiceProviderException.java new file mode 100755 index 0000000000..23bddcc999 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/texttranslation/NoServiceProviderException.java @@ -0,0 +1,17 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ +package org.sleuthkit.autopsy.texttranslation; + +/** + * Exception to indicate that no Service Provider could be found during the + * Lookup action. + */ +public class NoServiceProviderException extends Exception { + + public NoServiceProviderException(String msg) { + super(msg); + } +} \ No newline at end of file diff --git a/Core/src/org/sleuthkit/autopsy/texttranslation/TextTranslationService.java b/Core/src/org/sleuthkit/autopsy/texttranslation/TextTranslationService.java new file mode 100755 index 0000000000..a81ff589ab --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/texttranslation/TextTranslationService.java @@ -0,0 +1,89 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2018-2018 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.texttranslation; + +import java.util.Optional; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import org.openide.util.Lookup; + +/** + * Service for finding and running TextTranslator implementations + */ +public class TextTranslationService { + + private final Optional translator; + private final static ExecutorService pool; + private final static Integer MAX_POOL_SIZE = 10; + + static { + pool = Executors.newFixedThreadPool(MAX_POOL_SIZE); + } + + /** + * + */ + public TextTranslationService() { + translator = Optional.ofNullable(Lookup.getDefault() + .lookup(TextTranslator.class)); + } + + /** + * Performs a lookup for a TextTranslator service provider and if present, + * will use this provider to run translation on the input. + * + * @param input Input string to be translated + * + * @return Translation string + * + * @throws NoServiceProviderException Failed to find a Translation service + * provider + * @throws TranslationException System exception for classes to use + * when specific translation + * implementations fail + */ + public String translate(String input) throws NoServiceProviderException, TranslationException { + if (translator.isPresent()) { + return translator.get().translate(input); + } + throw new NoServiceProviderException( + "Could not find a TextTranslator service provider"); + } + + public void translate(String input, TranslationCallback tcb) { + pool.submit(() -> { + try { + String translation = translate(input); + tcb.onTranslation(translation); + } catch (NoServiceProviderException ex) { + tcb.onNoServiceProviderException(ex); + } catch (TranslationException ex) { + tcb.onTranslationException(ex); + } + }); + } + + /** + * + * @return + */ + public boolean hasProvider() { + return translator.isPresent(); + } +} \ No newline at end of file diff --git a/Core/src/org/sleuthkit/autopsy/texttranslation/TextTranslator.java b/Core/src/org/sleuthkit/autopsy/texttranslation/TextTranslator.java new file mode 100755 index 0000000000..380f79876e --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/texttranslation/TextTranslator.java @@ -0,0 +1,36 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2018-2018 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.texttranslation; + +/** + * Interface for creating text translators. Implementing classes will be picked + * up and run by the Text Translation Service. + */ +public interface TextTranslator { + + public String translate(String input) throws TranslationException; + + public default String[] translate(String[] inputs) throws TranslationException { + String[] outputs = new String[inputs.length]; + for(int i = 0; i < inputs.length; i++) { + outputs[i] = translate(inputs[i]); + } + return outputs; + } +} \ No newline at end of file diff --git a/Core/src/org/sleuthkit/autopsy/texttranslation/TranslationCallback.java b/Core/src/org/sleuthkit/autopsy/texttranslation/TranslationCallback.java new file mode 100755 index 0000000000..f390a4297c --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/texttranslation/TranslationCallback.java @@ -0,0 +1,19 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ +package org.sleuthkit.autopsy.texttranslation; + +/** + * + * @author dsmyda + */ +public interface TranslationCallback { + + public void onTranslation(String translation); + + public void onTranslationException(TranslationException ex); + + public void onNoServiceProviderException(NoServiceProviderException ex); +} diff --git a/Core/src/org/sleuthkit/autopsy/texttranslation/TranslationException.java b/Core/src/org/sleuthkit/autopsy/texttranslation/TranslationException.java new file mode 100755 index 0000000000..1fa59123fc --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/texttranslation/TranslationException.java @@ -0,0 +1,51 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2018-2018 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.texttranslation; + +/** + * Provides a system exception for the Text Translation errors + */ +public class TranslationException extends Exception { + + /** + * Constructs a new exception with null as its message. + */ + public TranslationException() { + super(); + } + + /** + * Constructs a new exception with the specified message. + * + * @param message The message. + */ + public TranslationException(String message) { + super(message); + } + + /** + * Constructs a new exception with the specified message and cause. + * + * @param message The message. + * @param cause The cause. + */ + public TranslationException(String message, Throwable cause) { + super(message, cause); + } +} \ No newline at end of file From 4e17668db87613f58522432dcae3366f1e21d95f Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Thu, 25 Oct 2018 14:32:11 -0400 Subject: [PATCH 019/129] Added some comments and move stuff around to make it clearer --- .../corecomponents/ViewPreferencesPanel.java | 4 +- .../datamodel/AbstractAbstractFileNode.java | 147 +++++++++++++----- .../datamodel/AbstractFsContentNode.java | 5 +- .../autopsy/datamodel/LayoutFileNode.java | 5 +- .../autopsy/datamodel/LocalDirectoryNode.java | 5 +- .../autopsy/datamodel/LocalFileNode.java | 12 +- .../datamodel/VirtualDirectoryNode.java | 5 +- .../NoServiceProviderException.java | 19 ++- .../TextTranslationService.java | 74 +++++---- .../texttranslation/TextTranslator.java | 7 - .../texttranslation/TranslationCallback.java | 37 ++++- 11 files changed, 218 insertions(+), 102 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.java b/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.java index 2d7cb019c8..4a9fce6f8d 100755 --- a/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.java +++ b/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.java @@ -46,8 +46,8 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { initComponents(); this.immediateUpdates = immediateUpdates; - TextTranslationService translationService = new TextTranslationService(); - if(!translationService.hasProvider()) { + TextTranslationService tts = new TextTranslationService(); + if(!tts.hasProvider()) { translatedNamesButton.setVisible(false); fileDisplayLabel.setVisible(false); } diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java index 2045106a69..276b67a37b 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java @@ -27,6 +27,7 @@ import java.util.Map; import java.util.Set; import java.util.logging.Level; import java.util.stream.Collectors; +import org.apache.commons.io.FilenameUtils; import org.apache.commons.lang3.StringUtils; import org.openide.nodes.Children; import org.openide.nodes.Sheet; @@ -72,11 +73,14 @@ public abstract class AbstractAbstractFileNode extends A private static final Logger logger = Logger.getLogger(AbstractAbstractFileNode.class.getName()); @NbBundle.Messages("AbstractAbstractFileNode.addFileProperty.desc=no description") private static final String NO_DESCR = AbstractAbstractFileNode_addFileProperty_desc(); + + private static final String TRANSLATION_AVAILABLE_EVENT = "TRANSLATION_AVAILABLE"; + private static final String NO_TRANSLATION = ""; private static final Set CASE_EVENTS_OF_INTEREST = EnumSet.of(Case.Events.CURRENT_CASE, Case.Events.CONTENT_TAG_ADDED, Case.Events.CONTENT_TAG_DELETED, Case.Events.CR_COMMENT_CHANGED); - - private String translateFileName = null; + + private String translatedFileName; /** * @param abstractFile file to wrap @@ -92,6 +96,7 @@ public abstract class AbstractAbstractFileNode extends A IngestManager.getInstance().addIngestModuleEventListener(weakPcl); } } + translatedFileName = null; // Listen for case events so that we can detect when the case is closed // or when tags are added. Case.addEventTypeSubscriber(CASE_EVENTS_OF_INTEREST, weakPcl); @@ -168,8 +173,8 @@ public abstract class AbstractAbstractFileNode extends A if (event.getContentID() == content.getId()) { updateSheet(); } - } else if (eventType.equals("TRANSLATION_AVAILABLE")) { - setTranslatedSourceName((String) evt.getNewValue()); + } else if (eventType.equals(TRANSLATION_AVAILABLE_EVENT)) { + this.translatedFileName = (String) evt.getNewValue(); updateSheet(); } }; @@ -244,39 +249,43 @@ public abstract class AbstractAbstractFileNode extends A return displayString; } } - - public void setTranslatedSourceName(String translation) { - translateFileName = translation; - } - - public String getTranslatedSourceName(TextTranslationService tts) { - if(translateFileName == null) { - if(tts.hasProvider()) { - tts.translate(this.content.getName(), new TranslationCallback() { - @Override - public void onTranslation(String translation) { - //Talk directy to the nodes PCL, to update when ready. - weakPcl.propertyChange(new PropertyChangeEvent( - AutopsyEvent.SourceType.LOCAL.toString(), - "TRANSLATION_AVAILABLE", "", translation)); - } - @Override - public void onTranslationException(TranslationException ex) { - - } - - @Override - public void onNoServiceProviderException(NoServiceProviderException ex) { - //Do nothing. - } - }); - } - return ""; + /** + * Attempts translation on the file name by kicking off a background task to do the + * translation. Once the background task is done, it will fire a PropertyChangeEvent, + * which will force this node to refresh itself, thus updating its translated name + * column. + * + * @return The file names translation. + */ + protected String getTranslatedFileName() { + //If already in complete English, don't translate. + if (this.content.getName().matches("^\\p{ASCII}+$")) { + return NO_TRANSLATION; } - return translateFileName; + + //If we already have a translation use that one. + if (translatedFileName != null) { + return translatedFileName; + } + + //If not, lets fire off a background translation that will update the UI + //when it is done. + TextTranslationService tts = new TextTranslationService(); + if (tts.hasProvider()) { + //Seperate out the base and ext from the contents file name. + String base = FilenameUtils.getBaseName(this.content.getName()); + String ext = FilenameUtils.getExtension(this.content.getName()); + + //Send only the base file name to be translated. Once the translation comes + //back fire a PropertyChangeEvent on this nodes PropertyChangeListener. + tts.translateAsynchronously(base, new TranslateFileNameCallback(ext, weakPcl)); + } + + //In the mean time, return a blank translation. + return NO_TRANSLATION; } - + /** * Fill map with AbstractFile properties * @@ -431,7 +440,7 @@ public abstract class AbstractAbstractFileNode extends A } catch (CorrelationAttributeNormalizationException ex) { logger.log(Level.WARNING, "Unable to normalize data to get count of datasources with correlation attribute", ex); } - + sheetSet.put( new NodeProperty<>(Bundle.AbstractAbstractFileNode_createSheet_count_name(), Bundle.AbstractAbstractFileNode_createSheet_count_displayName(), description, count)); } @@ -442,6 +451,7 @@ public abstract class AbstractAbstractFileNode extends A * * @param sheetSet the modifiable Sheet.Set returned by * Sheet.get(Sheet.PROPERTIES) + * * @deprecated */ @NbBundle.Messages("AbstractAbstractFileNode.tagsProperty.displayName=Tags") @@ -488,6 +498,7 @@ public abstract class AbstractAbstractFileNode extends A * @param file The file. * * @return The CSV list of hash set names. + * * @deprecated */ @Deprecated @@ -499,4 +510,70 @@ public abstract class AbstractAbstractFileNode extends A return ""; } } + + /** + * Implements the TranslationCallback interface so that the TextTranslationService + * can call these methods when an asynchronous translation task is complete. + */ + private class TranslateFileNameCallback implements TranslationCallback { + //Seperate out the base and ext from the contents file name. + private final String ext; + + private final PropertyChangeListener listener; + + public TranslateFileNameCallback(String ext, PropertyChangeListener listener) { + this.ext = ext; + this.listener = listener; + } + + /** + * Fires a PropertyChangeEvent on this nodes PropertyChangeListener + * when the translation is finished. Reconstruct the file name so we + * can properly display the translation. + * + * @param translation Result from the translation job submitted to + * Text translation service. + */ + @Override + public void onTranslationResult(String translation) { + //If we have no extension, then we shouldn't add the . + String extensionDelimiter = (ext.isEmpty()) ? "" : "."; + + //Talk directly to this nodes pcl, fire an update when the translation + //is complete. + if (translation.isEmpty()) { + listener.propertyChange(new PropertyChangeEvent( + AutopsyEvent.SourceType.LOCAL.toString(), + TRANSLATION_AVAILABLE_EVENT, null, translation)); + } else { + listener.propertyChange(new PropertyChangeEvent( + AutopsyEvent.SourceType.LOCAL.toString(), + TRANSLATION_AVAILABLE_EVENT, null, + translation + extensionDelimiter + ext)); + } + } + + /** + * Do nothing on a translation exception, the column will remain empty + * and there is nothing we can do as a fallback. + * + * @param ex Exception caught while translating. + */ + @Override + public void onTranslationException(TranslationException ex) { + //Do nothing + } + + /** + * Do nothing on a no service provider exception, in this implemention we + * are only calling for translation to be done if we know that a TextTranslator + * service provider is already defined. + * + * @param ex + */ + @Override + public void onNoServiceProviderException(NoServiceProviderException ex) { + //Do nothing + } + } } diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractFsContentNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractFsContentNode.java index 660957570a..ed0eb09b8d 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractFsContentNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractFsContentNode.java @@ -85,10 +85,9 @@ public abstract class AbstractFsContentNode extends Abst AbstractFilePropertyType.NAME.toString(), NO_DESCR, getName())); - - TextTranslationService tts = new TextTranslationService(); + if(UserPreferences.displayTranslationFileNames()) { - String translation = getTranslatedSourceName(tts); + String translation = getTranslatedFileName(); sheetSet.put(new NodeProperty<>("Translated Name", "Translated Name", "", translation)); } diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/LayoutFileNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/LayoutFileNode.java index c70e8675d7..56c77d0acd 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/LayoutFileNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/LayoutFileNode.java @@ -98,9 +98,8 @@ public class LayoutFileNode extends AbstractAbstractFileNode { NbBundle.getMessage(this.getClass(), "LayoutFileNode.createSheet.name.desc"), getName())); - TextTranslationService tts = new TextTranslationService(); - if(tts.hasProvider() && UserPreferences.displayTranslationFileNames()) { - String translation = getTranslatedSourceName(tts); + if(UserPreferences.displayTranslationFileNames()) { + String translation = getTranslatedFileName(); sheetSet.put(new NodeProperty<>("Translated Name", "Translated Name", "", translation)); } diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/LocalDirectoryNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/LocalDirectoryNode.java index 53ba3aedbc..8cd255fd5d 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/LocalDirectoryNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/LocalDirectoryNode.java @@ -69,9 +69,8 @@ public class LocalDirectoryNode extends SpecialDirectoryNode { Bundle.LocalDirectoryNode_createSheet_name_desc(), getName())); - TextTranslationService tts = new TextTranslationService(); - if(tts.hasProvider() && UserPreferences.displayTranslationFileNames()) { - String translation = getTranslatedSourceName(tts); + if(UserPreferences.displayTranslationFileNames()) { + String translation = getTranslatedFileName(); sheetSet.put(new NodeProperty<>("Translated Name", "Translated Name", "", translation)); } diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/LocalFileNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/LocalFileNode.java index 3a75a8a3fb..ddbc0dd4ae 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/LocalFileNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/LocalFileNode.java @@ -88,16 +88,8 @@ public class LocalFileNode extends AbstractAbstractFileNode { NbBundle.getMessage(this.getClass(), "LocalFileNode.createSheet.name.desc"), getName())); - TextTranslationService tts = new TextTranslationService(); - if(tts.hasProvider() && UserPreferences.displayTranslationFileNames()) { - String translation = ""; - try { - translation = tts.translate(this.content.getName()); - } catch (NoServiceProviderException ex) { - //Do nothing, can't happen due to call for hasProvider(). - } catch (TranslationException ex) { - System.out.println(ex); - } + if(UserPreferences.displayTranslationFileNames()) { + String translation = getTranslatedFileName(); sheetSet.put(new NodeProperty<>("Translated Name", "Translated Name", "", translation)); } diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/VirtualDirectoryNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/VirtualDirectoryNode.java index 1b9cdf31a9..8de2db173a 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/VirtualDirectoryNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/VirtualDirectoryNode.java @@ -95,9 +95,8 @@ public class VirtualDirectoryNode extends SpecialDirectoryNode { NbBundle.getMessage(this.getClass(), "VirtualDirectoryNode.createSheet.name.desc"), getName())); if (!this.content.isDataSource()) { - TextTranslationService tts = new TextTranslationService(); - if(tts.hasProvider() && UserPreferences.displayTranslationFileNames()) { - String translation = getTranslatedSourceName(tts); + if(UserPreferences.displayTranslationFileNames()) { + String translation = getTranslatedFileName(); sheetSet.put(new NodeProperty<>("Translated Name", "Translated Name", "", translation)); } diff --git a/Core/src/org/sleuthkit/autopsy/texttranslation/NoServiceProviderException.java b/Core/src/org/sleuthkit/autopsy/texttranslation/NoServiceProviderException.java index 23bddcc999..79590fcfd9 100755 --- a/Core/src/org/sleuthkit/autopsy/texttranslation/NoServiceProviderException.java +++ b/Core/src/org/sleuthkit/autopsy/texttranslation/NoServiceProviderException.java @@ -1,7 +1,20 @@ /* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. + * Autopsy Forensic Browser + * + * Copyright 2018-2018 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.sleuthkit.autopsy.texttranslation; diff --git a/Core/src/org/sleuthkit/autopsy/texttranslation/TextTranslationService.java b/Core/src/org/sleuthkit/autopsy/texttranslation/TextTranslationService.java index a81ff589ab..4e35734781 100755 --- a/Core/src/org/sleuthkit/autopsy/texttranslation/TextTranslationService.java +++ b/Core/src/org/sleuthkit/autopsy/texttranslation/TextTranslationService.java @@ -18,28 +18,25 @@ */ package org.sleuthkit.autopsy.texttranslation; +import com.google.common.util.concurrent.ThreadFactoryBuilder; import java.util.Optional; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.ThreadFactory; import org.openide.util.Lookup; /** * Service for finding and running TextTranslator implementations */ public class TextTranslationService { + + private final static Optional translator; - private final Optional translator; - private final static ExecutorService pool; + private static ExecutorService pool; private final static Integer MAX_POOL_SIZE = 10; - + static { - pool = Executors.newFixedThreadPool(MAX_POOL_SIZE); - } - - /** - * - */ - public TextTranslationService() { + //Perform look up for Text Translation implementations translator = Optional.ofNullable(Lookup.getDefault() .lookup(TextTranslator.class)); } @@ -65,25 +62,48 @@ public class TextTranslationService { throw new NoServiceProviderException( "Could not find a TextTranslator service provider"); } - - public void translate(String input, TranslationCallback tcb) { - pool.submit(() -> { - try { - String translation = translate(input); - tcb.onTranslation(translation); - } catch (NoServiceProviderException ex) { - tcb.onNoServiceProviderException(ex); - } catch (TranslationException ex) { - tcb.onTranslationException(ex); - } - }); - } - + /** - * - * @return + * Allows the translation task to happen asynchronously, promising to use + * the TranslationCallback methods when the translation is complete. + * + * @param input + * @param tcb + */ + public void translateAsynchronously(String input, TranslationCallback tcb) { + if(translator.isPresent()) { + //Delay thread pool initialization until an asynchronous task is first called. + //That way we don't have threads sitting parked in the background for no reason. + if (pool == null) { + ThreadFactory translationFactory = new ThreadFactoryBuilder() + .setNameFormat("translation-thread-%d") + .build(); + pool = Executors.newFixedThreadPool(MAX_POOL_SIZE, translationFactory); + } + + pool.submit(() -> { + try { + String translation = translate(input); + tcb.onTranslationResult(translation); + } catch (NoServiceProviderException ex) { + tcb.onNoServiceProviderException(ex); + } catch (TranslationException ex) { + tcb.onTranslationException(ex); + } + }); + } + + tcb.onNoServiceProviderException(new NoServiceProviderException( + "Could not find a TextTranslator service provider")); + } + + /** + * Returns if a TextTranslator lookup successfully found an implementing + * class. + * + * @return */ public boolean hasProvider() { return translator.isPresent(); } -} \ No newline at end of file +} diff --git a/Core/src/org/sleuthkit/autopsy/texttranslation/TextTranslator.java b/Core/src/org/sleuthkit/autopsy/texttranslation/TextTranslator.java index 380f79876e..4a813ed6cc 100755 --- a/Core/src/org/sleuthkit/autopsy/texttranslation/TextTranslator.java +++ b/Core/src/org/sleuthkit/autopsy/texttranslation/TextTranslator.java @@ -26,11 +26,4 @@ public interface TextTranslator { public String translate(String input) throws TranslationException; - public default String[] translate(String[] inputs) throws TranslationException { - String[] outputs = new String[inputs.length]; - for(int i = 0; i < inputs.length; i++) { - outputs[i] = translate(inputs[i]); - } - return outputs; - } } \ No newline at end of file diff --git a/Core/src/org/sleuthkit/autopsy/texttranslation/TranslationCallback.java b/Core/src/org/sleuthkit/autopsy/texttranslation/TranslationCallback.java index f390a4297c..4cbdb79265 100755 --- a/Core/src/org/sleuthkit/autopsy/texttranslation/TranslationCallback.java +++ b/Core/src/org/sleuthkit/autopsy/texttranslation/TranslationCallback.java @@ -1,19 +1,44 @@ /* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. + * Autopsy Forensic Browser + * + * Copyright 2018-2018 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.sleuthkit.autopsy.texttranslation; /** - * - * @author dsmyda + * Provides a way for translation results to be processed when they are + * */ public interface TranslationCallback { - public void onTranslation(String translation); + /** + * + * @param translation + */ + public void onTranslationResult(String translation); + /** + * + * @param ex + */ public void onTranslationException(TranslationException ex); + /** + * + * @param ex + */ public void onNoServiceProviderException(NoServiceProviderException ex); } From 7401fbe32caa7dd7cd8bc9b1394e4390c8e85674 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Thu, 25 Oct 2018 15:17:22 -0400 Subject: [PATCH 020/129] Added some comments --- .../corecomponents/ViewPreferencesPanel.java | 2 ++ .../TextTranslationService.java | 35 +++++++++++-------- .../texttranslation/TranslationCallback.java | 11 ++++-- 3 files changed, 30 insertions(+), 18 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.java b/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.java index 4a9fce6f8d..c6bc396fbe 100755 --- a/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.java +++ b/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.java @@ -46,6 +46,8 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { initComponents(); this.immediateUpdates = immediateUpdates; + //If there is not Text Translator implementation, then hide these buttons + //from the user. TextTranslationService tts = new TextTranslationService(); if(!tts.hasProvider()) { translatedNamesButton.setVisible(false); diff --git a/Core/src/org/sleuthkit/autopsy/texttranslation/TextTranslationService.java b/Core/src/org/sleuthkit/autopsy/texttranslation/TextTranslationService.java index 4e35734781..bac1ee9fb3 100755 --- a/Core/src/org/sleuthkit/autopsy/texttranslation/TextTranslationService.java +++ b/Core/src/org/sleuthkit/autopsy/texttranslation/TextTranslationService.java @@ -31,7 +31,7 @@ import org.openide.util.Lookup; public class TextTranslationService { private final static Optional translator; - + private static ExecutorService pool; private final static Integer MAX_POOL_SIZE = 10; @@ -42,8 +42,8 @@ public class TextTranslationService { } /** - * Performs a lookup for a TextTranslator service provider and if present, - * will use this provider to run translation on the input. + * Translates the input string using whichever TextTranslator Service Provider + * was found during lookup. * * @param input Input string to be translated * @@ -64,23 +64,28 @@ public class TextTranslationService { } /** - * Allows the translation task to happen asynchronously, promising to use - * the TranslationCallback methods when the translation is complete. + * Makes the call to translate(String) happen asynchronously on a background + * thread. When it is done, it will use the appropriate TranslationCallback + * method. + * + * @param input String to be translated + * @param tcb Interface for handling the translation result or any + * exceptions thrown while running translate. * - * @param input - * @param tcb */ public void translateAsynchronously(String input, TranslationCallback tcb) { - if(translator.isPresent()) { + if (translator.isPresent()) { //Delay thread pool initialization until an asynchronous task is first called. //That way we don't have threads sitting parked in the background for no reason. if (pool == null) { ThreadFactory translationFactory = new ThreadFactoryBuilder() - .setNameFormat("translation-thread-%d") - .build(); + .setNameFormat("translation-thread-%d") + .build(); pool = Executors.newFixedThreadPool(MAX_POOL_SIZE, translationFactory); } - + + //Submit the task to the pool, calling the appropriate method depending + //on the result of the translate operation. pool.submit(() -> { try { String translation = translate(input); @@ -91,17 +96,17 @@ public class TextTranslationService { tcb.onTranslationException(ex); } }); - } - + } + tcb.onNoServiceProviderException(new NoServiceProviderException( "Could not find a TextTranslator service provider")); } /** - * Returns if a TextTranslator lookup successfully found an implementing + * Returns if a TextTranslator lookup successfully found an implementing * class. * - * @return + * @return */ public boolean hasProvider() { return translator.isPresent(); diff --git a/Core/src/org/sleuthkit/autopsy/texttranslation/TranslationCallback.java b/Core/src/org/sleuthkit/autopsy/texttranslation/TranslationCallback.java index 4cbdb79265..c38617cbab 100755 --- a/Core/src/org/sleuthkit/autopsy/texttranslation/TranslationCallback.java +++ b/Core/src/org/sleuthkit/autopsy/texttranslation/TranslationCallback.java @@ -19,24 +19,29 @@ package org.sleuthkit.autopsy.texttranslation; /** - * Provides a way for translation results to be processed when they are - * + * This callback interface will be used by TextTranslationService when doing + * translations on background tasks. When the translation is done, it's result will + * be delegated to one of the following methods. It can either be successful or fail with + * exceptions. */ public interface TranslationCallback { /** + * Provides a method to handle the translation result * - * @param translation + * @param translation result of calling TextTranslationService. */ public void onTranslationResult(String translation); /** + * Provides a way to handle Translation Exceptions. * * @param ex */ public void onTranslationException(TranslationException ex); /** + * Provides a way to handle NoServiceProviderExceptions. * * @param ex */ From 7e97e5e195c17ffde8e4c8daf61c8ccbd10db46d Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Thu, 25 Oct 2018 15:18:38 -0400 Subject: [PATCH 021/129] reverted space is case.java --- Core/src/org/sleuthkit/autopsy/casemodule/Case.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/casemodule/Case.java b/Core/src/org/sleuthkit/autopsy/casemodule/Case.java index b5624f8fa2..dcb2af3eef 100644 --- a/Core/src/org/sleuthkit/autopsy/casemodule/Case.java +++ b/Core/src/org/sleuthkit/autopsy/casemodule/Case.java @@ -1475,7 +1475,7 @@ public class Case { public void notifyAddingDataSource(UUID eventId) { eventPublisher.publish(new AddingDataSourceEvent(eventId)); } - + /** * Notifies case event subscribers that a data source failed to be added to * the case. From f9606deea7706e9f231a612b7b0ef3df07eb2db5 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Thu, 25 Oct 2018 15:32:14 -0400 Subject: [PATCH 022/129] Added log message in --- .../datamodel/AbstractAbstractFileNode.java | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java index 276b67a37b..aef33be3ad 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java @@ -275,11 +275,10 @@ public abstract class AbstractAbstractFileNode extends A if (tts.hasProvider()) { //Seperate out the base and ext from the contents file name. String base = FilenameUtils.getBaseName(this.content.getName()); - String ext = FilenameUtils.getExtension(this.content.getName()); //Send only the base file name to be translated. Once the translation comes //back fire a PropertyChangeEvent on this nodes PropertyChangeListener. - tts.translateAsynchronously(base, new TranslateFileNameCallback(ext, weakPcl)); + tts.translateAsynchronously(base, new TranslateFileNameCallback(this.content, weakPcl)); } //In the mean time, return a blank translation. @@ -518,11 +517,13 @@ public abstract class AbstractAbstractFileNode extends A private class TranslateFileNameCallback implements TranslationCallback { //Seperate out the base and ext from the contents file name. private final String ext; + private final String originalFileName; private final PropertyChangeListener listener; - public TranslateFileNameCallback(String ext, PropertyChangeListener listener) { - this.ext = ext; + public TranslateFileNameCallback(AbstractFile file, PropertyChangeListener listener) { + this.ext = FilenameUtils.getExtension(content.getName()); + this.originalFileName = content.getName(); this.listener = listener; } @@ -554,18 +555,18 @@ public abstract class AbstractAbstractFileNode extends A } /** - * Do nothing on a translation exception, the column will remain empty + * Do nothing on a translation exception except log, the column will remain empty * and there is nothing we can do as a fallback. * * @param ex Exception caught while translating. */ @Override public void onTranslationException(TranslationException ex) { - //Do nothing + logger.log(Level.WARNING, "Could not successfully translate file name " + originalFileName, ex); } /** - * Do nothing on a no service provider exception, in this implemention we + * Do nothing on a no service provider exception except log, in this implemention we * are only calling for translation to be done if we know that a TextTranslator * service provider is already defined. * @@ -573,7 +574,8 @@ public abstract class AbstractAbstractFileNode extends A */ @Override public void onNoServiceProviderException(NoServiceProviderException ex) { - //Do nothing + logger.log(Level.WARNING, "Translate unsuccessful because no TextTranslator " + + "implementation was provided.", ex); } } } From cefbafa63c0572e3e0e0a90e8f2d2a469f1ac6ba Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Thu, 25 Oct 2018 15:39:52 -0400 Subject: [PATCH 023/129] Made the labels all point to the same bundle message --- .../autopsy/datamodel/AbstractAbstractFileNode.java | 2 +- .../autopsy/datamodel/AbstractFsContentNode.java | 3 ++- .../org/sleuthkit/autopsy/datamodel/LayoutFileNode.java | 7 +++++-- .../org/sleuthkit/autopsy/datamodel/LocalFileNode.java | 8 +++----- 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java index aef33be3ad..2516a92419 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java @@ -194,6 +194,7 @@ public abstract class AbstractAbstractFileNode extends A } @NbBundle.Messages({"AbstractAbstractFileNode.nameColLbl=Name", + "AbstractAbstractFileNode.translateFileName=Translated Name", "AbstractAbstractFileNode.locationColLbl=Location", "AbstractAbstractFileNode.modifiedTimeColLbl=Modified Time", "AbstractAbstractFileNode.changeTimeColLbl=Change Time", @@ -515,7 +516,6 @@ public abstract class AbstractAbstractFileNode extends A * can call these methods when an asynchronous translation task is complete. */ private class TranslateFileNameCallback implements TranslationCallback { - //Seperate out the base and ext from the contents file name. private final String ext; private final String originalFileName; diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractFsContentNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractFsContentNode.java index ed0eb09b8d..996977fabc 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractFsContentNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractFsContentNode.java @@ -88,7 +88,8 @@ public abstract class AbstractFsContentNode extends Abst if(UserPreferences.displayTranslationFileNames()) { String translation = getTranslatedFileName(); - sheetSet.put(new NodeProperty<>("Translated Name", "Translated Name", "", translation)); + sheetSet.put(new NodeProperty<>(Bundle.AbstractAbstractFileNode_translateFileName(), + Bundle.AbstractAbstractFileNode_translateFileName(), NO_DESCR, translation)); } addScoreProperty(sheetSet, tags); diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/LayoutFileNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/LayoutFileNode.java index 56c77d0acd..fa3c2f3b56 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/LayoutFileNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/LayoutFileNode.java @@ -98,9 +98,12 @@ public class LayoutFileNode extends AbstractAbstractFileNode { NbBundle.getMessage(this.getClass(), "LayoutFileNode.createSheet.name.desc"), getName())); + final String NO_DESCR = NbBundle.getMessage(this.getClass(), "LayoutFileNode.createSheet.noDescr.text"); + if(UserPreferences.displayTranslationFileNames()) { String translation = getTranslatedFileName(); - sheetSet.put(new NodeProperty<>("Translated Name", "Translated Name", "", translation)); + sheetSet.put(new NodeProperty<>(Bundle.AbstractAbstractFileNode_translateFileName(), + Bundle.AbstractAbstractFileNode_translateFileName(), NO_DESCR, translation)); } addScoreProperty(sheetSet, tags); @@ -114,7 +117,7 @@ public class LayoutFileNode extends AbstractAbstractFileNode { if (EamDbUtil.useCentralRepo() && UserPreferences.hideCentralRepoCommentsAndOccurrences() == false) { addCountProperty(sheetSet, correlationAttribute); } - final String NO_DESCR = NbBundle.getMessage(this.getClass(), "LayoutFileNode.createSheet.noDescr.text"); + for (Map.Entry entry : map.entrySet()) { sheetSet.put(new NodeProperty<>(entry.getKey(), entry.getKey(), NO_DESCR, entry.getValue())); } diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/LocalFileNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/LocalFileNode.java index ddbc0dd4ae..7ca92d5ee5 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/LocalFileNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/LocalFileNode.java @@ -42,9 +42,6 @@ import org.sleuthkit.autopsy.directorytree.ExtractAction; import org.sleuthkit.autopsy.directorytree.NewWindowViewAction; import org.sleuthkit.autopsy.directorytree.ViewContextAction; import org.sleuthkit.autopsy.modules.embeddedfileextractor.ExtractArchiveWithPasswordAction; -import org.sleuthkit.autopsy.texttranslation.NoServiceProviderException; -import org.sleuthkit.autopsy.texttranslation.TextTranslationService; -import org.sleuthkit.autopsy.texttranslation.TranslationException; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.ContentTag; @@ -88,9 +85,11 @@ public class LocalFileNode extends AbstractAbstractFileNode { NbBundle.getMessage(this.getClass(), "LocalFileNode.createSheet.name.desc"), getName())); + final String NO_DESCR = NbBundle.getMessage(this.getClass(), "LocalFileNode.createSheet.noDescr.text"); if(UserPreferences.displayTranslationFileNames()) { String translation = getTranslatedFileName(); - sheetSet.put(new NodeProperty<>("Translated Name", "Translated Name", "", translation)); + sheetSet.put(new NodeProperty<>(Bundle.AbstractAbstractFileNode_translateFileName(), + Bundle.AbstractAbstractFileNode_translateFileName(), NO_DESCR, translation)); } addScoreProperty(sheetSet, tags); @@ -104,7 +103,6 @@ public class LocalFileNode extends AbstractAbstractFileNode { if (EamDbUtil.useCentralRepo() && UserPreferences.hideCentralRepoCommentsAndOccurrences() == false) { addCountProperty(sheetSet, correlationAttribute); } - final String NO_DESCR = NbBundle.getMessage(this.getClass(), "LocalFileNode.createSheet.noDescr.text"); for (Map.Entry entry : map.entrySet()) { sheetSet.put(new NodeProperty<>(entry.getKey(), entry.getKey(), NO_DESCR, entry.getValue())); } From 1f3c2d2f0bf0c03787617efcdce8922761ce2b93 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Thu, 25 Oct 2018 15:40:51 -0400 Subject: [PATCH 024/129] Removed unused imports --- .../sleuthkit/autopsy/datamodel/AbstractFsContentNode.java | 1 - Core/src/org/sleuthkit/autopsy/datamodel/LayoutFileNode.java | 4 ---- 2 files changed, 5 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractFsContentNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractFsContentNode.java index 996977fabc..cd88832e54 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractFsContentNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractFsContentNode.java @@ -27,7 +27,6 @@ import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeIns import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbUtil; import org.sleuthkit.autopsy.core.UserPreferences; import org.sleuthkit.autopsy.coreutils.Logger; -import org.sleuthkit.autopsy.texttranslation.TextTranslationService; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.ContentTag; diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/LayoutFileNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/LayoutFileNode.java index fa3c2f3b56..3703030e8e 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/LayoutFileNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/LayoutFileNode.java @@ -27,7 +27,6 @@ import java.util.List; import java.util.Map; import javax.swing.Action; import org.openide.nodes.Sheet; -import org.openide.util.Exceptions; import org.openide.util.NbBundle; import org.openide.util.Utilities; import org.sleuthkit.autopsy.actions.AddContentTagAction; @@ -39,9 +38,6 @@ import org.sleuthkit.autopsy.coreutils.ContextMenuExtensionPoint; import org.sleuthkit.autopsy.directorytree.ExternalViewerAction; import org.sleuthkit.autopsy.directorytree.ExtractAction; import org.sleuthkit.autopsy.directorytree.NewWindowViewAction; -import org.sleuthkit.autopsy.texttranslation.NoServiceProviderException; -import org.sleuthkit.autopsy.texttranslation.TextTranslationService; -import org.sleuthkit.autopsy.texttranslation.TranslationException; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.ContentTag; import org.sleuthkit.datamodel.LayoutFile; From b4c8abad8f83447a9dfd822a373d8991bf615897 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Thu, 25 Oct 2018 16:02:39 -0400 Subject: [PATCH 025/129] 4305 allow correlation attrs to be added when ingest not running --- .../eventlisteners/IngestEventsListener.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java index ccca659574..cdbf490e9e 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java @@ -183,7 +183,10 @@ public class IngestEventsListener { @Override public void propertyChange(PropertyChangeEvent evt) { - if (getCeModuleInstanceCount() > 0) { + //if ingest is running we want there to check if there is a Correlation Engine module running + //sometimes artifacts are generated by DSPs or other sources while ingest is not running + //in these cases we still want to create correlation attributes for those artifacts when appropriate + if (!IngestManager.getInstance().isIngestRunning() || getCeModuleInstanceCount() > 0) { EamDb dbManager; try { dbManager = EamDb.getInstance(); @@ -197,7 +200,7 @@ public class IngestEventsListener { break; } } - } + } } } From 0edda0d0ae1e4651da4fa36b81cf53d2d5cf1b88 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Fri, 26 Oct 2018 08:47:52 -0400 Subject: [PATCH 026/129] Did codacy suggestions --- .../autopsy/datamodel/AbstractAbstractFileNode.java | 8 ++++---- .../autopsy/datamodel/LocalDirectoryNode.java | 3 --- .../autopsy/datamodel/VirtualDirectoryNode.java | 3 --- .../autopsy/texttranslation/TextTranslator.java | 2 +- .../autopsy/texttranslation/TranslationCallback.java | 10 +++++----- 5 files changed, 10 insertions(+), 16 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java index 2516a92419..1f32cc554b 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java @@ -561,8 +561,8 @@ public abstract class AbstractAbstractFileNode extends A * @param ex Exception caught while translating. */ @Override - public void onTranslationException(TranslationException ex) { - logger.log(Level.WARNING, "Could not successfully translate file name " + originalFileName, ex); + public void onTranslationException(TranslationException noTranslationEx) { + logger.log(Level.WARNING, "Could not successfully translate file name " + originalFileName, noTranslationEx); } /** @@ -573,9 +573,9 @@ public abstract class AbstractAbstractFileNode extends A * @param ex */ @Override - public void onNoServiceProviderException(NoServiceProviderException ex) { + public void onNoServiceProviderException(NoServiceProviderException noServiceEx) { logger.log(Level.WARNING, "Translate unsuccessful because no TextTranslator " - + "implementation was provided.", ex); + + "implementation was provided.", noServiceEx); } } } diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/LocalDirectoryNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/LocalDirectoryNode.java index 8cd255fd5d..1a80fb2737 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/LocalDirectoryNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/LocalDirectoryNode.java @@ -26,9 +26,6 @@ import org.openide.util.NbBundle; import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeInstance; import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbUtil; import org.sleuthkit.autopsy.core.UserPreferences; -import org.sleuthkit.autopsy.texttranslation.NoServiceProviderException; -import org.sleuthkit.autopsy.texttranslation.TextTranslationService; -import org.sleuthkit.autopsy.texttranslation.TranslationException; import org.sleuthkit.datamodel.ContentTag; import org.sleuthkit.datamodel.LocalDirectory; diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/VirtualDirectoryNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/VirtualDirectoryNode.java index 8de2db173a..6c6a4f6f1a 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/VirtualDirectoryNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/VirtualDirectoryNode.java @@ -32,9 +32,6 @@ import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeIns import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbUtil; import org.sleuthkit.autopsy.core.UserPreferences; import org.sleuthkit.autopsy.coreutils.Logger; -import org.sleuthkit.autopsy.texttranslation.NoServiceProviderException; -import org.sleuthkit.autopsy.texttranslation.TextTranslationService; -import org.sleuthkit.autopsy.texttranslation.TranslationException; import org.sleuthkit.datamodel.ContentTag; import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.TskCoreException; diff --git a/Core/src/org/sleuthkit/autopsy/texttranslation/TextTranslator.java b/Core/src/org/sleuthkit/autopsy/texttranslation/TextTranslator.java index 4a813ed6cc..33be6d04ca 100755 --- a/Core/src/org/sleuthkit/autopsy/texttranslation/TextTranslator.java +++ b/Core/src/org/sleuthkit/autopsy/texttranslation/TextTranslator.java @@ -24,6 +24,6 @@ package org.sleuthkit.autopsy.texttranslation; */ public interface TextTranslator { - public String translate(String input) throws TranslationException; + String translate(String input) throws TranslationException; } \ No newline at end of file diff --git a/Core/src/org/sleuthkit/autopsy/texttranslation/TranslationCallback.java b/Core/src/org/sleuthkit/autopsy/texttranslation/TranslationCallback.java index c38617cbab..8144f6fa1f 100755 --- a/Core/src/org/sleuthkit/autopsy/texttranslation/TranslationCallback.java +++ b/Core/src/org/sleuthkit/autopsy/texttranslation/TranslationCallback.java @@ -31,19 +31,19 @@ public interface TranslationCallback { * * @param translation result of calling TextTranslationService. */ - public void onTranslationResult(String translation); + void onTranslationResult(String translation); /** * Provides a way to handle Translation Exceptions. * - * @param ex + * @param noTranslationEx */ - public void onTranslationException(TranslationException ex); + void onTranslationException(TranslationException noTranslationEx); /** * Provides a way to handle NoServiceProviderExceptions. * - * @param ex + * @param noServiceEx */ - public void onNoServiceProviderException(NoServiceProviderException ex); + void onNoServiceProviderException(NoServiceProviderException noServiceEx); } From 5e817a63e5e7df67a57120be59786b77a6cdb224 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Fri, 26 Oct 2018 08:59:26 -0400 Subject: [PATCH 027/129] Made TextTranslationService a singleton --- .../corecomponents/ViewPreferencesPanel.java | 2 +- .../datamodel/AbstractAbstractFileNode.java | 2 +- .../texttranslation/TextTranslationService.java | 15 +++++++++++---- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.java b/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.java index c6bc396fbe..e65302db7f 100755 --- a/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.java +++ b/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.java @@ -48,7 +48,7 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { //If there is not Text Translator implementation, then hide these buttons //from the user. - TextTranslationService tts = new TextTranslationService(); + TextTranslationService tts = TextTranslationService.getInstance(); if(!tts.hasProvider()) { translatedNamesButton.setVisible(false); fileDisplayLabel.setVisible(false); diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java index 1f32cc554b..a63f441a2b 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java @@ -272,7 +272,7 @@ public abstract class AbstractAbstractFileNode extends A //If not, lets fire off a background translation that will update the UI //when it is done. - TextTranslationService tts = new TextTranslationService(); + TextTranslationService tts = TextTranslationService.getInstance(); if (tts.hasProvider()) { //Seperate out the base and ext from the contents file name. String base = FilenameUtils.getBaseName(this.content.getName()); diff --git a/Core/src/org/sleuthkit/autopsy/texttranslation/TextTranslationService.java b/Core/src/org/sleuthkit/autopsy/texttranslation/TextTranslationService.java index bac1ee9fb3..fb33425a8f 100755 --- a/Core/src/org/sleuthkit/autopsy/texttranslation/TextTranslationService.java +++ b/Core/src/org/sleuthkit/autopsy/texttranslation/TextTranslationService.java @@ -29,17 +29,24 @@ import org.openide.util.Lookup; * Service for finding and running TextTranslator implementations */ public class TextTranslationService { + + private final static TextTranslationService tts = new TextTranslationService(); - private final static Optional translator; + private final Optional translator; private static ExecutorService pool; private final static Integer MAX_POOL_SIZE = 10; - - static { - //Perform look up for Text Translation implementations + + private TextTranslationService(){ + //Perform look up for Text Translation implementations ONLY ONCE during + //class loading. translator = Optional.ofNullable(Lookup.getDefault() .lookup(TextTranslator.class)); } + + public static TextTranslationService getInstance() { + return tts; + } /** * Translates the input string using whichever TextTranslator Service Provider From af192b2c1d54ec3e44ff6586f734d1f6f176ebab Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Fri, 26 Oct 2018 09:13:48 -0400 Subject: [PATCH 028/129] Changed panel text according to Brian's request --- .../org/sleuthkit/autopsy/corecomponents/Bundle.properties | 4 ++-- .../autopsy/corecomponents/ViewPreferencesPanel.form | 4 ++-- .../autopsy/corecomponents/ViewPreferencesPanel.java | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/corecomponents/Bundle.properties b/Core/src/org/sleuthkit/autopsy/corecomponents/Bundle.properties index efd861fb24..3e8e75dad7 100644 --- a/Core/src/org/sleuthkit/autopsy/corecomponents/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/corecomponents/Bundle.properties @@ -192,5 +192,5 @@ ViewPreferencesPanel.centralRepoLabel.text=Do not use Central Repository for: ViewPreferencesPanel.commentsOccurencesColumnsCheckbox.text=C(omments) and O(ccurences) columns to reduce loading times ViewPreferencesPanel.deletedFilesLimitCheckbox.text=Limit to 10,000 ViewPreferencesPanel.deletedFilesLimitLabel.text=Limit number of deleted files displayed: -ViewPreferencesPanel.fileDisplayLabel.text=When displaying files: -ViewPreferencesPanel.translatedNamesButton.text=Show translated file names +ViewPreferencesPanel.fileDisplayLabel.text=Translate text in the: +ViewPreferencesPanel.translatedNamesButton.text=Table diff --git a/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.form b/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.form index 096c24af11..1f76dde4dd 100755 --- a/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.form +++ b/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.form @@ -176,9 +176,9 @@ - - + + diff --git a/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.java b/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.java index e65302db7f..af9a940537 100755 --- a/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.java +++ b/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.java @@ -346,9 +346,9 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) .addComponent(useGMTTimeRadioButton))) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) - .addGroup(globalSettingsPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) - .addComponent(hideOtherUsersTagsLabel) - .addComponent(fileDisplayLabel)) + .addGroup(globalSettingsPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addComponent(fileDisplayLabel) + .addComponent(hideOtherUsersTagsLabel)) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) .addGroup(globalSettingsPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) .addComponent(hideOtherUsersTagsCheckbox) From 95b981c99115fa7bd727a79ae265118df26facb1 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Mon, 29 Oct 2018 09:01:51 -0400 Subject: [PATCH 029/129] Final codacy issues --- .../org/sleuthkit/autopsy/datamodel/LocalDirectoryNode.java | 5 +++-- .../autopsy/texttranslation/TextTranslationService.java | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/LocalDirectoryNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/LocalDirectoryNode.java index 1a80fb2737..bd59e8ca67 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/LocalDirectoryNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/LocalDirectoryNode.java @@ -66,9 +66,11 @@ public class LocalDirectoryNode extends SpecialDirectoryNode { Bundle.LocalDirectoryNode_createSheet_name_desc(), getName())); + final String NO_DESCR = Bundle.LocalDirectoryNode_createSheet_noDesc(); if(UserPreferences.displayTranslationFileNames()) { String translation = getTranslatedFileName(); - sheetSet.put(new NodeProperty<>("Translated Name", "Translated Name", "", translation)); + sheetSet.put(new NodeProperty<>(Bundle.AbstractAbstractFileNode_translateFileName(), + Bundle.AbstractAbstractFileNode_translateFileName(), NO_DESCR, translation)); } addScoreProperty(sheetSet, tags); @@ -87,7 +89,6 @@ public class LocalDirectoryNode extends SpecialDirectoryNode { Map map = new LinkedHashMap<>(); fillPropertyMap(map, getContent()); - final String NO_DESCR = Bundle.LocalDirectoryNode_createSheet_noDesc(); for (Map.Entry entry : map.entrySet()) { sheetSet.put(new NodeProperty<>(entry.getKey(), entry.getKey(), NO_DESCR, entry.getValue())); } diff --git a/Core/src/org/sleuthkit/autopsy/texttranslation/TextTranslationService.java b/Core/src/org/sleuthkit/autopsy/texttranslation/TextTranslationService.java index fb33425a8f..04e4e07835 100755 --- a/Core/src/org/sleuthkit/autopsy/texttranslation/TextTranslationService.java +++ b/Core/src/org/sleuthkit/autopsy/texttranslation/TextTranslationService.java @@ -28,7 +28,7 @@ import org.openide.util.Lookup; /** * Service for finding and running TextTranslator implementations */ -public class TextTranslationService { +public final class TextTranslationService { private final static TextTranslationService tts = new TextTranslationService(); From 5ec17adf8e0943176f5f26766496d89d4f3b4cfb Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Mon, 29 Oct 2018 12:30:46 -0400 Subject: [PATCH 030/129] 4305 add icon for tsk_wifi_network artifacts --- .../autopsy/datamodel/ExtractedContent.java | 2 ++ .../org/sleuthkit/autopsy/images/network-wifi.png | Bin 0 -> 453 bytes .../org/sleuthkit/autopsy/report/ReportHTML.java | 2 ++ 3 files changed, 4 insertions(+) create mode 100644 Core/src/org/sleuthkit/autopsy/images/network-wifi.png diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/ExtractedContent.java b/Core/src/org/sleuthkit/autopsy/datamodel/ExtractedContent.java index 3ab5f6cb90..e950d0e870 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/ExtractedContent.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/ExtractedContent.java @@ -156,6 +156,8 @@ public class ExtractedContent implements AutopsyVisitableItem { return filePath + "drive_network.png"; //NON-NLS } else if (typeID == BlackboardArtifact.ARTIFACT_TYPE.TSK_FACE_DETECTED.getTypeID()) { return filePath + "face.png"; //NON-NLS + } else if (typeID == BlackboardArtifact.ARTIFACT_TYPE.TSK_WIFI_NETWORK.getTypeID()) { + return filePath + "network-wifi.png"; //NON-NLS } return filePath + "artifact-icon.png"; //NON-NLS } diff --git a/Core/src/org/sleuthkit/autopsy/images/network-wifi.png b/Core/src/org/sleuthkit/autopsy/images/network-wifi.png new file mode 100644 index 0000000000000000000000000000000000000000..5d7cb76ca165db10f3df4f8b07813eee8b335a7c GIT binary patch literal 453 zcmV;$0XqJPP)xN5fD!z_WZTrMgZ%0MZJdrZ`nVY0$CtoPti%e>BfZr& v_5MM;>Ij(+V Date: Mon, 29 Oct 2018 12:40:23 -0400 Subject: [PATCH 031/129] 4305 remove partial code for adding imei and mac address correlation attrs --- .../datamodel/CorrelationAttributeInstance.java | 8 +------- .../datamodel/CorrelationAttributeNormalizer.java | 2 -- .../centralrepository/datamodel/EamArtifactUtil.java | 7 ------- 3 files changed, 1 insertion(+), 16 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeInstance.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeInstance.java index eb790f866b..20845ff447 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeInstance.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeInstance.java @@ -247,8 +247,6 @@ public class CorrelationAttributeInstance implements Serializable { public static final int PHONE_TYPE_ID = 3; public static final int USBID_TYPE_ID = 4; public static final int SSID_TYPE_ID = 5; - public static final int MAC_TYPE_ID = 6; - public static final int IMEI_TYPE_ID = 7; /** * Load the default correlation types @@ -261,9 +259,7 @@ public class CorrelationAttributeInstance implements Serializable { "CorrelationType.EMAIL.displayName=Email Addresses", "CorrelationType.PHONE.displayName=Phone Numbers", "CorrelationType.USBID.displayName=USB Devices", - "CorrelationType.SSID.displayName=Wireless Networks", - "CorrelationType.MAC.displayName=MAC Address", - "CorrelationType.IMEI.displayName=IMEI Number"}) + "CorrelationType.SSID.displayName=Wireless Networks"}) public static List getDefaultCorrelationTypes() throws EamDbException { List DEFAULT_CORRELATION_TYPES = new ArrayList<>(); DEFAULT_CORRELATION_TYPES.add(new CorrelationAttributeInstance.Type(FILES_TYPE_ID, Bundle.CorrelationType_FILES_displayName(), "file", true, true)); // NON-NLS @@ -272,8 +268,6 @@ public class CorrelationAttributeInstance implements Serializable { DEFAULT_CORRELATION_TYPES.add(new CorrelationAttributeInstance.Type(PHONE_TYPE_ID, Bundle.CorrelationType_PHONE_displayName(), "phone_number", true, true)); // NON-NLS DEFAULT_CORRELATION_TYPES.add(new CorrelationAttributeInstance.Type(USBID_TYPE_ID, Bundle.CorrelationType_USBID_displayName(), "usb_devices", true, true)); // NON-NLS DEFAULT_CORRELATION_TYPES.add(new CorrelationAttributeInstance.Type(SSID_TYPE_ID, Bundle.CorrelationType_SSID_displayName(), "wireless_networks", true, true)); // NON-NLS - DEFAULT_CORRELATION_TYPES.add(new CorrelationAttributeInstance.Type(MAC_TYPE_ID, Bundle.CorrelationType_MAC_displayName(), "mac_address", true, true)); // NON-NLS - DEFAULT_CORRELATION_TYPES.add(new CorrelationAttributeInstance.Type(IMEI_TYPE_ID, Bundle.CorrelationType_IMEI_displayName(), "imei_number", true, true)); // NON-NLS return DEFAULT_CORRELATION_TYPES; } diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeNormalizer.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeNormalizer.java index c7787389f4..4ce04769c8 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeNormalizer.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeNormalizer.java @@ -64,8 +64,6 @@ final public class CorrelationAttributeNormalizer { case CorrelationAttributeInstance.USBID_TYPE_ID: return normalizeUsbId(data); case CorrelationAttributeInstance.SSID_TYPE_ID: - case CorrelationAttributeInstance.MAC_TYPE_ID: - case CorrelationAttributeInstance.IMEI_TYPE_ID: return data; default: final String errorMessage = String.format( diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactUtil.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactUtil.java index c3b18e62f5..2477d0fc41 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactUtil.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactUtil.java @@ -159,7 +159,6 @@ public class EamArtifactUtil { return null; } } - } else if (correlationType.getId() == CorrelationAttributeInstance.USBID_TYPE_ID && BlackboardArtifact.ARTIFACT_TYPE.TSK_DEVICE_ATTACHED.getTypeID() == artifactTypeID) { @@ -169,12 +168,6 @@ public class EamArtifactUtil { && BlackboardArtifact.ARTIFACT_TYPE.TSK_WIFI_NETWORK.getTypeID() == artifactTypeID) { value = bbArtifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SSID)).getValueString(); } -// else if (correlationType.getId() == CorrelationAttributeInstance.MAC_TYPE_ID) { -// value = bbArtifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_BSSID)).getValueString(); -// } else if (correlationType.getId() == CorrelationAttributeInstance.IMEI_TYPE_ID) { -// value = bbArtifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_BSSID)).getValueString(); -// } - } catch (TskCoreException ex) { logger.log(Level.SEVERE, "Error getting attribute while getting type from BlackboardArtifact.", ex); // NON-NLS return null; From 6ae37630a432f4bb31c8f122294182053f1e3418 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Mon, 29 Oct 2018 15:41:59 -0400 Subject: [PATCH 032/129] Fixed codacy suggestions --- .../embeddedfileextractor/SevenZipExtractor.java | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java index c8f0fce062..dc4f190c09 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java +++ b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java @@ -800,12 +800,7 @@ class SevenZipExtractor { MutableEncodedFileOutputStream(String localAbsPath) throws IOException { super(new FileOutputStream(localAbsPath), TskData.EncodingType.XOR1); } - - @Override - public void write(byte[] bytes) throws IOException { - super.write(bytes); - } - + /** * Update the OutputStream to point to a new File. This method mutates the state so * that there is no overhead of creating a new object and buffer. Additionally, the @@ -837,6 +832,14 @@ class SevenZipExtractor { } } + /** + * UnpackStream used by the SevenZipBindings to do archive extraction. A memory + * leak exists in the SevenZip library that will not let go of the Streams until + * the entire archive extraction is complete. So, to compensate this UnpackStream + * uses a MutableEncodedFileOutputStream, which supports setting a new disk + * location for the contents to be written to (as opposed to creating a new steam + * for each file). + */ private final static class UnpackStream implements ISequentialOutStream { private final MutableEncodedFileOutputStream output; From 82aa5a063923218902f732d1816c19b376e80075 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Mon, 29 Oct 2018 16:00:46 -0400 Subject: [PATCH 033/129] 4305 CR upgrade code for wireless_networks correlation attr --- .../datamodel/AbstractSqlEamDb.java | 47 +++++++++++ .../centralrepository/datamodel/EamDb.java | 4 +- .../datamodel/PostgresEamDbSettings.java | 78 +++++++++++------ .../datamodel/SqliteEamDbSettings.java | 83 ++++++++++++------- 4 files changed, 155 insertions(+), 57 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java index 68e5b4f8c2..b14d9043af 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java @@ -3085,6 +3085,7 @@ abstract class AbstractSqlEamDb implements EamDb { ResultSet resultSet = null; Statement statement = null; + PreparedStatement preparedStatement = null; Connection conn = null; try { @@ -3131,7 +3132,52 @@ abstract class AbstractSqlEamDb implements EamDb { // regardless of whether this succeeds. EamDbUtil.insertDefaultOrganization(conn); } + if (dbSchemaVersion.compareTo(new CaseDbSchemaVersionNumber(1, 2)) < 0) { + //update central repository to be able to store new correlation attributes + EamDbPlatformEnum selectedPlatform = EamDbPlatformEnum.getSelectedPlatform(); + final String addSsidTableTemplate; + final String addCaseIdIndexTemplate; + final String addDataSourceIdIndexTemplate; + final String addValueIndexTemplate; + final String addKnownStatusIndexTemplate; + switch (selectedPlatform) { + case POSTGRESQL: + //create table //settings.getStatementTemplate then statement.execute here with the added table + //add correlation attr to correlation_attrs table + addSsidTableTemplate = PostgresEamDbSettings.getCreateArtifactInstancesTableTemplate(); + addCaseIdIndexTemplate = PostgresEamDbSettings.getAddCaseIdIndexTemplate(); + addDataSourceIdIndexTemplate = PostgresEamDbSettings.getAddDataSourceIdIndexTemplate(); + addValueIndexTemplate = PostgresEamDbSettings.getAddValueIndexTemplate(); + addKnownStatusIndexTemplate = PostgresEamDbSettings.getAddKnownStatusIndexTemplate(); + break; + case SQLITE: + addSsidTableTemplate = SqliteEamDbSettings.getCreateArtifactInstancesTableTemplate(); + addCaseIdIndexTemplate = SqliteEamDbSettings.getAddCaseIdIndexTemplate(); + addDataSourceIdIndexTemplate = SqliteEamDbSettings.getAddDataSourceIdIndexTemplate(); + addValueIndexTemplate = SqliteEamDbSettings.getAddValueIndexTemplate(); + addKnownStatusIndexTemplate = SqliteEamDbSettings.getAddKnownStatusIndexTemplate(); + break; + default: + throw new EamDbException("Currently selected database platform \"" + selectedPlatform.name() + "\" can not be upgraded."); + } + final String wirelessNetworsDbTableName = "wireless_networks"; + final String wirelessNetworksTableInstanceName = wirelessNetworsDbTableName + "_instances"; + final String addAttributeSql = "INSERT INTO correlation_types(id, display_name, db_table_name, supported, enabled) VALUES (?, ?, ?, ?, ?)"; + preparedStatement = conn.prepareStatement(addAttributeSql); + preparedStatement.setInt(1, CorrelationAttributeInstance.SSID_TYPE_ID); + preparedStatement.setString(2, Bundle.CorrelationType_SSID_displayName()); + preparedStatement.setString(3, wirelessNetworsDbTableName); + preparedStatement.setInt(4, 1); + preparedStatement.setInt(5, 1); + preparedStatement.execute(); + statement.execute(String.format(addSsidTableTemplate, wirelessNetworksTableInstanceName, wirelessNetworksTableInstanceName)); + statement.execute(String.format(addCaseIdIndexTemplate, wirelessNetworksTableInstanceName, wirelessNetworksTableInstanceName)); + statement.execute(String.format(addDataSourceIdIndexTemplate, wirelessNetworksTableInstanceName, wirelessNetworksTableInstanceName)); + statement.execute(String.format(addValueIndexTemplate, wirelessNetworksTableInstanceName, wirelessNetworksTableInstanceName)); + statement.execute(String.format(addKnownStatusIndexTemplate, wirelessNetworksTableInstanceName, wirelessNetworksTableInstanceName)); + + } if (!updateSchemaVersion(conn)) { throw new EamDbException("Error updating schema version"); } @@ -3149,6 +3195,7 @@ abstract class AbstractSqlEamDb implements EamDb { throw ex; } finally { EamDbUtil.closeResultSet(resultSet); + EamDbUtil.closeStatement(preparedStatement); EamDbUtil.closeStatement(statement); EamDbUtil.closeConnection(conn); } diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDb.java index c7e385928d..418181d8ab 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDb.java @@ -31,9 +31,9 @@ import org.sleuthkit.datamodel.CaseDbSchemaVersionNumber; */ public interface EamDb { - public static final int SCHEMA_VERSION = 1; + public static final int SCHEMA_VERSION = 2; public static final CaseDbSchemaVersionNumber CURRENT_DB_SCHEMA_VERSION - = new CaseDbSchemaVersionNumber(1, 1); + = new CaseDbSchemaVersionNumber(1, 2); /** diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDbSettings.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDbSettings.java index 77ab8c23db..adaf7c0d9c 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDbSettings.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDbSettings.java @@ -35,8 +35,9 @@ import org.sleuthkit.autopsy.coreutils.TextConverterException; /** * Settings for the Postgres implementation of the Central Repository database - * - * NOTE: This is public scope because the options panel calls it directly to set/get + * + * NOTE: This is public scope because the options panel calls it directly to + * set/get */ public final class PostgresEamDbSettings { @@ -266,7 +267,7 @@ public final class PostgresEamDbSettings { return true; } - + public boolean deleteDatabase() { Connection conn = getEphemeralConnection(true); if (null == conn) { @@ -391,26 +392,13 @@ public final class PostgresEamDbSettings { createCorrelationTypesTable.append("CONSTRAINT correlation_types_names UNIQUE (display_name, db_table_name)"); createCorrelationTypesTable.append(")"); - // Each "%s" will be replaced with the relevant TYPE_instances table name. - StringBuilder createArtifactInstancesTableTemplate = new StringBuilder(); - createArtifactInstancesTableTemplate.append("CREATE TABLE IF NOT EXISTS %s ("); - createArtifactInstancesTableTemplate.append("id SERIAL PRIMARY KEY,"); - createArtifactInstancesTableTemplate.append("case_id integer NOT NULL,"); - createArtifactInstancesTableTemplate.append("data_source_id integer NOT NULL,"); - createArtifactInstancesTableTemplate.append("value text NOT NULL,"); - createArtifactInstancesTableTemplate.append("file_path text NOT NULL,"); - createArtifactInstancesTableTemplate.append("known_status integer NOT NULL,"); - createArtifactInstancesTableTemplate.append("comment text,"); - createArtifactInstancesTableTemplate.append("CONSTRAINT %s_multi_unique_ UNIQUE (data_source_id, value, file_path),"); - createArtifactInstancesTableTemplate.append("foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL,"); - createArtifactInstancesTableTemplate.append("foreign key (data_source_id) references data_sources(id) ON UPDATE SET NULL ON DELETE SET NULL"); - createArtifactInstancesTableTemplate.append(")"); + String createArtifactInstancesTableTemplate = getCreateArtifactInstancesTableTemplate(); - // Each "%s" will be replaced with the relevant TYPE_instances table name. - String instancesIdx1 = "CREATE INDEX IF NOT EXISTS %s_case_id ON %s (case_id)"; - String instancesIdx2 = "CREATE INDEX IF NOT EXISTS %s_data_source_id ON %s (data_source_id)"; - String instancesIdx3 = "CREATE INDEX IF NOT EXISTS %s_value ON %s (value)"; - String instancesIdx4 = "CREATE INDEX IF NOT EXISTS %s_value_known_status ON %s (value, known_status)"; + String instancesIdx1 = getAddCaseIdIndexTemplate(); + String instancesIdx2 = getAddDataSourceIdIndexTemplate(); + + String instancesIdx3 = getAddValueIndexTemplate(); + String instancesIdx4 = getAddKnownStatusIndexTemplate(); StringBuilder createDbInfoTable = new StringBuilder(); createDbInfoTable.append("CREATE TABLE IF NOT EXISTS db_info ("); @@ -447,14 +435,14 @@ public final class PostgresEamDbSettings { // Create a separate instance and reference table for each correlation type List DEFAULT_CORRELATION_TYPES = CorrelationAttributeInstance.getDefaultCorrelationTypes(); - + String reference_type_dbname; String instance_type_dbname; for (CorrelationAttributeInstance.Type type : DEFAULT_CORRELATION_TYPES) { reference_type_dbname = EamDbUtil.correlationTypeToReferenceTableName(type); instance_type_dbname = EamDbUtil.correlationTypeToInstanceTableName(type); - - stmt.execute(String.format(createArtifactInstancesTableTemplate.toString(), instance_type_dbname, instance_type_dbname)); + + stmt.execute(String.format(createArtifactInstancesTableTemplate, instance_type_dbname, instance_type_dbname)); stmt.execute(String.format(instancesIdx1, instance_type_dbname, instance_type_dbname)); stmt.execute(String.format(instancesIdx2, instance_type_dbname, instance_type_dbname)); stmt.execute(String.format(instancesIdx3, instance_type_dbname, instance_type_dbname)); @@ -465,7 +453,7 @@ public final class PostgresEamDbSettings { stmt.execute(String.format(createReferenceTypesTableTemplate.toString(), reference_type_dbname, reference_type_dbname)); stmt.execute(String.format(referenceTypesIdx1, reference_type_dbname, reference_type_dbname)); stmt.execute(String.format(referenceTypesIdx2, reference_type_dbname, reference_type_dbname)); - } + } } } catch (SQLException ex) { @@ -480,6 +468,44 @@ public final class PostgresEamDbSettings { return true; } + static String getCreateArtifactInstancesTableTemplate() { + // Each "%s" will be replaced with the relevant TYPE_instances table name. + StringBuilder createArtifactInstancesTableTemplate = new StringBuilder(); + createArtifactInstancesTableTemplate.append("CREATE TABLE IF NOT EXISTS %s ("); + createArtifactInstancesTableTemplate.append("id SERIAL PRIMARY KEY,"); + createArtifactInstancesTableTemplate.append("case_id integer NOT NULL,"); + createArtifactInstancesTableTemplate.append("data_source_id integer NOT NULL,"); + createArtifactInstancesTableTemplate.append("value text NOT NULL,"); + createArtifactInstancesTableTemplate.append("file_path text NOT NULL,"); + createArtifactInstancesTableTemplate.append("known_status integer NOT NULL,"); + createArtifactInstancesTableTemplate.append("comment text,"); + createArtifactInstancesTableTemplate.append("CONSTRAINT %s_multi_unique_ UNIQUE (data_source_id, value, file_path),"); + createArtifactInstancesTableTemplate.append("foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL,"); + createArtifactInstancesTableTemplate.append("foreign key (data_source_id) references data_sources(id) ON UPDATE SET NULL ON DELETE SET NULL"); + createArtifactInstancesTableTemplate.append(")"); + return createArtifactInstancesTableTemplate.toString(); + } + + static String getAddCaseIdIndexTemplate() { + // Each "%s" will be replaced with the relevant TYPE_instances table name. + return "CREATE INDEX IF NOT EXISTS %s_case_id ON %s (case_id)"; + } + + static String getAddDataSourceIdIndexTemplate() { + // Each "%s" will be replaced with the relevant TYPE_instances table name. + return "CREATE INDEX IF NOT EXISTS %s_data_source_id ON %s (data_source_id)"; + } + + static String getAddValueIndexTemplate() { + // Each "%s" will be replaced with the relevant TYPE_instances table name. + return "CREATE INDEX IF NOT EXISTS %s_value ON %s (value)"; + } + + static String getAddKnownStatusIndexTemplate() { + // Each "%s" will be replaced with the relevant TYPE_instances table name. + return "CREATE INDEX IF NOT EXISTS %s_value_known_status ON %s (value, known_status)"; + } + public boolean insertDefaultDatabaseContent() { Connection conn = getEphemeralConnection(false); if (null == conn) { diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDbSettings.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDbSettings.java index 4894a570e6..c321e3b412 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDbSettings.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDbSettings.java @@ -35,8 +35,9 @@ import org.sleuthkit.autopsy.coreutils.PlatformUtil; /** * Settings for the sqlite implementation of the Central Repository database - * - * NOTE: This is public scope because the options panel calls it directly to set/get + * + * NOTE: This is public scope because the options panel calls it directly to + * set/get */ public final class SqliteEamDbSettings { @@ -95,7 +96,7 @@ public final class SqliteEamDbSettings { ModuleSettings.setConfigSetting("CentralRepository", "db.sqlite.dbDirectory", getDbDirectory()); // NON-NLS ModuleSettings.setConfigSetting("CentralRepository", "db.sqlite.bulkThreshold", Integer.toString(getBulkThreshold())); // NON-NLS } - + /** * Verify that the db file exists. * @@ -103,11 +104,11 @@ public final class SqliteEamDbSettings { */ public boolean dbFileExists() { File dbFile = new File(getFileNameWithPath()); - if(! dbFile.exists()){ + if (!dbFile.exists()) { return false; } // It's unlikely, but make sure the file isn't actually a directory - return ( ! dbFile.isDirectory()); + return (!dbFile.isDirectory()); } /** @@ -148,10 +149,11 @@ public final class SqliteEamDbSettings { return true; } - + /** * Delete the database - * @return + * + * @return */ public boolean deleteDatabase() { File dbFile = new File(this.getFileNameWithPath()); @@ -333,26 +335,13 @@ public final class SqliteEamDbSettings { createCorrelationTypesTable.append("CONSTRAINT correlation_types_names UNIQUE (display_name, db_table_name)"); createCorrelationTypesTable.append(")"); - // Each "%s" will be replaced with the relevant TYPE_instances table name. - StringBuilder createArtifactInstancesTableTemplate = new StringBuilder(); - createArtifactInstancesTableTemplate.append("CREATE TABLE IF NOT EXISTS %s ("); - createArtifactInstancesTableTemplate.append("id integer primary key autoincrement NOT NULL,"); - createArtifactInstancesTableTemplate.append("case_id integer NOT NULL,"); - createArtifactInstancesTableTemplate.append("data_source_id integer NOT NULL,"); - createArtifactInstancesTableTemplate.append("value text NOT NULL,"); - createArtifactInstancesTableTemplate.append("file_path text NOT NULL,"); - createArtifactInstancesTableTemplate.append("known_status integer NOT NULL,"); - createArtifactInstancesTableTemplate.append("comment text,"); - createArtifactInstancesTableTemplate.append("CONSTRAINT %s_multi_unique UNIQUE(data_source_id, value, file_path) ON CONFLICT IGNORE,"); - createArtifactInstancesTableTemplate.append("foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL,"); - createArtifactInstancesTableTemplate.append("foreign key (data_source_id) references data_sources(id) ON UPDATE SET NULL ON DELETE SET NULL"); - createArtifactInstancesTableTemplate.append(")"); + String createArtifactInstancesTableTemplate = getCreateArtifactInstancesTableTemplate(); - // Each "%s" will be replaced with the relevant TYPE_instances table name. - String instancesIdx1 = "CREATE INDEX IF NOT EXISTS %s_case_id ON %s (case_id)"; - String instancesIdx2 = "CREATE INDEX IF NOT EXISTS %s_data_source_id ON %s (data_source_id)"; - String instancesIdx3 = "CREATE INDEX IF NOT EXISTS %s_value ON %s (value)"; - String instancesIdx4 = "CREATE INDEX IF NOT EXISTS %s_value_known_status ON %s (value, known_status)"; + String instancesIdx1 = getAddCaseIdIndexTemplate(); + String instancesIdx2 = getAddDataSourceIdIndexTemplate(); + + String instancesIdx3 = getAddValueIndexTemplate(); + String instancesIdx4 = getAddKnownStatusIndexTemplate(); StringBuilder createDbInfoTable = new StringBuilder(); createDbInfoTable.append("CREATE TABLE IF NOT EXISTS db_info ("); @@ -402,7 +391,7 @@ public final class SqliteEamDbSettings { reference_type_dbname = EamDbUtil.correlationTypeToReferenceTableName(type); instance_type_dbname = EamDbUtil.correlationTypeToInstanceTableName(type); - stmt.execute(String.format(createArtifactInstancesTableTemplate.toString(), instance_type_dbname, instance_type_dbname)); + stmt.execute(String.format(createArtifactInstancesTableTemplate, instance_type_dbname, instance_type_dbname)); stmt.execute(String.format(instancesIdx1, instance_type_dbname, instance_type_dbname)); stmt.execute(String.format(instancesIdx2, instance_type_dbname, instance_type_dbname)); stmt.execute(String.format(instancesIdx3, instance_type_dbname, instance_type_dbname)); @@ -427,6 +416,44 @@ public final class SqliteEamDbSettings { return true; } + static String getCreateArtifactInstancesTableTemplate() { + // Each "%s" will be replaced with the relevant TYPE_instances table name. + StringBuilder createArtifactInstancesTableTemplate = new StringBuilder(); + createArtifactInstancesTableTemplate.append("CREATE TABLE IF NOT EXISTS %s ("); + createArtifactInstancesTableTemplate.append("id integer primary key autoincrement NOT NULL,"); + createArtifactInstancesTableTemplate.append("case_id integer NOT NULL,"); + createArtifactInstancesTableTemplate.append("data_source_id integer NOT NULL,"); + createArtifactInstancesTableTemplate.append("value text NOT NULL,"); + createArtifactInstancesTableTemplate.append("file_path text NOT NULL,"); + createArtifactInstancesTableTemplate.append("known_status integer NOT NULL,"); + createArtifactInstancesTableTemplate.append("comment text,"); + createArtifactInstancesTableTemplate.append("CONSTRAINT %s_multi_unique UNIQUE(data_source_id, value, file_path) ON CONFLICT IGNORE,"); + createArtifactInstancesTableTemplate.append("foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL,"); + createArtifactInstancesTableTemplate.append("foreign key (data_source_id) references data_sources(id) ON UPDATE SET NULL ON DELETE SET NULL"); + createArtifactInstancesTableTemplate.append(")"); + return createArtifactInstancesTableTemplate.toString(); + } + + static String getAddCaseIdIndexTemplate() { + // Each "%s" will be replaced with the relevant TYPE_instances table name. + return "CREATE INDEX IF NOT EXISTS %s_case_id ON %s (case_id)"; + } + + static String getAddDataSourceIdIndexTemplate() { + // Each "%s" will be replaced with the relevant TYPE_instances table name. + return "CREATE INDEX IF NOT EXISTS %s_data_source_id ON %s (data_source_id)"; + } + + static String getAddValueIndexTemplate() { + // Each "%s" will be replaced with the relevant TYPE_instances table name. + return "CREATE INDEX IF NOT EXISTS %s_value ON %s (value)"; + } + + static String getAddKnownStatusIndexTemplate() { + // Each "%s" will be replaced with the relevant TYPE_instances table name. + return "CREATE INDEX IF NOT EXISTS %s_value_known_status ON %s (value, known_status)"; + } + public boolean insertDefaultDatabaseContent() { Connection conn = getEphemeralConnection(); if (null == conn) { @@ -490,8 +517,6 @@ public final class SqliteEamDbSettings { } } - - /** * @return the dbDirectory */ From e4142387929817e2ad47810ea2e047a1ca333045 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Tue, 30 Oct 2018 08:39:03 -0400 Subject: [PATCH 034/129] This import keeps getting automatically added and its causing Codacy to complain --- .../autopsy/modules/embeddedfileextractor/SevenZipExtractor.java | 1 - 1 file changed, 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java index dc4f190c09..0deea2baf0 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java +++ b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java @@ -43,7 +43,6 @@ import net.sf.sevenzipjbinding.IArchiveExtractCallback; import net.sf.sevenzipjbinding.ICryptoGetTextPassword; import net.sf.sevenzipjbinding.PropID; import org.netbeans.api.progress.ProgressHandle; -import org.openide.util.Exceptions; import org.openide.util.NbBundle; import org.openide.util.NbBundle.Messages; import org.sleuthkit.autopsy.casemodule.Case; From b5c5c64970b74bf3459517226fa3c9430c3aae5f Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Tue, 30 Oct 2018 11:22:23 -0400 Subject: [PATCH 035/129] 4305 correct logic for enabling other occurances content viewer --- .../DataContentViewerOtherCases.java | 56 +++++++++---------- 1 file changed, 25 insertions(+), 31 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/contentviewer/DataContentViewerOtherCases.java b/Core/src/org/sleuthkit/autopsy/centralrepository/contentviewer/DataContentViewerOtherCases.java index c6301bfb89..0080ade4fb 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/contentviewer/DataContentViewerOtherCases.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/contentviewer/DataContentViewerOtherCases.java @@ -127,7 +127,7 @@ public class DataContentViewerOtherCases extends JPanel implements DataContentVi } } else if (jmi.equals(showCommonalityMenuItem)) { showCommonalityDetails(); - } + } } }; @@ -419,7 +419,7 @@ public class DataContentViewerOtherCases extends JPanel implements DataContentVi } // we can correlate based on the MD5 if it is enabled - if (this.file != null && EamDb.isEnabled()) { + if (this.file != null && EamDb.isEnabled() && this.file.getSize() > 0) { try { List artifactTypes = EamDb.getInstance().getDefinedCorrelationTypes(); @@ -447,27 +447,23 @@ public class DataContentViewerOtherCases extends JPanel implements DataContentVi } catch (EamDbException | TskCoreException ex) { LOGGER.log(Level.SEVERE, "Error connecting to DB", ex); // NON-NLS } + // If EamDb not enabled, get the Files default correlation type to allow Other Occurances to be enabled. + } else if (this.file != null && this.file.getSize() > 0) { + String md5 = this.file.getMd5Hash(); + if (md5 != null && !md5.isEmpty()) { + try { + final CorrelationAttributeInstance.Type fileAttributeType + = CorrelationAttributeInstance.getDefaultCorrelationTypes() + .stream() + .filter(attrType -> attrType.getId() == CorrelationAttributeInstance.FILES_TYPE_ID) + .findAny() + .get(); - } else { - - // If EamDb not enabled, get the Files default correlation type to allow Other Occurances to be enabled. - if (this.file != null) { - String md5 = this.file.getMd5Hash(); - if (md5 != null && !md5.isEmpty()) { - try { - final CorrelationAttributeInstance.Type fileAttributeType - = CorrelationAttributeInstance.getDefaultCorrelationTypes() - .stream() - .filter(attrType -> attrType.getId() == CorrelationAttributeInstance.FILES_TYPE_ID) - .findAny() - .get(); - - ret.add(new CorrelationAttributeInstance(fileAttributeType, md5)); - } catch (EamDbException ex) { - LOGGER.log(Level.SEVERE, "Error connecting to DB", ex); // NON-NLS - } catch (CorrelationAttributeNormalizationException ex) { - LOGGER.log(Level.INFO, String.format("Unable to create CorrelationAttributeInstance for value %s", md5), ex); // NON-NLS - } + ret.add(new CorrelationAttributeInstance(fileAttributeType, md5)); + } catch (EamDbException ex) { + LOGGER.log(Level.SEVERE, "Error connecting to DB", ex); // NON-NLS + } catch (CorrelationAttributeNormalizationException ex) { + LOGGER.log(Level.INFO, String.format("Unable to create CorrelationAttributeInstance for value %s", md5), ex); // NON-NLS } } } @@ -515,9 +511,9 @@ public class DataContentViewerOtherCases extends JPanel implements DataContentVi * artifact. If the central repo is not enabled, this will only return files * from the current case with matching MD5 hashes. * - * @param corAttr CorrelationAttribute to query for + * @param corAttr CorrelationAttribute to query for * @param dataSourceName Data source to filter results - * @param deviceId Device Id to filter results + * @param deviceId Device Id to filter results * * @return A collection of correlated artifact instances */ @@ -580,7 +576,7 @@ public class DataContentViewerOtherCases extends JPanel implements DataContentVi * Get all other abstract files in the current case with the same MD5 as the * selected node. * - * @param corAttr The CorrelationAttribute containing the MD5 to search for + * @param corAttr The CorrelationAttribute containing the MD5 to search for * @param openCase The current case * * @return List of matching AbstractFile objects @@ -657,11 +653,9 @@ public class DataContentViewerOtherCases extends JPanel implements DataContentVi // - The central repo is disabled and the backing file has a valid MD5 hash this.file = this.getAbstractFileFromNode(node); if (EamDb.isEnabled()) { - return this.file != null - && this.file.getSize() > 0 - && !getCorrelationAttributesFromNode(node).isEmpty(); + return !getCorrelationAttributesFromNode(node).isEmpty(); } else { - return this.file != null + return this.file != null && this.file.getSize() > 0 && ((this.file.getMd5Hash() != null) && (!this.file.getMd5Hash().isEmpty())); } @@ -733,8 +727,8 @@ public class DataContentViewerOtherCases extends JPanel implements DataContentVi * Adjust a given column for the text provided. * * @param columnIndex The index of the column to adjust. - * @param text The text whose length will be used to adjust the column - * width. + * @param text The text whose length will be used to adjust the + * column width. */ private void setColumnWidthToText(int columnIndex, String text) { TableColumn column = otherCasesTable.getColumnModel().getColumn(columnIndex); From 4b7e074a56792fe651f256b4d50e354ffb7c8d3d Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Tue, 30 Oct 2018 15:31:28 -0400 Subject: [PATCH 036/129] Cleaned up some stuff and made things not public api --- .../datamodel/AbstractAbstractFileNode.java | 49 +++++-------------- .../autopsy/datamodel/FileProperty.java | 12 +++-- 2 files changed, 19 insertions(+), 42 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java index 16cdf400d9..b2dd93705c 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java @@ -67,7 +67,7 @@ import org.sleuthkit.datamodel.TskData; * @param type of the AbstractFile to encapsulate */ public abstract class AbstractAbstractFileNode extends AbstractContentNode { - + private static final Logger logger = Logger.getLogger(AbstractAbstractFileNode.class.getName()); @NbBundle.Messages("AbstractAbstractFileNode.addFileProperty.desc=no description") private static final String NO_DESCR = AbstractAbstractFileNode_addFileProperty_desc(); @@ -268,38 +268,14 @@ public abstract class AbstractAbstractFileNode extends A } }, COMMENT(AbstractAbstractFileNode_createSheet_comment_name()) { - Optional> result = Optional.empty(); - List scoreAndCommentTags; - CorrelationAttributeInstance correlationAttribute; - - private void initResult(AbstractFile content) { - scoreAndCommentTags = getContentTagsFromDatabase(content); - correlationAttribute = null; - if (EamDbUtil.useCentralRepo() - && !UserPreferences.hideCentralRepoCommentsAndOccurrences()) { - if (EamDbUtil.useCentralRepo() && UserPreferences.hideCentralRepoCommentsAndOccurrences() == false) { - correlationAttribute = getCorrelationAttributeInstance(content); - } - } - result = Optional.of(getCommentProperty(scoreAndCommentTags, correlationAttribute)); - } - @Override public Object getPropertyValue(AbstractFile content) { - if (!this.result.isPresent()) { - initResult(content); + List scoreAndCommentTags = getContentTagsFromDatabase(content); + CorrelationAttributeInstance correlationAttribute = null; + if (!UserPreferences.hideCentralRepoCommentsAndOccurrences()) { + correlationAttribute = getCorrelationAttributeInstance(content); } - HasCommentStatus res = result.get().getRight(); - result = Optional.empty(); - return res; - } - - @Override - public String getDescription(AbstractFile content) { - if (!this.result.isPresent()) { - initResult(content); - } - return result.get().getLeft(); + return getCommentProperty(scoreAndCommentTags, correlationAttribute); } }, COUNT(AbstractAbstractFileNode_createSheet_count_name()) { @@ -324,8 +300,8 @@ public abstract class AbstractAbstractFileNode extends A @Override public boolean isDisabled() { - return !EamDbUtil.useCentralRepo() - || UserPreferences.hideCentralRepoCommentsAndOccurrences(); + return UserPreferences.hideCentralRepoCommentsAndOccurrences() + || !EamDbUtil.useCentralRepo(); } @Override @@ -335,7 +311,6 @@ public abstract class AbstractAbstractFileNode extends A } return result.get().getLeft(); } - }, LOCATION(AbstractAbstractFileNode_locationColLbl()) { @Override @@ -469,7 +444,7 @@ public abstract class AbstractAbstractFileNode extends A return displayString; } - public static FileProperty getPropertyFromDisplayName(String displayName) { + static FileProperty getPropertyFromDisplayName(String displayName) { for (FileProperty p : AbstractFilePropertyType.values()) { if (p.getPropertyName().equals(displayName)) { return p; @@ -478,7 +453,7 @@ public abstract class AbstractAbstractFileNode extends A return null; } } - + /** * Fill map with AbstractFile properties * @@ -528,7 +503,7 @@ public abstract class AbstractAbstractFileNode extends A */ @NbBundle.Messages({ "AbstractAbstractFileNode.createSheet.comment.displayName=C"}) - private static Pair getCommentProperty(List tags, CorrelationAttributeInstance attribute) { + private static HasCommentStatus getCommentProperty(List tags, CorrelationAttributeInstance attribute) { HasCommentStatus status = tags.size() > 0 ? HasCommentStatus.TAG_NO_COMMENT : HasCommentStatus.NO_COMMENT; @@ -546,7 +521,7 @@ public abstract class AbstractAbstractFileNode extends A status = HasCommentStatus.CR_COMMENT; } } - return Pair.of(NO_DESCR, status); + return status; } /** diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/FileProperty.java b/Core/src/org/sleuthkit/autopsy/datamodel/FileProperty.java index 49b2df6a32..113e252d2f 100755 --- a/Core/src/org/sleuthkit/autopsy/datamodel/FileProperty.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/FileProperty.java @@ -11,19 +11,21 @@ import org.sleuthkit.datamodel.AbstractFile; * * @author dsmyda */ -public interface FileProperty { + +//No modifier means its is only package usable (not part of public api) +interface FileProperty { /** * * @param content * @return */ - public Object getPropertyValue(AbstractFile content); + Object getPropertyValue(AbstractFile content); /** * * @return */ - public default String getPropertyName(){ + default String getPropertyName(){ return this.toString(); } @@ -31,7 +33,7 @@ public interface FileProperty { * * @return */ - public default boolean isDisabled() { + default boolean isDisabled() { return false; } @@ -40,7 +42,7 @@ public interface FileProperty { * @param content * @return */ - public default String getDescription(AbstractFile content) { + default String getDescription(AbstractFile content) { return ""; } } From 8576c46f829bb83b38e181e044e61b69b3bcb828 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Tue, 30 Oct 2018 16:00:05 -0400 Subject: [PATCH 037/129] 4305 add comments to new methods regarding getting templates for database statements --- .../datamodel/AbstractSqlEamDb.java | 5 ++- .../datamodel/EamArtifactUtil.java | 7 ++-- .../datamodel/PostgresEamDbSettings.java | 39 ++++++++++++++++++ .../datamodel/SqliteEamDbSettings.java | 41 ++++++++++++++++++- 4 files changed, 85 insertions(+), 7 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java index b14d9043af..4364845648 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java @@ -3140,10 +3140,9 @@ abstract class AbstractSqlEamDb implements EamDb { final String addDataSourceIdIndexTemplate; final String addValueIndexTemplate; final String addKnownStatusIndexTemplate; + //get the data base specific code for creating a new _instance table switch (selectedPlatform) { case POSTGRESQL: - //create table //settings.getStatementTemplate then statement.execute here with the added table - //add correlation attr to correlation_attrs table addSsidTableTemplate = PostgresEamDbSettings.getCreateArtifactInstancesTableTemplate(); addCaseIdIndexTemplate = PostgresEamDbSettings.getAddCaseIdIndexTemplate(); addDataSourceIdIndexTemplate = PostgresEamDbSettings.getAddDataSourceIdIndexTemplate(); @@ -3162,6 +3161,7 @@ abstract class AbstractSqlEamDb implements EamDb { } final String wirelessNetworsDbTableName = "wireless_networks"; final String wirelessNetworksTableInstanceName = wirelessNetworsDbTableName + "_instances"; + //add the wireless_networks attribute to the correlation_types table final String addAttributeSql = "INSERT INTO correlation_types(id, display_name, db_table_name, supported, enabled) VALUES (?, ?, ?, ?, ?)"; preparedStatement = conn.prepareStatement(addAttributeSql); preparedStatement.setInt(1, CorrelationAttributeInstance.SSID_TYPE_ID); @@ -3171,6 +3171,7 @@ abstract class AbstractSqlEamDb implements EamDb { preparedStatement.setInt(5, 1); preparedStatement.execute(); + //create a new wireless_networks_instances table and add indexes for its columns statement.execute(String.format(addSsidTableTemplate, wirelessNetworksTableInstanceName, wirelessNetworksTableInstanceName)); statement.execute(String.format(addCaseIdIndexTemplate, wirelessNetworksTableInstanceName, wirelessNetworksTableInstanceName)); statement.execute(String.format(addDataSourceIdIndexTemplate, wirelessNetworksTableInstanceName, wirelessNetworksTableInstanceName)); diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactUtil.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactUtil.java index 2477d0fc41..aca0471345 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactUtil.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactUtil.java @@ -163,11 +163,10 @@ public class EamArtifactUtil { && BlackboardArtifact.ARTIFACT_TYPE.TSK_DEVICE_ATTACHED.getTypeID() == artifactTypeID) { value = bbArtifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DEVICE_ID)).getValueString(); - } - else if (correlationType.getId() == CorrelationAttributeInstance.SSID_TYPE_ID + } else if (correlationType.getId() == CorrelationAttributeInstance.SSID_TYPE_ID && BlackboardArtifact.ARTIFACT_TYPE.TSK_WIFI_NETWORK.getTypeID() == artifactTypeID) { value = bbArtifact.getAttribute(new BlackboardAttribute.Type(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SSID)).getValueString(); - } + } } catch (TskCoreException ex) { logger.log(Level.SEVERE, "Error getting attribute while getting type from BlackboardArtifact.", ex); // NON-NLS return null; @@ -187,7 +186,7 @@ public class EamArtifactUtil { * Uses the determined type and vallue, then looks up instance details to * create proper CorrelationAttributeInstance. * - * @param bbArtifact the blackboard artifatc + * @param bbArtifact the blackboard artifact * @param correlationType the given type * @param value the artifact value * diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDbSettings.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDbSettings.java index adaf7c0d9c..6ee454f915 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDbSettings.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDbSettings.java @@ -468,6 +468,13 @@ public final class PostgresEamDbSettings { return true; } + /** + * Get the template String for creating a new _instances table in a Postgres + * central repository. %s will exist in the template where the name of the + * new table will be addedd. + * + * @return a String which is a template for cretating a new _instances table + */ static String getCreateArtifactInstancesTableTemplate() { // Each "%s" will be replaced with the relevant TYPE_instances table name. StringBuilder createArtifactInstancesTableTemplate = new StringBuilder(); @@ -486,21 +493,53 @@ public final class PostgresEamDbSettings { return createArtifactInstancesTableTemplate.toString(); } + /** + * Get the template for creating an index on the case_id column of an + * instance table. %s will exist in the template where the name of the new + * table will be addedd. + * + * @return a String which is a template for adding an index to the case_id + * column of a _instances table + */ static String getAddCaseIdIndexTemplate() { // Each "%s" will be replaced with the relevant TYPE_instances table name. return "CREATE INDEX IF NOT EXISTS %s_case_id ON %s (case_id)"; } + /** + * Get the template for creating an index on the data_source_id column of an + * instance table. %s will exist in the template where the name of the new + * table will be addedd. + * + * @return a String which is a template for adding an index to the + * data_source_id column of a _instances table + */ static String getAddDataSourceIdIndexTemplate() { // Each "%s" will be replaced with the relevant TYPE_instances table name. return "CREATE INDEX IF NOT EXISTS %s_data_source_id ON %s (data_source_id)"; } + /** + * Get the template for creating an index on the value column of an instance + * table. %s will exist in the template where the name of the new table will + * be addedd. + * + * @return a String which is a template for adding an index to the value + * column of a _instances table + */ static String getAddValueIndexTemplate() { // Each "%s" will be replaced with the relevant TYPE_instances table name. return "CREATE INDEX IF NOT EXISTS %s_value ON %s (value)"; } + /** + * Get the template for creating an index on the known_status column of an + * instance table. %s will exist in the template where the name of the new + * table will be addedd. + * + * @return a String which is a template for adding an index to the + * known_status column of a _instances table + */ static String getAddKnownStatusIndexTemplate() { // Each "%s" will be replaced with the relevant TYPE_instances table name. return "CREATE INDEX IF NOT EXISTS %s_value_known_status ON %s (value, known_status)"; diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDbSettings.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDbSettings.java index c321e3b412..615e49e523 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDbSettings.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDbSettings.java @@ -415,7 +415,14 @@ public final class SqliteEamDbSettings { } return true; } - + + /** + * Get the template String for creating a new _instances table in a Sqlite + * central repository. %s will exist in the template where the name of the + * new table will be addedd. + * + * @return a String which is a template for cretating a new _instances table + */ static String getCreateArtifactInstancesTableTemplate() { // Each "%s" will be replaced with the relevant TYPE_instances table name. StringBuilder createArtifactInstancesTableTemplate = new StringBuilder(); @@ -434,21 +441,53 @@ public final class SqliteEamDbSettings { return createArtifactInstancesTableTemplate.toString(); } + /** + * Get the template for creating an index on the case_id column of an + * instance table. %s will exist in the template where the name of the new + * table will be addedd. + * + * @return a String which is a template for adding an index to the case_id + * column of a _instances table + */ static String getAddCaseIdIndexTemplate() { // Each "%s" will be replaced with the relevant TYPE_instances table name. return "CREATE INDEX IF NOT EXISTS %s_case_id ON %s (case_id)"; } + /** + * Get the template for creating an index on the data_source_id column of an + * instance table. %s will exist in the template where the name of the new + * table will be addedd. + * + * @return a String which is a template for adding an index to the + * data_source_id column of a _instances table + */ static String getAddDataSourceIdIndexTemplate() { // Each "%s" will be replaced with the relevant TYPE_instances table name. return "CREATE INDEX IF NOT EXISTS %s_data_source_id ON %s (data_source_id)"; } + /** + * Get the template for creating an index on the value column of an instance + * table. %s will exist in the template where the name of the new table will + * be addedd. + * + * @return a String which is a template for adding an index to the value + * column of a _instances table + */ static String getAddValueIndexTemplate() { // Each "%s" will be replaced with the relevant TYPE_instances table name. return "CREATE INDEX IF NOT EXISTS %s_value ON %s (value)"; } + /** + * Get the template for creating an index on the known_status column of an + * instance table. %s will exist in the template where the name of the new + * table will be addedd. + * + * @return a String which is a template for adding an index to the + * known_status column of a _instances table + */ static String getAddKnownStatusIndexTemplate() { // Each "%s" will be replaced with the relevant TYPE_instances table name. return "CREATE INDEX IF NOT EXISTS %s_value_known_status ON %s (value, known_status)"; From 8c7f987049ad11525abd35a104f9a0144072fd1a Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Tue, 30 Oct 2018 16:53:37 -0400 Subject: [PATCH 038/129] 4305 prevent older versions of Autopsy from lowering CR schema version number --- .../centralrepository/datamodel/AbstractSqlEamDb.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java index 4364845648..981a793f8d 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java @@ -3120,7 +3120,11 @@ abstract class AbstractSqlEamDb implements EamDb { logger.log(Level.INFO, "Central Repository is up to date"); return; } - + if (dbSchemaVersion.compareTo(CURRENT_DB_SCHEMA_VERSION) > 0) { + logger.log(Level.INFO, "Central Repository is of newer version than software creates") + return; + } + // Update from 1.0 to 1.1 if (dbSchemaVersion.compareTo(new CaseDbSchemaVersionNumber(1, 1)) < 0) { statement.execute("ALTER TABLE reference_sets ADD COLUMN known_status INTEGER;"); //NON-NLS From e4c08c87ef628fe52a0b21c4ea2ff30e67e005e3 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Tue, 30 Oct 2018 16:55:39 -0400 Subject: [PATCH 039/129] 4305 add missing semi-colon --- .../autopsy/centralrepository/datamodel/AbstractSqlEamDb.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java index 981a793f8d..fb80bbdacc 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java @@ -3121,7 +3121,7 @@ abstract class AbstractSqlEamDb implements EamDb { return; } if (dbSchemaVersion.compareTo(CURRENT_DB_SCHEMA_VERSION) > 0) { - logger.log(Level.INFO, "Central Repository is of newer version than software creates") + logger.log(Level.INFO, "Central Repository is of newer version than software creates"); return; } From 1c1163ed09bd5489a83b1b204df0d7107a7a5a06 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Tue, 30 Oct 2018 16:58:55 -0400 Subject: [PATCH 040/129] 4305 add update to 1.2 comment --- .../autopsy/centralrepository/datamodel/AbstractSqlEamDb.java | 1 + 1 file changed, 1 insertion(+) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java index fb80bbdacc..766bab44e3 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java @@ -3136,6 +3136,7 @@ abstract class AbstractSqlEamDb implements EamDb { // regardless of whether this succeeds. EamDbUtil.insertDefaultOrganization(conn); } + //Update to 1.2 if (dbSchemaVersion.compareTo(new CaseDbSchemaVersionNumber(1, 2)) < 0) { //update central repository to be able to store new correlation attributes EamDbPlatformEnum selectedPlatform = EamDbPlatformEnum.getSelectedPlatform(); From 7e647388dfd878d2781952d9f4a585d696a3b1a1 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Wed, 31 Oct 2018 10:17:02 -0400 Subject: [PATCH 041/129] Refactored everything and testing to see if things are working as expected --- Core/nbproject/project.xml | 4 - .../datamodel/AbstractAbstractFileNode.java | 614 +++++++++--------- .../autopsy/datamodel/FileProperty.java | 60 +- .../autopsy/datamodel/LayoutFileNode.java | 4 - .../autopsy/datamodel/LocalDirectoryNode.java | 54 -- .../datamodel/VirtualDirectoryNode.java | 30 +- .../TextTranslationService.java | 46 -- .../texttranslation/TranslationCallback.java | 49 -- 8 files changed, 341 insertions(+), 520 deletions(-) delete mode 100755 Core/src/org/sleuthkit/autopsy/texttranslation/TranslationCallback.java diff --git a/Core/nbproject/project.xml b/Core/nbproject/project.xml index 14b3c6afad..2c6b0fce9a 100644 --- a/Core/nbproject/project.xml +++ b/Core/nbproject/project.xml @@ -338,10 +338,6 @@ org.sleuthkit.autopsy.modules.vmextractor org.sleuthkit.autopsy.progress org.sleuthkit.autopsy.report -<<<<<<< HEAD -======= - org.sleuthkit.autopsy.tabulardatareader ->>>>>>> 4316-text-translation-service org.sleuthkit.autopsy.texttranslation org.sleuthkit.datamodel diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java index c67a93c687..a60a7e3bce 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java @@ -21,13 +21,12 @@ package org.sleuthkit.autopsy.datamodel; import java.beans.PropertyChangeEvent; import java.beans.PropertyChangeListener; import java.util.ArrayList; -import java.util.Arrays; import java.util.EnumSet; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.logging.Level; import java.util.stream.Collectors; import org.apache.commons.io.FilenameUtils; @@ -52,13 +51,13 @@ import org.sleuthkit.autopsy.core.UserPreferences; import org.sleuthkit.autopsy.corecomponents.DataResultViewerTable.Score; import org.sleuthkit.autopsy.coreutils.Logger; import static org.sleuthkit.autopsy.datamodel.Bundle.*; +import static org.sleuthkit.autopsy.datamodel.AbstractAbstractFileNode.AbstractFilePropertyType.*; import org.sleuthkit.autopsy.corecomponents.DataResultViewerTable.HasCommentStatus; import org.sleuthkit.autopsy.events.AutopsyEvent; import org.sleuthkit.autopsy.ingest.IngestManager; import org.sleuthkit.autopsy.ingest.ModuleContentEvent; import org.sleuthkit.autopsy.texttranslation.NoServiceProviderException; import org.sleuthkit.autopsy.texttranslation.TextTranslationService; -import org.sleuthkit.autopsy.texttranslation.TranslationCallback; import org.sleuthkit.autopsy.texttranslation.TranslationException; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.BlackboardArtifact; @@ -77,14 +76,17 @@ public abstract class AbstractAbstractFileNode extends A private static final Logger logger = Logger.getLogger(AbstractAbstractFileNode.class.getName()); @NbBundle.Messages("AbstractAbstractFileNode.addFileProperty.desc=no description") private static final String NO_DESCR = AbstractAbstractFileNode_addFileProperty_desc(); - + private static final String TRANSLATION_AVAILABLE_EVENT = "TRANSLATION_AVAILABLE"; private static final String NO_TRANSLATION = ""; private static final Set CASE_EVENTS_OF_INTEREST = EnumSet.of(Case.Events.CURRENT_CASE, Case.Events.CONTENT_TAG_ADDED, Case.Events.CONTENT_TAG_DELETED, Case.Events.CR_COMMENT_CHANGED); - private String translatedFileName; + private volatile String translatedFileName; + + private static final ExecutorService pool; + private static final Integer MAX_POOL_SIZE = 10; /** * @param abstractFile file to wrap @@ -106,6 +108,10 @@ public abstract class AbstractAbstractFileNode extends A Case.addEventTypeSubscriber(CASE_EVENTS_OF_INTEREST, weakPcl); } + static { + pool = Executors.newFixedThreadPool(MAX_POOL_SIZE); + } + /** * The finalizer removes event listeners as the BlackboardArtifactNode is * being garbage collected. Yes, we know that finalizers are considered to @@ -192,6 +198,10 @@ public abstract class AbstractAbstractFileNode extends A * unregistering of the listener in removeListeners() below. */ private final PropertyChangeListener weakPcl = WeakListeners.propertyChange(pcl, null); + + Sheet getBlankSheet() { + return super.createSheet(); + } private void updateSheet() { this.setSheet(createSheet()); @@ -200,33 +210,30 @@ public abstract class AbstractAbstractFileNode extends A @Override protected Sheet createSheet() { Sheet sheet = super.createSheet(); - Sheet.Set sheetSet = sheet.get(Sheet.PROPERTIES); - if (sheetSet == null) { - sheetSet = Sheet.createPropertiesSet(); - sheet.put(sheetSet); - } + Sheet.Set sheetSet = Sheet.createPropertiesSet(); + sheet.put(sheetSet); + + List properties = getProperties(); - Map map = new LinkedHashMap<>(); - fillPropertyMap(map, getContent()); - - for (Map.Entry entry : map.entrySet()) { - String desc = Bundle.AbstractFsContentNode_noDesc_text(); - - FileProperty p = AbstractFilePropertyType.getPropertyFromDisplayName(entry.getKey()); - if (!p.getDescription(content).isEmpty()) { - desc = p.getDescription(content); + //Add only the enabled properties to the sheet! + for (FileProperty property : properties) { + if (property.isEnabled()) { + sheetSet.put(new NodeProperty<>( + property.getPropertyName(), + property.getPropertyName(), + property.getDescription(), + property.getPropertyValue())); } - sheetSet.put(new NodeProperty<>(entry.getKey(), entry.getKey(), desc, entry.getValue())); } return sheet; } @NbBundle.Messages({"AbstractAbstractFileNode.nameColLbl=Name", + "AbstractAbstractFileNode.translateFileName=Translated Name", "AbstractAbstractFileNode.createSheet.score.name=S", "AbstractAbstractFileNode.createSheet.comment.name=C", "AbstractAbstractFileNode.createSheet.count.name=O", - "AbstractAbstractFileNode.translateFileName=Translated Name", "AbstractAbstractFileNode.locationColLbl=Location", "AbstractAbstractFileNode.modifiedTimeColLbl=Modified Time", "AbstractAbstractFileNode.changeTimeColLbl=Change Time", @@ -247,207 +254,33 @@ public abstract class AbstractAbstractFileNode extends A "AbstractAbstractFileNode.objectId=Object ID", "AbstractAbstractFileNode.mimeType=MIME Type", "AbstractAbstractFileNode.extensionColLbl=Extension"}) - public enum AbstractFilePropertyType implements FileProperty { + public enum AbstractFilePropertyType { - NAME(AbstractAbstractFileNode_nameColLbl()) { - @Override - public Object getPropertyValue(AbstractFile content) { - return getContentDisplayName(content); - } - }, - SCORE(AbstractAbstractFileNode_createSheet_score_name()) { - Optional> result = Optional.empty(); - List scoreAndCommentTags; - CorrelationAttributeInstance correlationAttribute; - - private void initResult(AbstractFile content) { - scoreAndCommentTags = getContentTagsFromDatabase(content); - result = Optional.of(getScoreProperty(content, scoreAndCommentTags)); - } - - @Override - public Object getPropertyValue(AbstractFile content) { - if (!this.result.isPresent()) { - initResult(content); - } - Score res = result.get().getRight(); - result = Optional.empty(); - return res; - } - - @Override - public String getDescription(AbstractFile content) { - if (!this.result.isPresent()) { - initResult(content); - } - return result.get().getLeft(); - } - }, - COMMENT(AbstractAbstractFileNode_createSheet_comment_name()) { - @Override - public Object getPropertyValue(AbstractFile content) { - List scoreAndCommentTags = getContentTagsFromDatabase(content); - CorrelationAttributeInstance correlationAttribute = null; - if (!UserPreferences.hideCentralRepoCommentsAndOccurrences()) { - correlationAttribute = getCorrelationAttributeInstance(content); - } - return getCommentProperty(scoreAndCommentTags, correlationAttribute); - } - }, - COUNT(AbstractAbstractFileNode_createSheet_count_name()) { - Optional> result = Optional.empty(); - List scoreAndCommentTags; - CorrelationAttributeInstance correlationAttribute; - - private void initResult(AbstractFile content) { - correlationAttribute = getCorrelationAttributeInstance(content); - result = Optional.of(getCountProperty(correlationAttribute)); - } - - @Override - public Object getPropertyValue(AbstractFile content) { - if (!result.isPresent()) { - initResult(content); - } - Long res = result.get().getRight(); - result = Optional.empty(); - return res; - } - - @Override - public boolean isDisabled() { - return UserPreferences.hideCentralRepoCommentsAndOccurrences() - || !EamDbUtil.useCentralRepo(); - } - - @Override - public String getDescription(AbstractFile content) { - if (!result.isPresent()) { - initResult(content); - } - return result.get().getLeft(); - } - }, - LOCATION(AbstractAbstractFileNode_locationColLbl()) { - @Override - public Object getPropertyValue(AbstractFile content) { - return getContentPath(content); - } - }, - MOD_TIME(AbstractAbstractFileNode_modifiedTimeColLbl()) { - @Override - public Object getPropertyValue(AbstractFile content) { - return ContentUtils.getStringTime(content.getMtime(), content); - } - }, - CHANGED_TIME(AbstractAbstractFileNode_changeTimeColLbl()) { - @Override - public Object getPropertyValue(AbstractFile content) { - return ContentUtils.getStringTime(content.getCtime(), content); - } - }, - ACCESS_TIME(AbstractAbstractFileNode_accessTimeColLbl()) { - @Override - public Object getPropertyValue(AbstractFile content) { - return ContentUtils.getStringTime(content.getAtime(), content); - } - }, - CREATED_TIME(AbstractAbstractFileNode_createdTimeColLbl()) { - @Override - public Object getPropertyValue(AbstractFile content) { - return ContentUtils.getStringTime(content.getCrtime(), content); - } - }, - SIZE(AbstractAbstractFileNode_sizeColLbl()) { - @Override - public Object getPropertyValue(AbstractFile content) { - return content.getSize(); - } - }, - FLAGS_DIR(AbstractAbstractFileNode_flagsDirColLbl()) { - @Override - public Object getPropertyValue(AbstractFile content) { - return content.getDirFlagAsString(); - } - }, - FLAGS_META(AbstractAbstractFileNode_flagsMetaColLbl()) { - @Override - public Object getPropertyValue(AbstractFile content) { - return content.getMetaFlagsAsString(); - } - }, - MODE(AbstractAbstractFileNode_modeColLbl()) { - @Override - public Object getPropertyValue(AbstractFile content) { - return content.getModesAsString(); - } - }, - USER_ID(AbstractAbstractFileNode_useridColLbl()) { - @Override - public Object getPropertyValue(AbstractFile content) { - return content.getUid(); - } - }, - GROUP_ID(AbstractAbstractFileNode_groupidColLbl()) { - @Override - public Object getPropertyValue(AbstractFile content) { - return content.getGid(); - } - }, - META_ADDR(AbstractAbstractFileNode_metaAddrColLbl()) { - @Override - public Object getPropertyValue(AbstractFile content) { - return content.getMetaAddr(); - } - }, - ATTR_ADDR(AbstractAbstractFileNode_attrAddrColLbl()) { - @Override - public Object getPropertyValue(AbstractFile content) { - return content.getAttrType().getValue() + "-" + content.getAttributeId(); - } - }, - TYPE_DIR(AbstractAbstractFileNode_typeDirColLbl()) { - @Override - public Object getPropertyValue(AbstractFile content) { - return content.getDirType().getLabel(); - } - }, - TYPE_META(AbstractAbstractFileNode_typeMetaColLbl()) { - @Override - public Object getPropertyValue(AbstractFile content) { - return content.getMetaType().toString(); - } - }, - KNOWN(AbstractAbstractFileNode_knownColLbl()) { - @Override - public Object getPropertyValue(AbstractFile content) { - return content.getKnown().getName(); - } - }, - MD5HASH(AbstractAbstractFileNode_md5HashColLbl()) { - @Override - public Object getPropertyValue(AbstractFile content) { - return StringUtils.defaultString(content.getMd5Hash()); - } - }, - ObjectID(AbstractAbstractFileNode_objectId()) { - @Override - public Object getPropertyValue(AbstractFile content) { - return content.getId(); - } - }, - MIMETYPE(AbstractAbstractFileNode_mimeType()) { - @Override - public Object getPropertyValue(AbstractFile content) { - return StringUtils.defaultString(content.getMIMEType()); - } - }, - EXTENSION(AbstractAbstractFileNode_extensionColLbl()) { - @Override - public Object getPropertyValue(AbstractFile content) { - return content.getNameExtension(); - } - }; + NAME(AbstractAbstractFileNode_nameColLbl()), + TRANSLATION(AbstractAbstractFileNode_translateFileName()), + SCORE(AbstractAbstractFileNode_createSheet_score_name()), + COMMENT(AbstractAbstractFileNode_createSheet_comment_name()), + OCCURRENCES(AbstractAbstractFileNode_createSheet_count_name()), + LOCATION(AbstractAbstractFileNode_locationColLbl()), + MOD_TIME(AbstractAbstractFileNode_modifiedTimeColLbl()), + CHANGED_TIME(AbstractAbstractFileNode_changeTimeColLbl()), + ACCESS_TIME(AbstractAbstractFileNode_accessTimeColLbl()), + CREATED_TIME(AbstractAbstractFileNode_createdTimeColLbl()), + SIZE(AbstractAbstractFileNode_sizeColLbl()), + FLAGS_DIR(AbstractAbstractFileNode_flagsDirColLbl()), + FLAGS_META(AbstractAbstractFileNode_flagsMetaColLbl()), + MODE(AbstractAbstractFileNode_modeColLbl()), + USER_ID(AbstractAbstractFileNode_useridColLbl()), + GROUP_ID(AbstractAbstractFileNode_groupidColLbl()), + META_ADDR(AbstractAbstractFileNode_metaAddrColLbl()), + ATTR_ADDR(AbstractAbstractFileNode_attrAddrColLbl()), + TYPE_DIR(AbstractAbstractFileNode_typeDirColLbl()), + TYPE_META(AbstractAbstractFileNode_typeMetaColLbl()), + KNOWN(AbstractAbstractFileNode_knownColLbl()), + MD5HASH(AbstractAbstractFileNode_md5HashColLbl()), + ObjectID(AbstractAbstractFileNode_objectId()), + MIMETYPE(AbstractAbstractFileNode_mimeType()), + EXTENSION(AbstractAbstractFileNode_extensionColLbl()); final private String displayString; @@ -459,31 +292,22 @@ public abstract class AbstractAbstractFileNode extends A public String toString() { return displayString; } - - static FileProperty getPropertyFromDisplayName(String displayName) { - for (FileProperty p : AbstractFilePropertyType.values()) { - if (p.getPropertyName().equals(displayName)) { - return p; - } - } - return null; - } } /** - * Attempts translation on the file name by kicking off a background task to do the - * translation. Once the background task is done, it will fire a PropertyChangeEvent, - * which will force this node to refresh itself, thus updating its translated name - * column. + * Attempts translation on the file name by kicking off a background task to + * do the translation. Once the background task is done, it will fire a + * PropertyChangeEvent, which will force this node to refresh itself, thus + * updating its translated name column. * * @return The file names translation. */ - protected String getTranslatedFileName() { + private String getTranslatedFileName(AbstractFile content) { //If already in complete English, don't translate. - if (this.content.getName().matches("^\\p{ASCII}+$")) { + if (content.getName().matches("^\\p{ASCII}+$")) { return NO_TRANSLATION; } - + //If we already have a translation use that one. if (translatedFileName != null) { return translatedFileName; @@ -494,29 +318,221 @@ public abstract class AbstractAbstractFileNode extends A TextTranslationService tts = TextTranslationService.getInstance(); if (tts.hasProvider()) { //Seperate out the base and ext from the contents file name. - String base = FilenameUtils.getBaseName(this.content.getName()); + String base = FilenameUtils.getBaseName(content.getName()); //Send only the base file name to be translated. Once the translation comes //back fire a PropertyChangeEvent on this nodes PropertyChangeListener. - tts.translateAsynchronously(base, new TranslateFileNameCallback(this.content, weakPcl)); + pool.submit(() -> { + try { + String translation = tts.translate(base); + String ext = FilenameUtils.getExtension(content.getName()); + + //If we have no extension, then we shouldn't add the . + String extensionDelimiter = (ext.isEmpty()) ? "" : "."; + + //Talk directly to this nodes pcl, fire an update when the translation + //is complete. + if (!translation.isEmpty()) { + weakPcl.propertyChange(new PropertyChangeEvent( + AutopsyEvent.SourceType.LOCAL.toString(), + TRANSLATION_AVAILABLE_EVENT, null, + translation + extensionDelimiter + ext)); + } + } catch (NoServiceProviderException noServiceEx) { + logger.log(Level.WARNING, "Translate unsuccessful because no TextTranslator " + + "implementation was provided.", noServiceEx); + } catch (TranslationException noTranslationEx) { + logger.log(Level.WARNING, "Could not successfully translate file name " + + content.getName(), noTranslationEx); + } + }); } //In the mean time, return a blank translation. return NO_TRANSLATION; } - + /** - * Fill map with AbstractFile properties * - * @param map map with preserved ordering, where property names/values - * are put - * @param content The content to get properties for. + * @return */ - static public void fillPropertyMap(Map map, AbstractFile content) { - Arrays.asList(AbstractFilePropertyType.values()) - .stream() - .filter(p -> !p.isDisabled()) - .forEach((p) -> map.put(p.getPropertyName(), p.getPropertyValue(content))); + List getProperties() { + ArrayList properties = new ArrayList<>(); + + properties.add(new FileProperty(NAME.toString()) { + @Override + public Object getPropertyValue() { + return getContentDisplayName(content); + } + }); + properties.add(new FileProperty(TRANSLATION.toString()) { + @Override + public Object getPropertyValue() { + return getTranslatedFileName(content); + } + + @Override + public boolean isEnabled() { + return UserPreferences.displayTranslationFileNames(); + } + }); + + List tags = getContentTagsFromDatabase(content); + CorrelationAttributeInstance correlationAttribute = getCorrelationAttributeInstance(); + properties.add(new FileProperty(SCORE.toString()) { + @Override + public Object getPropertyValue() { + Pair result = getScoreProperty(content, tags); + setDescription(result.getLeft()); + return result.getRight(); + } + }); + properties.add(new FileProperty(COMMENT.toString()) { + @Override + public Object getPropertyValue() { + return getCommentProperty(tags, correlationAttribute); + } + + @Override + public boolean isEnabled() { + return !UserPreferences.hideCentralRepoCommentsAndOccurrences(); + } + }); + properties.add(new FileProperty(OCCURRENCES.toString()) { + @Override + public Object getPropertyValue() { + Pair result = getCountProperty(correlationAttribute); + setDescription(result.getLeft()); + return result.getRight(); + } + + @Override + public boolean isEnabled() { + return !UserPreferences.hideCentralRepoCommentsAndOccurrences(); + } + }); + properties.add(new FileProperty(LOCATION.toString()) { + @Override + public Object getPropertyValue() { + return getContentPath(content); + } + }); + properties.add(new FileProperty(MOD_TIME.toString()) { + @Override + public Object getPropertyValue() { + return ContentUtils.getStringTime(content.getMtime(), content); + } + }); + properties.add(new FileProperty(CHANGED_TIME.toString()) { + @Override + public Object getPropertyValue() { + return ContentUtils.getStringTime(content.getCtime(), content); + } + }); + properties.add(new FileProperty(ACCESS_TIME.toString()) { + @Override + public Object getPropertyValue() { + return ContentUtils.getStringTime(content.getAtime(), content); + } + }); + properties.add(new FileProperty(CREATED_TIME.toString()) { + @Override + public Object getPropertyValue() { + return ContentUtils.getStringTime(content.getCrtime(), content); + } + }); + properties.add(new FileProperty(SIZE.toString()) { + @Override + public Object getPropertyValue() { + return content.getSize(); + } + }); + properties.add(new FileProperty(FLAGS_DIR.toString()) { + @Override + public Object getPropertyValue() { + return content.getDirFlagAsString(); + } + }); + properties.add(new FileProperty(FLAGS_META.toString()) { + @Override + public Object getPropertyValue() { + return content.getMetaFlagsAsString(); + } + }); + properties.add(new FileProperty(MODE.toString()) { + @Override + public Object getPropertyValue() { + return content.getModesAsString(); + } + }); + properties.add(new FileProperty(USER_ID.toString()) { + @Override + public Object getPropertyValue() { + return content.getUid(); + } + }); + properties.add(new FileProperty(GROUP_ID.toString()) { + @Override + public Object getPropertyValue() { + return content.getGid(); + } + }); + properties.add(new FileProperty(META_ADDR.toString()) { + @Override + public Object getPropertyValue() { + return content.getMetaAddr(); + } + }); + properties.add(new FileProperty(ATTR_ADDR.toString()) { + @Override + public Object getPropertyValue() { + return content.getAttrType().getValue() + "-" + content.getAttributeId(); + } + }); + properties.add(new FileProperty(TYPE_DIR.toString()) { + @Override + public Object getPropertyValue() { + return content.getDirType().getLabel(); + } + }); + properties.add(new FileProperty(TYPE_META.toString()) { + @Override + public Object getPropertyValue() { + return content.getMetaType().toString(); + } + }); + properties.add(new FileProperty(KNOWN.toString()) { + @Override + public Object getPropertyValue() { + return content.getKnown().getName(); + } + }); + properties.add(new FileProperty(MD5HASH.toString()) { + @Override + public Object getPropertyValue() { + return StringUtils.defaultString(content.getMd5Hash()); + } + }); + properties.add(new FileProperty(ObjectID.toString()) { + @Override + public Object getPropertyValue() { + return content.getId(); + } + }); + properties.add(new FileProperty(MIMETYPE.toString()) { + @Override + public Object getPropertyValue() { + return StringUtils.defaultString(content.getMIMEType()); + } + }); + properties.add(new FileProperty(EXTENSION.toString()) { + @Override + public Object getPropertyValue() { + return content.getNameExtension(); + } + }); + + return properties; } /** @@ -524,7 +540,7 @@ public abstract class AbstractAbstractFileNode extends A * * @return a list of tags that are associated with the file */ - private static List getContentTagsFromDatabase(AbstractFile content) { + private List getContentTagsFromDatabase(AbstractFile content) { List tags = new ArrayList<>(); try { tags.addAll(Case.getCurrentCaseThrows().getServices().getTagsManager().getContentTagsByContent(content)); @@ -534,7 +550,7 @@ public abstract class AbstractAbstractFileNode extends A return tags; } - private static CorrelationAttributeInstance getCorrelationAttributeInstance(AbstractFile content) { + private CorrelationAttributeInstance getCorrelationAttributeInstance() { CorrelationAttributeInstance correlationAttribute = null; if (EamDbUtil.useCentralRepo()) { correlationAttribute = EamArtifactUtil.getInstanceFromContent(content); @@ -554,7 +570,7 @@ public abstract class AbstractAbstractFileNode extends A */ @NbBundle.Messages({ "AbstractAbstractFileNode.createSheet.comment.displayName=C"}) - private static HasCommentStatus getCommentProperty(List tags, CorrelationAttributeInstance attribute) { + private HasCommentStatus getCommentProperty(List tags, CorrelationAttributeInstance attribute) { HasCommentStatus status = tags.size() > 0 ? HasCommentStatus.TAG_NO_COMMENT : HasCommentStatus.NO_COMMENT; @@ -712,70 +728,34 @@ public abstract class AbstractAbstractFileNode extends A } /** - * Implements the TranslationCallback interface so that the TextTranslationService - * can call these methods when an asynchronous translation task is complete. + * Fill map with AbstractFile properties + * + * @param map map with preserved ordering, where property names/values + * are put + * @param content The content to get properties for. */ - private class TranslateFileNameCallback implements TranslationCallback { - private final String ext; - private final String originalFileName; - - private final PropertyChangeListener listener; - - public TranslateFileNameCallback(AbstractFile file, PropertyChangeListener listener) { - this.ext = FilenameUtils.getExtension(content.getName()); - this.originalFileName = content.getName(); - this.listener = listener; - } - - /** - * Fires a PropertyChangeEvent on this nodes PropertyChangeListener - * when the translation is finished. Reconstruct the file name so we - * can properly display the translation. - * - * @param translation Result from the translation job submitted to - * Text translation service. - */ - @Override - public void onTranslationResult(String translation) { - //If we have no extension, then we shouldn't add the . - String extensionDelimiter = (ext.isEmpty()) ? "" : "."; - - //Talk directly to this nodes pcl, fire an update when the translation - //is complete. - if (translation.isEmpty()) { - listener.propertyChange(new PropertyChangeEvent( - AutopsyEvent.SourceType.LOCAL.toString(), - TRANSLATION_AVAILABLE_EVENT, null, translation)); - } else { - listener.propertyChange(new PropertyChangeEvent( - AutopsyEvent.SourceType.LOCAL.toString(), - TRANSLATION_AVAILABLE_EVENT, null, - translation + extensionDelimiter + ext)); - } - } - - /** - * Do nothing on a translation exception except log, the column will remain empty - * and there is nothing we can do as a fallback. - * - * @param ex Exception caught while translating. - */ - @Override - public void onTranslationException(TranslationException noTranslationEx) { - logger.log(Level.WARNING, "Could not successfully translate file name " + originalFileName, noTranslationEx); - } - - /** - * Do nothing on a no service provider exception except log, in this implemention we - * are only calling for translation to be done if we know that a TextTranslator - * service provider is already defined. - * - * @param ex - */ - @Override - public void onNoServiceProviderException(NoServiceProviderException noServiceEx) { - logger.log(Level.WARNING, "Translate unsuccessful because no TextTranslator " - + "implementation was provided.", noServiceEx); - } + @Deprecated + static public void fillPropertyMap(Map map, AbstractFile content) { + map.put(NAME.toString(), getContentDisplayName(content)); + map.put(LOCATION.toString(), getContentPath(content)); + map.put(MOD_TIME.toString(), ContentUtils.getStringTime(content.getMtime(), content)); + map.put(CHANGED_TIME.toString(), ContentUtils.getStringTime(content.getCtime(), content)); + map.put(ACCESS_TIME.toString(), ContentUtils.getStringTime(content.getAtime(), content)); + map.put(CREATED_TIME.toString(), ContentUtils.getStringTime(content.getCrtime(), content)); + map.put(SIZE.toString(), content.getSize()); + map.put(FLAGS_DIR.toString(), content.getDirFlagAsString()); + map.put(FLAGS_META.toString(), content.getMetaFlagsAsString()); + map.put(MODE.toString(), content.getModesAsString()); + map.put(USER_ID.toString(), content.getUid()); + map.put(GROUP_ID.toString(), content.getGid()); + map.put(META_ADDR.toString(), content.getMetaAddr()); + map.put(ATTR_ADDR.toString(), content.getAttrType().getValue() + "-" + content.getAttributeId()); + map.put(TYPE_DIR.toString(), content.getDirType().getLabel()); + map.put(TYPE_META.toString(), content.getMetaType().toString()); + map.put(KNOWN.toString(), content.getKnown().getName()); + map.put(MD5HASH.toString(), StringUtils.defaultString(content.getMd5Hash())); + map.put(ObjectID.toString(), content.getId()); + map.put(MIMETYPE.toString(), StringUtils.defaultString(content.getMIMEType())); + map.put(EXTENSION.toString(), content.getNameExtension()); } } diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/FileProperty.java b/Core/src/org/sleuthkit/autopsy/datamodel/FileProperty.java index 113e252d2f..c5f43f913e 100755 --- a/Core/src/org/sleuthkit/autopsy/datamodel/FileProperty.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/FileProperty.java @@ -5,36 +5,19 @@ */ package org.sleuthkit.autopsy.datamodel; -import org.sleuthkit.datamodel.AbstractFile; - /** * * @author dsmyda */ //No modifier means its is only package usable (not part of public api) -interface FileProperty { - /** - * - * @param content - * @return - */ - Object getPropertyValue(AbstractFile content); +abstract class FileProperty { - /** - * - * @return - */ - default String getPropertyName(){ - return this.toString(); - } + private String description = ""; + private final String PROPERTY_NAME; - /** - * - * @return - */ - default boolean isDisabled() { - return false; + public FileProperty(String propertyName) { + PROPERTY_NAME = propertyName; } /** @@ -42,7 +25,36 @@ interface FileProperty { * @param content * @return */ - default String getDescription(AbstractFile content) { - return ""; + public abstract Object getPropertyValue(); + + /** + * + * @return + */ + public String getPropertyName(){ + return PROPERTY_NAME; + } + /** + * + * @return + */ + public boolean isEnabled() { + return true; + } + + /* + * + */ + protected void setDescription(String description) { + this.description = description; + } + + /** + * + * @param content + * @return + */ + public String getDescription() { + return description; } } diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/LayoutFileNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/LayoutFileNode.java index 1bea2cb25e..555d61d77d 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/LayoutFileNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/LayoutFileNode.java @@ -106,10 +106,6 @@ public class LayoutFileNode extends AbstractAbstractFileNode { return actionsList.toArray(new Action[actionsList.size()]); } - void fillPropertyMap(Map map) { - AbstractAbstractFileNode.fillPropertyMap(map, getContent()); - } - @Override public String getItemType() { return getClass().getName(); diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/LocalDirectoryNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/LocalDirectoryNode.java index 37f1d1a136..3e72ae92aa 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/LocalDirectoryNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/LocalDirectoryNode.java @@ -37,60 +37,6 @@ public class LocalDirectoryNode extends SpecialDirectoryNode { } - @Override -<<<<<<< HEAD -======= - @NbBundle.Messages({ - "LocalDirectoryNode.createSheet.name.name=Name", - "LocalDirectoryNode.createSheet.name.displayName=Name", - "LocalDirectoryNode.createSheet.name.desc=no description", - "LocalDirectoryNode.createSheet.noDesc=no description"}) - protected Sheet createSheet() { - Sheet sheet = super.createSheet(); - Sheet.Set sheetSet = sheet.get(Sheet.PROPERTIES); - if (sheetSet == null) { - sheetSet = Sheet.createPropertiesSet(); - sheet.put(sheetSet); - } - - List tags = getContentTagsFromDatabase(); - sheetSet.put(new NodeProperty<>(Bundle.LocalDirectoryNode_createSheet_name_name(), - Bundle.LocalDirectoryNode_createSheet_name_displayName(), - Bundle.LocalDirectoryNode_createSheet_name_desc(), - getName())); - - final String NO_DESCR = Bundle.LocalDirectoryNode_createSheet_noDesc(); - if(UserPreferences.displayTranslationFileNames()) { - String translation = getTranslatedFileName(); - sheetSet.put(new NodeProperty<>(Bundle.AbstractAbstractFileNode_translateFileName(), - Bundle.AbstractAbstractFileNode_translateFileName(), NO_DESCR, translation)); - } - - addScoreProperty(sheetSet, tags); - - CorrelationAttributeInstance correlationAttribute = null; - if (EamDbUtil.useCentralRepo() && UserPreferences.hideCentralRepoCommentsAndOccurrences() == false) { - correlationAttribute = getCorrelationAttributeInstance(); - } - addCommentProperty(sheetSet, tags, correlationAttribute); - - if (EamDbUtil.useCentralRepo() && UserPreferences.hideCentralRepoCommentsAndOccurrences() == false) { - addCountProperty(sheetSet, correlationAttribute); - } - // At present, a LocalDirectory will never be a datasource - the top level of a logical - // file set is a VirtualDirectory - Map map = new LinkedHashMap<>(); - fillPropertyMap(map, getContent()); - - for (Map.Entry entry : map.entrySet()) { - sheetSet.put(new NodeProperty<>(entry.getKey(), entry.getKey(), NO_DESCR, entry.getValue())); - } - - return sheet; - } - - @Override ->>>>>>> 4316-text-translation-service public T accept(ContentNodeVisitor visitor) { return visitor.visit(this); } diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/VirtualDirectoryNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/VirtualDirectoryNode.java index 01479a6082..0c3bf4b48a 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/VirtualDirectoryNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/VirtualDirectoryNode.java @@ -73,33 +73,18 @@ public class VirtualDirectoryNode extends SpecialDirectoryNode { "VirtualDirectoryNode.createSheet.deviceId.displayName=Device ID", "VirtualDirectoryNode.createSheet.deviceId.desc=Device ID of the image"}) protected Sheet createSheet() { - Sheet sheet = super.createSheet(); - Sheet.Set sheetSet = sheet.get(Sheet.PROPERTIES); - if (sheetSet == null) { - sheetSet = Sheet.createPropertiesSet(); + //Do a special strategy for virtual directories.. + if(this.content.isDataSource()){ + Sheet sheet = getBlankSheet(); + Sheet.Set sheetSet = Sheet.createPropertiesSet(); sheet.put(sheetSet); - } - - sheetSet.put(new NodeProperty<>(NbBundle.getMessage(this.getClass(), "VirtualDirectoryNode.createSheet.name.name"), + + sheetSet.put(new NodeProperty<>(NbBundle.getMessage(this.getClass(), "VirtualDirectoryNode.createSheet.name.name"), NbBundle.getMessage(this.getClass(), "VirtualDirectoryNode.createSheet.name.displayName"), NbBundle.getMessage(this.getClass(), "VirtualDirectoryNode.createSheet.name.desc"), getName())); - if (!this.content.isDataSource()) { - Map map = new LinkedHashMap<>(); - fillPropertyMap(map, getContent()); - - for (Map.Entry entry : map.entrySet()) { - String desc = NbBundle.getMessage(this.getClass(), "VirtualDirectoryNode.createSheet.noDesc"); - FileProperty p = AbstractFilePropertyType.getPropertyFromDisplayName(entry.getKey()); - if(!p.getDescription(content).isEmpty()) { - desc = p.getDescription(content); - } - sheetSet.put(new NodeProperty<>(entry.getKey(), entry.getKey(), desc, entry.getValue())); - } - - } else { sheetSet.put(new NodeProperty<>(Bundle.VirtualDirectoryNode_createSheet_type_name(), Bundle.VirtualDirectoryNode_createSheet_type_displayName(), Bundle.VirtualDirectoryNode_createSheet_type_desc(), @@ -133,7 +118,8 @@ public class VirtualDirectoryNode extends SpecialDirectoryNode { } - return sheet; + //Otherwise default to the AAFN createSheet method. + return super.createSheet(); } @Override diff --git a/Core/src/org/sleuthkit/autopsy/texttranslation/TextTranslationService.java b/Core/src/org/sleuthkit/autopsy/texttranslation/TextTranslationService.java index eff494ac8f..9e9432248b 100755 --- a/Core/src/org/sleuthkit/autopsy/texttranslation/TextTranslationService.java +++ b/Core/src/org/sleuthkit/autopsy/texttranslation/TextTranslationService.java @@ -18,11 +18,7 @@ */ package org.sleuthkit.autopsy.texttranslation; -import com.google.common.util.concurrent.ThreadFactoryBuilder; import java.util.Optional; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.ThreadFactory; import org.openide.util.Lookup; @@ -35,9 +31,6 @@ public final class TextTranslationService { private final static TextTranslationService tts = new TextTranslationService(); private final Optional translator; - - private static ExecutorService pool; - private final static Integer MAX_POOL_SIZE = 10; private TextTranslationService(){ //Perform look up for Text Translation implementations ONLY ONCE during @@ -72,45 +65,6 @@ public final class TextTranslationService { "Could not find a TextTranslator service provider"); } - /** - * Makes the call to translate(String) happen asynchronously on a background - * thread. When it is done, it will use the appropriate TranslationCallback - * method. - * - * @param input String to be translated - * @param tcb Interface for handling the translation result or any - * exceptions thrown while running translate. - * - */ - public void translateAsynchronously(String input, TranslationCallback tcb) { - if (translator.isPresent()) { - //Delay thread pool initialization until an asynchronous task is first called. - //That way we don't have threads sitting parked in the background for no reason. - if (pool == null) { - ThreadFactory translationFactory = new ThreadFactoryBuilder() - .setNameFormat("translation-thread-%d") - .build(); - pool = Executors.newFixedThreadPool(MAX_POOL_SIZE, translationFactory); - } - - //Submit the task to the pool, calling the appropriate method depending - //on the result of the translate operation. - pool.submit(() -> { - try { - String translation = translate(input); - tcb.onTranslationResult(translation); - } catch (NoServiceProviderException ex) { - tcb.onNoServiceProviderException(ex); - } catch (TranslationException ex) { - tcb.onTranslationException(ex); - } - }); - } - - tcb.onNoServiceProviderException(new NoServiceProviderException( - "Could not find a TextTranslator service provider")); - } - /** * Returns if a TextTranslator lookup successfully found an implementing * class. diff --git a/Core/src/org/sleuthkit/autopsy/texttranslation/TranslationCallback.java b/Core/src/org/sleuthkit/autopsy/texttranslation/TranslationCallback.java deleted file mode 100755 index 8144f6fa1f..0000000000 --- a/Core/src/org/sleuthkit/autopsy/texttranslation/TranslationCallback.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Autopsy Forensic Browser - * - * Copyright 2018-2018 Basis Technology Corp. - * Contact: carrier sleuthkit org - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.sleuthkit.autopsy.texttranslation; - -/** - * This callback interface will be used by TextTranslationService when doing - * translations on background tasks. When the translation is done, it's result will - * be delegated to one of the following methods. It can either be successful or fail with - * exceptions. - */ -public interface TranslationCallback { - - /** - * Provides a method to handle the translation result - * - * @param translation result of calling TextTranslationService. - */ - void onTranslationResult(String translation); - - /** - * Provides a way to handle Translation Exceptions. - * - * @param noTranslationEx - */ - void onTranslationException(TranslationException noTranslationEx); - - /** - * Provides a way to handle NoServiceProviderExceptions. - * - * @param noServiceEx - */ - void onNoServiceProviderException(NoServiceProviderException noServiceEx); -} From 2dc3021e1a8b653817920c9bc49e14bc0167f1f2 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Wed, 31 Oct 2018 12:41:57 -0400 Subject: [PATCH 042/129] 4305 flag previously tagged as notable changed to being global setting --- .../datamodel/EamDbUtil.java | 59 ++++++++--- .../eventlisteners/IngestEventsListener.java | 13 ++- .../ingestmodule/IngestModule.java | 11 +-- .../ingestmodule/IngestModuleFactory.java | 39 +------- .../ingestmodule/IngestSettings.java | 71 -------------- .../ingestmodule/IngestSettingsPanel.form | 63 ------------ .../ingestmodule/IngestSettingsPanel.java | 98 ------------------- .../optionspanel/Bundle.properties | 3 +- .../optionspanel/GlobalSettingsPanel.form | 41 +++++--- .../optionspanel/GlobalSettingsPanel.java | 35 ++++--- 10 files changed, 111 insertions(+), 322 deletions(-) delete mode 100755 Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettings.java delete mode 100755 Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.form delete mode 100755 Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.java diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDbUtil.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDbUtil.java index e30ed174eb..db6687222a 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDbUtil.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDbUtil.java @@ -39,6 +39,7 @@ public class EamDbUtil { private final static Logger LOGGER = Logger.getLogger(EamDbUtil.class.getName()); private static final String CENTRAL_REPO_NAME = "CentralRepository"; private static final String CENTRAL_REPO_USE_KEY = "db.useCentralRepo"; + private static final String CENTRAL_REPO_FLAG_ITEMS_KEY = "db.flagNotableItems"; private static final String DEFAULT_ORG_NAME = "Not Specified"; /** @@ -192,9 +193,9 @@ public class EamDbUtil { } /** - * Upgrade the current central reposity to the newest version. If the upgrade - * fails, the central repository will be disabled and the current settings - * will be cleared. + * Upgrade the current central reposity to the newest version. If the + * upgrade fails, the central repository will be disabled and the current + * settings will be cleared. * * @return true if the upgrade succeeds, false otherwise. */ @@ -202,7 +203,7 @@ public class EamDbUtil { if (!EamDb.isEnabled()) { return true; } - + CoordinationService.Lock lock = null; try { EamDb db = EamDb.getInstance(); @@ -218,23 +219,23 @@ public class EamDbUtil { LOGGER.log(Level.SEVERE, "Error updating central repository", ex); // Disable the central repo and clear the current settings. - try{ + try { if (null != EamDb.getInstance()) { EamDb.getInstance().shutdownConnections(); } - } catch (EamDbException ex2){ + } catch (EamDbException ex2) { LOGGER.log(Level.SEVERE, "Error shutting down central repo connection pool", ex); - } + } setUseCentralRepo(false); EamDbPlatformEnum.setSelectedPlatform(EamDbPlatformEnum.DISABLED.name()); EamDbPlatformEnum.saveSelectedPlatform(); - + return false; } finally { - if(lock != null){ - try{ + if (lock != null) { + try { lock.release(); - } catch (CoordinationServiceException ex){ + } catch (CoordinationServiceException ex) { LOGGER.log(Level.SEVERE, "Error releasing database lock", ex); } } @@ -255,6 +256,7 @@ public class EamDbUtil { * Check whether the given org is the default organization. * * @param org + * * @return true if it is the default org, false otherwise */ public static boolean isDefaultOrg(EamOrganization org) { @@ -265,6 +267,7 @@ public class EamDbUtil { * Add the default organization to the database * * @param conn + * * @return true if successful, false otherwise */ static boolean insertDefaultOrganization(Connection conn) { @@ -295,7 +298,7 @@ public class EamDbUtil { * If the Central Repos use has been enabled. * * @return true if the Central Repo may be configured, false if it should - * not be able to be + * not be able to be */ public static boolean useCentralRepo() { return Boolean.parseBoolean(ModuleSettings.getConfigSetting(CENTRAL_REPO_NAME, CENTRAL_REPO_USE_KEY)); @@ -306,12 +309,40 @@ public class EamDbUtil { * configured. * * @param centralRepoCheckBoxIsSelected - true if the central repo can be - * used + * used */ public static void setUseCentralRepo(boolean centralRepoCheckBoxIsSelected) { ModuleSettings.setConfigSetting(CENTRAL_REPO_NAME, CENTRAL_REPO_USE_KEY, Boolean.toString(centralRepoCheckBoxIsSelected)); } + /** + * If items previously tagged as notable should be flagged after adding them + * to the Central Repo. + * + * @return true if the previously tagged as notable items should be flagged, + * false if previously tagged as notable items should not be flagged + */ + public static boolean flagNotableItems() { + String flagNotableItems = ModuleSettings.getConfigSetting(CENTRAL_REPO_NAME, CENTRAL_REPO_FLAG_ITEMS_KEY); + if (flagNotableItems == null){ //getConfigSetting can return null if the setting did not exist + setFlagNotableItems(true); + return true; + } + return Boolean.parseBoolean(flagNotableItems); + } + + /** + * Saves the setting for whether items previously tagged as notable should + * be flagged after adding them to the Central Repo. + * + * @param flagNotableItems - true if the previously tagged as notable items + * should be flagged, false if previously tagged as + * notable items should not be flagged + */ + public static void setFlagNotableItems(boolean flagNotableItems) { + ModuleSettings.setConfigSetting(CENTRAL_REPO_NAME, CENTRAL_REPO_FLAG_ITEMS_KEY, Boolean.toString(flagNotableItems)); + } + /** * Use the current settings and the validation query to test the connection * to the database. @@ -365,7 +396,7 @@ public class EamDbUtil { * Close the prepared statement. * * @param preparedStatement The prepared statement to be closed. - * + * * @deprecated Use closeStatement() instead. * * @throws EamDbException diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java index cdbf490e9e..7cdf06f82c 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java @@ -47,6 +47,7 @@ import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardAttribute; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.autopsy.centralrepository.datamodel.EamDb; +import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbUtil; import org.sleuthkit.autopsy.coreutils.ThreadUtils; /** @@ -183,9 +184,9 @@ public class IngestEventsListener { @Override public void propertyChange(PropertyChangeEvent evt) { - //if ingest is running we want there to check if there is a Correlation Engine module running - //sometimes artifacts are generated by DSPs or other sources while ingest is not running - //in these cases we still want to create correlation attributes for those artifacts when appropriate + //if ingest is running we want there to check if there is a Correlation Engine module running + //sometimes artifacts are generated by DSPs or other sources while ingest is not running + //in these cases we still want to create correlation attributes for those artifacts when appropriate if (!IngestManager.getInstance().isIngestRunning() || getCeModuleInstanceCount() > 0) { EamDb dbManager; try { @@ -196,11 +197,13 @@ public class IngestEventsListener { } switch (IngestManager.IngestModuleEvent.valueOf(evt.getPropertyName())) { case DATA_ADDED: { - jobProcessingExecutor.submit(new DataAddedTask(dbManager, evt, isFlagNotableItems())); + //if ingest isn't running read the flag notable items global setting otherwise use the value that existed when ingest was started + boolean flagNotable = IngestManager.getInstance().isIngestRunning() ? isFlagNotableItems() : EamDbUtil.flagNotableItems(); + jobProcessingExecutor.submit(new DataAddedTask(dbManager, evt, flagNotable)); break; } } - } + } } } diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java index 21baf59454..e091902b85 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java @@ -42,6 +42,7 @@ import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationDataSource; import org.sleuthkit.autopsy.centralrepository.datamodel.EamDb; import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbPlatformEnum; import org.sleuthkit.autopsy.centralrepository.datamodel.EamArtifactUtil; +import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbUtil; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardAttribute; @@ -71,7 +72,6 @@ final class IngestModule implements FileIngestModule { private CorrelationDataSource eamDataSource; private Blackboard blackboard; private CorrelationAttributeInstance.Type filesType; - private final boolean flagTaggedNotableItems; /** @@ -79,8 +79,8 @@ final class IngestModule implements FileIngestModule { * * @param settings The ingest settings for the module instance. */ - IngestModule(IngestSettings settings) { - flagTaggedNotableItems = settings.isFlagTaggedNotableItems(); + IngestModule() { + flagTaggedNotableItems = EamDbUtil.flagNotableItems(); } @Override @@ -214,9 +214,8 @@ final class IngestModule implements FileIngestModule { /* * Tell the IngestEventsListener to flag notable items based on the - * current module's configuration. This is a work around for the lack of - * an artifacts pipeline. Note that this can be changed by another - * module instance. All modules are affected by the value. While not + * global setting. This is a work around for the lack of + * an artifacts pipeline. All modules are affected by the value. While not * ideal, this will be good enough until a better solution can be * posited. * diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModuleFactory.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModuleFactory.java index fc5dcd9c81..8d6068a408 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModuleFactory.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModuleFactory.java @@ -26,8 +26,6 @@ import org.sleuthkit.autopsy.ingest.IngestModuleGlobalSettingsPanel; import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettings; import org.sleuthkit.autopsy.centralrepository.optionspanel.GlobalSettingsPanel; import org.sleuthkit.autopsy.coreutils.Version; -import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettingsPanel; -import org.sleuthkit.autopsy.ingest.NoIngestModuleIngestJobSettings; /** * Factory for Central Repository ingest modules @@ -68,17 +66,7 @@ public class IngestModuleFactory extends IngestModuleFactoryAdapter { @Override public FileIngestModule createFileIngestModule(IngestModuleIngestJobSettings settings) { - if (settings instanceof IngestSettings) { - return new IngestModule((IngestSettings) settings); - } - /* - * Compatibility check for older versions. - */ - if (settings instanceof NoIngestModuleIngestJobSettings) { - return new IngestModule(new IngestSettings()); - } - - throw new IllegalArgumentException("Expected settings argument to be an instance of IngestSettings"); + return new IngestModule(); } @Override @@ -92,30 +80,5 @@ public class IngestModuleFactory extends IngestModuleFactoryAdapter { globalOptionsPanel.load(); return globalOptionsPanel; } - - @Override - public IngestModuleIngestJobSettings getDefaultIngestJobSettings() { - return new IngestSettings(); - } - - @Override - public boolean hasIngestJobSettingsPanel() { - return true; - } - - @Override - public IngestModuleIngestJobSettingsPanel getIngestJobSettingsPanel(IngestModuleIngestJobSettings settings) { - if (settings instanceof IngestSettings) { - return new IngestSettingsPanel((IngestSettings) settings); - } - /* - * Compatibility check for older versions. - */ - if (settings instanceof NoIngestModuleIngestJobSettings) { - return new IngestSettingsPanel(new IngestSettings()); - } - - throw new IllegalArgumentException("Expected settings argument to be an instance of IngestSettings"); - } } diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettings.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettings.java deleted file mode 100755 index 32ab9e9f2d..0000000000 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettings.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Central Repository - * - * Copyright 2018 Basis Technology Corp. - * Contact: carrier sleuthkit org - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.sleuthkit.autopsy.centralrepository.ingestmodule; - -import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettings; - -/** - * Ingest job settings for the Correlation Engine module. - */ -final class IngestSettings implements IngestModuleIngestJobSettings { - - private static final long serialVersionUID = 1L; - - private boolean flagTaggedNotableItems; - - /** - * Instantiate the ingest job settings with default values. - */ - IngestSettings() { - this.flagTaggedNotableItems = IngestModule.DEFAULT_FLAG_TAGGED_NOTABLE_ITEMS; - } - - /** - * Instantiate the ingest job settings. - * - * @param flagTaggedNotableItems Flag previously tagged notable items. - */ - IngestSettings(boolean flagTaggedNotableItems) { - this.flagTaggedNotableItems = flagTaggedNotableItems; - } - - @Override - public long getVersionNumber() { - return serialVersionUID; - } - - /** - * Are previously tagged notable items to be flagged? - * - * @return True if flagging; otherwise false. - */ - boolean isFlagTaggedNotableItems() { - return flagTaggedNotableItems; - } - - /** - * Flag or ignore previously identified notable items. - * - * @param ignorePreviousNotableItems Are previously tagged notable items to - * be flagged? - */ - void setFlagTaggedNotableItems(boolean flagTaggedNotableItems) { - this.flagTaggedNotableItems = flagTaggedNotableItems; - } -} diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.form b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.form deleted file mode 100755 index 564031cb72..0000000000 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.form +++ /dev/null @@ -1,63 +0,0 @@ - - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.java deleted file mode 100755 index ed36c71287..0000000000 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Central Repository - * - * Copyright 2018 Basis Technology Corp. - * Contact: carrier sleuthkit org - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.sleuthkit.autopsy.centralrepository.ingestmodule; - -import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettings; -import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettingsPanel; - -/** - * Ingest job settings panel for the Correlation Engine module. - */ -@SuppressWarnings("PMD.SingularField") // UI widgets cause lots of false positives -final class IngestSettingsPanel extends IngestModuleIngestJobSettingsPanel { - - /** - * Creates new form IngestSettingsPanel - */ - public IngestSettingsPanel(IngestSettings settings) { - initComponents(); - customizeComponents(settings); - } - - /** - * Update components with values from the ingest job settings. - * - * @param settings The ingest job settings. - */ - private void customizeComponents(IngestSettings settings) { - flagTaggedNotableItemsCheckbox.setSelected(settings.isFlagTaggedNotableItems()); - } - - @Override - public IngestModuleIngestJobSettings getSettings() { - return new IngestSettings(flagTaggedNotableItemsCheckbox.isSelected()); - } - - /** - * This method is called from within the constructor to initialize the form. - * WARNING: Do NOT modify this code. The content of this method is always - * regenerated by the Form Editor. - */ - @SuppressWarnings("unchecked") - // //GEN-BEGIN:initComponents - private void initComponents() { - - ingestSettingsLabel = new javax.swing.JLabel(); - flagTaggedNotableItemsCheckbox = new javax.swing.JCheckBox(); - - ingestSettingsLabel.setFont(new java.awt.Font("Tahoma", 1, 11)); // NOI18N - org.openide.awt.Mnemonics.setLocalizedText(ingestSettingsLabel, org.openide.util.NbBundle.getMessage(IngestSettingsPanel.class, "IngestSettingsPanel.ingestSettingsLabel.text")); // NOI18N - - org.openide.awt.Mnemonics.setLocalizedText(flagTaggedNotableItemsCheckbox, org.openide.util.NbBundle.getMessage(IngestSettingsPanel.class, "IngestSettingsPanel.flagTaggedNotableItemsCheckbox.text")); // NOI18N - - javax.swing.GroupLayout layout = new javax.swing.GroupLayout(this); - this.setLayout(layout); - layout.setHorizontalGroup( - layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addGroup(layout.createSequentialGroup() - .addContainerGap() - .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addGroup(layout.createSequentialGroup() - .addGap(10, 10, 10) - .addComponent(flagTaggedNotableItemsCheckbox)) - .addComponent(ingestSettingsLabel)) - .addContainerGap(65, Short.MAX_VALUE)) - ); - layout.setVerticalGroup( - layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addGroup(layout.createSequentialGroup() - .addContainerGap() - .addComponent(ingestSettingsLabel) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) - .addComponent(flagTaggedNotableItemsCheckbox) - .addContainerGap(245, Short.MAX_VALUE)) - ); - }// //GEN-END:initComponents - - // Variables declaration - do not modify//GEN-BEGIN:variables - private javax.swing.JCheckBox flagTaggedNotableItemsCheckbox; - private javax.swing.JLabel ingestSettingsLabel; - // End of variables declaration//GEN-END:variables - -} diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/Bundle.properties b/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/Bundle.properties index b45b6f7579..a5bbe04992 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/Bundle.properties @@ -59,7 +59,6 @@ GlobalSettingsPanel.bnManageProperties.text=Manage Correlation Properties EamDbSettingsDialog.lbDatabaseDesc.text=Database File: EamDbSettingsDialog.lbFullDbPath.text= GlobalSettingsPanel.cbUseCentralRepo.text=Use a central repository -GlobalSettingsPanel.correlationPropertiesTextArea.text=Choose which file and result properties to store in the central repository for later correlation.\n GlobalSettingsPanel.organizationTextArea.text=Organization information can be tracked in the central repository. GlobalSettingsPanel.manageOrganizationButton.text=Manage Organizations GlobalSettingsPanel.lbCentralRepository.text=A central repository allows you to correlate files and results between cases. @@ -76,3 +75,5 @@ GlobalSettingsPanel.Case\ Details.AccessibleContext.accessibleName=Cases Details ShowCasesDialog.caseDetailsTable.AccessibleContext.accessibleDescription=Click column name to sort. GlobalSettingsPanel.casesTextArea.text=Display table that lists central repository case details. GlobalSettingsPanel.ingestRunningWarningLabel.text=Cannot make changes to central repository settings when ingest is running! +GlobalSettingsPanel.flagTaggedNotableItemsCheckbox.text=Flag items previously tagged as notable +GlobalSettingsPanel.correlationPropertiesTextArea.text=Choose which file and result properties to store in the central repository for later correlation.\n diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/GlobalSettingsPanel.form b/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/GlobalSettingsPanel.form index c3a8f678d6..cb9f3c1b5a 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/GlobalSettingsPanel.form +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/GlobalSettingsPanel.form @@ -89,13 +89,13 @@
- + - + - + - + @@ -242,24 +242,30 @@ - + - - - - + + + + + - + - - + + + + + + + @@ -301,7 +307,7 @@ - + @@ -314,6 +320,13 @@ + + + + + + + @@ -334,7 +347,7 @@ - + diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/GlobalSettingsPanel.java b/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/GlobalSettingsPanel.java index b8c42eb66d..356c5475e6 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/GlobalSettingsPanel.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/GlobalSettingsPanel.java @@ -120,6 +120,7 @@ public final class GlobalSettingsPanel extends IngestModuleGlobalSettingsPanel i bnManageTypes = new javax.swing.JButton(); correlationPropertiesScrollPane = new javax.swing.JScrollPane(); correlationPropertiesTextArea = new javax.swing.JTextArea(); + flagTaggedNotableItemsCheckbox = new javax.swing.JCheckBox(); organizationPanel = new javax.swing.JPanel(); manageOrganizationButton = new javax.swing.JButton(); organizationScrollPane = new javax.swing.JScrollPane(); @@ -222,13 +223,15 @@ public final class GlobalSettingsPanel extends IngestModuleGlobalSettingsPanel i correlationPropertiesTextArea.setColumns(20); correlationPropertiesTextArea.setFont(new java.awt.Font("Tahoma", 0, 11)); // NOI18N correlationPropertiesTextArea.setLineWrap(true); - correlationPropertiesTextArea.setRows(2); + correlationPropertiesTextArea.setRows(1); correlationPropertiesTextArea.setText(org.openide.util.NbBundle.getMessage(GlobalSettingsPanel.class, "GlobalSettingsPanel.correlationPropertiesTextArea.text")); // NOI18N correlationPropertiesTextArea.setToolTipText(""); correlationPropertiesTextArea.setWrapStyleWord(true); correlationPropertiesTextArea.setBorder(null); correlationPropertiesScrollPane.setViewportView(correlationPropertiesTextArea); + org.openide.awt.Mnemonics.setLocalizedText(flagTaggedNotableItemsCheckbox, org.openide.util.NbBundle.getMessage(GlobalSettingsPanel.class, "GlobalSettingsPanel.flagTaggedNotableItemsCheckbox.text")); // NOI18N + javax.swing.GroupLayout pnCorrelationPropertiesLayout = new javax.swing.GroupLayout(pnCorrelationProperties); pnCorrelationProperties.setLayout(pnCorrelationPropertiesLayout); pnCorrelationPropertiesLayout.setHorizontalGroup( @@ -236,17 +239,21 @@ public final class GlobalSettingsPanel extends IngestModuleGlobalSettingsPanel i .addGroup(pnCorrelationPropertiesLayout.createSequentialGroup() .addContainerGap() .addGroup(pnCorrelationPropertiesLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addComponent(correlationPropertiesScrollPane) .addGroup(pnCorrelationPropertiesLayout.createSequentialGroup() - .addComponent(bnManageTypes) - .addGap(0, 0, Short.MAX_VALUE))) - .addContainerGap()) + .addComponent(correlationPropertiesScrollPane, javax.swing.GroupLayout.PREFERRED_SIZE, 497, javax.swing.GroupLayout.PREFERRED_SIZE) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addComponent(flagTaggedNotableItemsCheckbox)) + .addComponent(bnManageTypes)) + .addContainerGap(javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)) ); pnCorrelationPropertiesLayout.setVerticalGroup( pnCorrelationPropertiesLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addGroup(javax.swing.GroupLayout.Alignment.TRAILING, pnCorrelationPropertiesLayout.createSequentialGroup() - .addGap(7, 7, 7) - .addComponent(correlationPropertiesScrollPane, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE) + .addGroup(pnCorrelationPropertiesLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addGroup(pnCorrelationPropertiesLayout.createSequentialGroup() + .addContainerGap() + .addComponent(flagTaggedNotableItemsCheckbox)) + .addComponent(correlationPropertiesScrollPane, javax.swing.GroupLayout.DEFAULT_SIZE, 32, Short.MAX_VALUE)) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) .addComponent(bnManageTypes) .addGap(8, 8, 8)) @@ -281,7 +288,7 @@ public final class GlobalSettingsPanel extends IngestModuleGlobalSettingsPanel i .addGroup(organizationPanelLayout.createSequentialGroup() .addContainerGap() .addGroup(organizationPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addComponent(organizationScrollPane, javax.swing.GroupLayout.DEFAULT_SIZE, 992, Short.MAX_VALUE) + .addComponent(organizationScrollPane, javax.swing.GroupLayout.DEFAULT_SIZE, 980, Short.MAX_VALUE) .addGroup(organizationPanelLayout.createSequentialGroup() .addComponent(manageOrganizationButton) .addGap(0, 0, Short.MAX_VALUE))) @@ -382,13 +389,13 @@ public final class GlobalSettingsPanel extends IngestModuleGlobalSettingsPanel i .addComponent(ingestRunningWarningLabel)) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) .addComponent(pnDatabaseConfiguration, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addGap(0, 0, 0) .addComponent(pnCorrelationProperties, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addGap(0, 0, 0) .addComponent(organizationPanel, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addGap(0, 0, 0) .addComponent(casesPanel, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addGap(0, 0, 0) .addComponent(tbOops, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE) .addContainerGap()) ); @@ -451,6 +458,7 @@ public final class GlobalSettingsPanel extends IngestModuleGlobalSettingsPanel i enableAllSubComponents(false); EamDbPlatformEnum selectedPlatform = EamDbPlatformEnum.getSelectedPlatform(); cbUseCentralRepo.setSelected(EamDbUtil.useCentralRepo()); // NON-NLS + flagTaggedNotableItemsCheckbox.setSelected(EamDbUtil.flagNotableItems()); switch (selectedPlatform) { case POSTGRESQL: PostgresEamDbSettings dbSettingsPg = new PostgresEamDbSettings(); @@ -480,6 +488,7 @@ public final class GlobalSettingsPanel extends IngestModuleGlobalSettingsPanel i @Override public void store() { // Click OK or Apply on Options Panel EamDbUtil.setUseCentralRepo(cbUseCentralRepo.isSelected()); + EamDbUtil.setFlagNotableItems(flagTaggedNotableItemsCheckbox.isSelected()); } /** @@ -625,6 +634,7 @@ public final class GlobalSettingsPanel extends IngestModuleGlobalSettingsPanel i showCasesButton.setEnabled(enable && !ingestRunning); casesPanel.setEnabled(enable && !ingestRunning); casesTextArea.setEnabled(enable && !ingestRunning); + flagTaggedNotableItemsCheckbox.setEnabled(enable && !ingestRunning); return true; } @@ -637,6 +647,7 @@ public final class GlobalSettingsPanel extends IngestModuleGlobalSettingsPanel i private javax.swing.JCheckBox cbUseCentralRepo; private javax.swing.JScrollPane correlationPropertiesScrollPane; private javax.swing.JTextArea correlationPropertiesTextArea; + private javax.swing.JCheckBox flagTaggedNotableItemsCheckbox; private javax.swing.JLabel ingestRunningWarningLabel; private javax.swing.JPanel jPanel1; private javax.swing.JScrollPane jScrollPane1; From 5d2a52d86f14df4ab868d424349a03f27809f498 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Wed, 31 Oct 2018 12:49:13 -0400 Subject: [PATCH 043/129] Now testing and going to batch the SCO tasks into one pool afterwards --- .../datamodel/AbstractAbstractFileNode.java | 205 +++++++++++++----- .../autopsy/datamodel/FileProperty.java | 2 +- 2 files changed, 153 insertions(+), 54 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java index a60a7e3bce..f4d1756ce6 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java @@ -83,7 +83,7 @@ public abstract class AbstractAbstractFileNode extends A private static final Set CASE_EVENTS_OF_INTEREST = EnumSet.of(Case.Events.CURRENT_CASE, Case.Events.CONTENT_TAG_ADDED, Case.Events.CONTENT_TAG_DELETED, Case.Events.CR_COMMENT_CHANGED); - private volatile String translatedFileName; + private volatile List currentProperties; private static final ExecutorService pool; private static final Integer MAX_POOL_SIZE = 10; @@ -102,7 +102,6 @@ public abstract class AbstractAbstractFileNode extends A IngestManager.getInstance().addIngestModuleEventListener(weakPcl); } } - translatedFileName = null; // Listen for case events so that we can detect when the case is closed // or when tags are added. Case.addEventTypeSubscriber(CASE_EVENTS_OF_INTEREST, weakPcl); @@ -131,6 +130,14 @@ public abstract class AbstractAbstractFileNode extends A IngestManager.getInstance().removeIngestModuleEventListener(weakPcl); Case.removeEventTypeSubscriber(CASE_EVENTS_OF_INTEREST, weakPcl); } + + //Events signals to indicate the background tasks have completed processing. + private enum NodeSpecificEvents { + TRANSLATION_AVAILABLE_EVENT, + SCORE_AVAILABLE_EVENT, + COMMENT_AVAILABLE_EVENT, + OCCURRENCES_AVAILABLE_EVENT; + } private final PropertyChangeListener pcl = (PropertyChangeEvent evt) -> { String eventType = evt.getPropertyName(); @@ -183,9 +190,46 @@ public abstract class AbstractAbstractFileNode extends A if (event.getContentID() == content.getId()) { updateSheet(); } - } else if (eventType.equals(TRANSLATION_AVAILABLE_EVENT)) { - this.translatedFileName = (String) evt.getNewValue(); - updateSheet(); + } else if (eventType.equals(NodeSpecificEvents.TRANSLATION_AVAILABLE_EVENT.toString())) { + updateProperty(TRANSLATION.toString(), new FileProperty(TRANSLATION.toString()) { + @Override + public Object getPropertyValue() { + return evt.getNewValue(); + } + }); + } else if(eventType.equals(NodeSpecificEvents.SCORE_AVAILABLE_EVENT.toString())) { + Pair scoreAndDescription = (Pair) evt.getNewValue(); + updateProperty(SCORE.toString(), new FileProperty(SCORE.toString()) { + @Override + public Object getPropertyValue() { + return scoreAndDescription.getLeft(); + } + + @Override + public String getDescription() { + return scoreAndDescription.getRight(); + } + }); + } else if(eventType.equals(NodeSpecificEvents.COMMENT_AVAILABLE_EVENT.toString())) { + updateProperty(COMMENT.toString(), new FileProperty(COMMENT.toString()) { + @Override + public Object getPropertyValue() { + return evt.getNewValue(); + } + }); + } else if(eventType.equals(NodeSpecificEvents.OCCURRENCES_AVAILABLE_EVENT.toString())) { + Pair countAndDescription = (Pair) evt.getNewValue(); + updateProperty(OCCURRENCES.toString(), new FileProperty(OCCURRENCES.toString()) { + @Override + public Object getPropertyValue() { + return countAndDescription.getLeft(); + } + + @Override + public String getDescription() { + return countAndDescription.getRight(); + } + }); } }; @@ -202,8 +246,39 @@ public abstract class AbstractAbstractFileNode extends A Sheet getBlankSheet() { return super.createSheet(); } + + private synchronized void updateProperty(String propName, FileProperty newProp) { + for(int i = 0; i < currentProperties.size(); i++) { + FileProperty property = currentProperties.get(i); + if(property.getPropertyName().equals(propName)) { + currentProperties.set(i, newProp); + } + } + + Sheet sheet = super.createSheet(); + Sheet.Set sheetSet = Sheet.createPropertiesSet(); + sheet.put(sheetSet); + + for (FileProperty property : currentProperties) { + if (property.isEnabled()) { + sheetSet.put(new NodeProperty<>( + property.getPropertyName(), + property.getPropertyName(), + property.getDescription(), + property.getPropertyValue())); + } + } + + this.setSheet(sheet); + } - private void updateSheet() { + //Race condition if not synchronized. The main thread could be updating the + //sheet with blank properties, if a background task is complete, it may finish + //the updateSheet() call before the main thread. If that's the case, main thread's + //sheet will be the one to 'win' the UI and we will see stale data. Problem + //for multi-user cases if both the CR comment is updated from elsewhere and the background + //task is completed and also trying to update. + private synchronized void updateSheet() { this.setSheet(createSheet()); } @@ -213,10 +288,12 @@ public abstract class AbstractAbstractFileNode extends A Sheet.Set sheetSet = Sheet.createPropertiesSet(); sheet.put(sheetSet); - List properties = getProperties(); + //This will fire off fresh background tasks. + List newProperties = getProperties(); + currentProperties = newProperties; //Add only the enabled properties to the sheet! - for (FileProperty property : properties) { + for (FileProperty property : newProperties) { if (property.isEnabled()) { sheetSet.put(new NodeProperty<>( property.getPropertyName(), @@ -302,50 +379,38 @@ public abstract class AbstractAbstractFileNode extends A * * @return The file names translation. */ - private String getTranslatedFileName(AbstractFile content) { + private String getTranslatedFileName() { //If already in complete English, don't translate. if (content.getName().matches("^\\p{ASCII}+$")) { return NO_TRANSLATION; } - //If we already have a translation use that one. - if (translatedFileName != null) { - return translatedFileName; - } - - //If not, lets fire off a background translation that will update the UI + //Lets fire off a background translation that will update the UI //when it is done. TextTranslationService tts = TextTranslationService.getInstance(); if (tts.hasProvider()) { //Seperate out the base and ext from the contents file name. String base = FilenameUtils.getBaseName(content.getName()); - //Send only the base file name to be translated. Once the translation comes - //back fire a PropertyChangeEvent on this nodes PropertyChangeListener. - pool.submit(() -> { - try { - String translation = tts.translate(base); - String ext = FilenameUtils.getExtension(content.getName()); + try { + String translation = tts.translate(base); + String ext = FilenameUtils.getExtension(content.getName()); - //If we have no extension, then we shouldn't add the . - String extensionDelimiter = (ext.isEmpty()) ? "" : "."; + //If we have no extension, then we shouldn't add the . + String extensionDelimiter = (ext.isEmpty()) ? "" : "."; - //Talk directly to this nodes pcl, fire an update when the translation - //is complete. - if (!translation.isEmpty()) { - weakPcl.propertyChange(new PropertyChangeEvent( - AutopsyEvent.SourceType.LOCAL.toString(), - TRANSLATION_AVAILABLE_EVENT, null, - translation + extensionDelimiter + ext)); - } - } catch (NoServiceProviderException noServiceEx) { - logger.log(Level.WARNING, "Translate unsuccessful because no TextTranslator " - + "implementation was provided.", noServiceEx); - } catch (TranslationException noTranslationEx) { - logger.log(Level.WARNING, "Could not successfully translate file name " - + content.getName(), noTranslationEx); + //Talk directly to this nodes pcl, fire an update when the translation + //is complete. + if (!translation.isEmpty()) { + return translation + extensionDelimiter + ext; } - }); + } catch (NoServiceProviderException noServiceEx) { + logger.log(Level.WARNING, "Translate unsuccessful because no TextTranslator " + + "implementation was provided.", noServiceEx); + } catch (TranslationException noTranslationEx) { + logger.log(Level.WARNING, "Could not successfully translate file name " + + content.getName(), noTranslationEx); + } } //In the mean time, return a blank translation. @@ -353,8 +418,14 @@ public abstract class AbstractAbstractFileNode extends A } /** - * - * @return + * Creates a list of properties for this file node. Each property has it's own + * strategy for getting it's value, it's own description, name, and ability to be + * disabled. The FileProperty abstract class provides a wrapper for all + * of these characteristics. Additionally, with a return value of a list, any + * children classes of this node may reorder or omit any of these properties as + * they see fit for their use case. + * + * @return List of file properties associated with this file nodes content. */ List getProperties() { ArrayList properties = new ArrayList<>(); @@ -368,7 +439,14 @@ public abstract class AbstractAbstractFileNode extends A properties.add(new FileProperty(TRANSLATION.toString()) { @Override public Object getPropertyValue() { - return getTranslatedFileName(content); + pool.submit(() -> { + weakPcl.propertyChange(new PropertyChangeEvent( + AutopsyEvent.SourceType.LOCAL.toString(), + NodeSpecificEvents.TRANSLATION_AVAILABLE_EVENT.toString(), + null, + getTranslatedFileName())); + }); + return ""; } @Override @@ -382,15 +460,30 @@ public abstract class AbstractAbstractFileNode extends A properties.add(new FileProperty(SCORE.toString()) { @Override public Object getPropertyValue() { - Pair result = getScoreProperty(content, tags); - setDescription(result.getLeft()); - return result.getRight(); + pool.submit(() -> { + List tags = getContentTagsFromDatabase(content); + weakPcl.propertyChange(new PropertyChangeEvent( + AutopsyEvent.SourceType.LOCAL.toString(), + NodeSpecificEvents.SCORE_AVAILABLE_EVENT.toString(), + null, + getScorePropertyAndDescription(tags))); + }); + return ""; } }); properties.add(new FileProperty(COMMENT.toString()) { @Override public Object getPropertyValue() { - return getCommentProperty(tags, correlationAttribute); + pool.submit(() -> { + List tags = getContentTagsFromDatabase(content); + CorrelationAttributeInstance correlationAttribute = getCorrelationAttributeInstance(); + weakPcl.propertyChange(new PropertyChangeEvent( + AutopsyEvent.SourceType.LOCAL.toString(), + NodeSpecificEvents.COMMENT_AVAILABLE_EVENT.toString(), + null, + getCommentProperty(tags, correlationAttribute))); + }); + return ""; } @Override @@ -401,9 +494,15 @@ public abstract class AbstractAbstractFileNode extends A properties.add(new FileProperty(OCCURRENCES.toString()) { @Override public Object getPropertyValue() { - Pair result = getCountProperty(correlationAttribute); - setDescription(result.getLeft()); - return result.getRight(); + pool.submit(() -> { + CorrelationAttributeInstance correlationAttribute = getCorrelationAttributeInstance(); + weakPcl.propertyChange(new PropertyChangeEvent( + AutopsyEvent.SourceType.LOCAL.toString(), + NodeSpecificEvents.OCCURRENCES_AVAILABLE_EVENT.toString(), + null, + getCountPropertyAndDescription(correlationAttribute))); + }); + return ""; } @Override @@ -606,9 +705,9 @@ public abstract class AbstractAbstractFileNode extends A "AbstractAbstractFileNode.createSheet.taggedFile.description=File has been tagged.", "AbstractAbstractFileNode.createSheet.notableTaggedFile.description=File tagged with notable tag.", "AbstractAbstractFileNode.createSheet.noScore.description=No score"}) - private static Pair getScoreProperty(AbstractFile content, List tags) { + private Pair getScorePropertyAndDescription(List tags) { Score score = Score.NO_SCORE; - String description = Bundle.AbstractAbstractFileNode_createSheet_noScore_description(); + String description = ""; if (content.getKnown() == TskData.FileKnown.BAD) { score = Score.NOTABLE_SCORE; description = Bundle.AbstractAbstractFileNode_createSheet_notableFile_description(); @@ -632,7 +731,7 @@ public abstract class AbstractAbstractFileNode extends A } } } - return Pair.of(description, score); + return Pair.of(score, description); } @NbBundle.Messages({ @@ -641,7 +740,7 @@ public abstract class AbstractAbstractFileNode extends A "AbstractAbstractFileNode.createSheet.count.hashLookupNotRun.description=Hash lookup had not been run on this file when the column was populated", "# {0} - occuranceCount", "AbstractAbstractFileNode.createSheet.count.description=There were {0} datasource(s) found with occurances of the correlation value"}) - private static Pair getCountProperty(CorrelationAttributeInstance attribute) { + private static Pair getCountPropertyAndDescription(CorrelationAttributeInstance attribute) { Long count = -1L; //The column renderer will not display negative values, negative value used when count unavailble to preserve sorting String description = Bundle.AbstractAbstractFileNode_createSheet_count_noCentralRepo_description(); try { @@ -658,7 +757,7 @@ public abstract class AbstractAbstractFileNode extends A logger.log(Level.WARNING, "Unable to normalize data to get count of datasources with correlation attribute", ex); } - return Pair.of(description, count); + return Pair.of(count, description); } /** diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/FileProperty.java b/Core/src/org/sleuthkit/autopsy/datamodel/FileProperty.java index c5f43f913e..0755f1b29b 100755 --- a/Core/src/org/sleuthkit/autopsy/datamodel/FileProperty.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/FileProperty.java @@ -45,7 +45,7 @@ abstract class FileProperty { /* * */ - protected void setDescription(String description) { + public void setDescription(String description) { this.description = description; } From 650dbb963701cfd09b9ccc17ccac929576262b8d Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Wed, 31 Oct 2018 14:03:18 -0400 Subject: [PATCH 044/129] 4305 modify placement of flag previously notable setting, restore cr ingestsettings file --- .../ingestmodule/IngestSettings.java | 71 +++++++++++++++++++ .../optionspanel/Bundle.properties | 1 + .../optionspanel/GlobalSettingsPanel.form | 43 +++++++---- .../optionspanel/GlobalSettingsPanel.java | 34 +++++---- 4 files changed, 122 insertions(+), 27 deletions(-) create mode 100755 Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettings.java diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettings.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettings.java new file mode 100755 index 0000000000..32ab9e9f2d --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettings.java @@ -0,0 +1,71 @@ +/* + * Central Repository + * + * Copyright 2018 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.centralrepository.ingestmodule; + +import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettings; + +/** + * Ingest job settings for the Correlation Engine module. + */ +final class IngestSettings implements IngestModuleIngestJobSettings { + + private static final long serialVersionUID = 1L; + + private boolean flagTaggedNotableItems; + + /** + * Instantiate the ingest job settings with default values. + */ + IngestSettings() { + this.flagTaggedNotableItems = IngestModule.DEFAULT_FLAG_TAGGED_NOTABLE_ITEMS; + } + + /** + * Instantiate the ingest job settings. + * + * @param flagTaggedNotableItems Flag previously tagged notable items. + */ + IngestSettings(boolean flagTaggedNotableItems) { + this.flagTaggedNotableItems = flagTaggedNotableItems; + } + + @Override + public long getVersionNumber() { + return serialVersionUID; + } + + /** + * Are previously tagged notable items to be flagged? + * + * @return True if flagging; otherwise false. + */ + boolean isFlagTaggedNotableItems() { + return flagTaggedNotableItems; + } + + /** + * Flag or ignore previously identified notable items. + * + * @param ignorePreviousNotableItems Are previously tagged notable items to + * be flagged? + */ + void setFlagTaggedNotableItems(boolean flagTaggedNotableItems) { + this.flagTaggedNotableItems = flagTaggedNotableItems; + } +} diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/Bundle.properties b/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/Bundle.properties index a5bbe04992..88a2150bd6 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/Bundle.properties @@ -77,3 +77,4 @@ GlobalSettingsPanel.casesTextArea.text=Display table that lists central reposito GlobalSettingsPanel.ingestRunningWarningLabel.text=Cannot make changes to central repository settings when ingest is running! GlobalSettingsPanel.flagTaggedNotableItemsCheckbox.text=Flag items previously tagged as notable GlobalSettingsPanel.correlationPropertiesTextArea.text=Choose which file and result properties to store in the central repository for later correlation.\n +GlobalSettingsPanel.generateInterestingItemsLabel.text=Generate interesting items to: diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/GlobalSettingsPanel.form b/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/GlobalSettingsPanel.form index cb9f3c1b5a..24efccdc99 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/GlobalSettingsPanel.form +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/GlobalSettingsPanel.form @@ -57,7 +57,7 @@ - + @@ -243,33 +243,39 @@ - + + + + + - - - + + + + + + - - + - - - - - - - + + + + + + + @@ -327,6 +333,13 @@ + + + + + + + @@ -347,7 +360,7 @@ - + diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/GlobalSettingsPanel.java b/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/GlobalSettingsPanel.java index 356c5475e6..ed7fa1b1d0 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/GlobalSettingsPanel.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/GlobalSettingsPanel.java @@ -121,6 +121,7 @@ public final class GlobalSettingsPanel extends IngestModuleGlobalSettingsPanel i correlationPropertiesScrollPane = new javax.swing.JScrollPane(); correlationPropertiesTextArea = new javax.swing.JTextArea(); flagTaggedNotableItemsCheckbox = new javax.swing.JCheckBox(); + generateInterestingItemsLabel = new javax.swing.JLabel(); organizationPanel = new javax.swing.JPanel(); manageOrganizationButton = new javax.swing.JButton(); organizationScrollPane = new javax.swing.JScrollPane(); @@ -232,31 +233,38 @@ public final class GlobalSettingsPanel extends IngestModuleGlobalSettingsPanel i org.openide.awt.Mnemonics.setLocalizedText(flagTaggedNotableItemsCheckbox, org.openide.util.NbBundle.getMessage(GlobalSettingsPanel.class, "GlobalSettingsPanel.flagTaggedNotableItemsCheckbox.text")); // NOI18N + org.openide.awt.Mnemonics.setLocalizedText(generateInterestingItemsLabel, org.openide.util.NbBundle.getMessage(GlobalSettingsPanel.class, "GlobalSettingsPanel.generateInterestingItemsLabel.text")); // NOI18N + javax.swing.GroupLayout pnCorrelationPropertiesLayout = new javax.swing.GroupLayout(pnCorrelationProperties); pnCorrelationProperties.setLayout(pnCorrelationPropertiesLayout); pnCorrelationPropertiesLayout.setHorizontalGroup( pnCorrelationPropertiesLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addGroup(pnCorrelationPropertiesLayout.createSequentialGroup() .addContainerGap() + .addGroup(pnCorrelationPropertiesLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addComponent(bnManageTypes) + .addComponent(correlationPropertiesScrollPane, javax.swing.GroupLayout.PREFERRED_SIZE, 468, javax.swing.GroupLayout.PREFERRED_SIZE)) .addGroup(pnCorrelationPropertiesLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addGroup(pnCorrelationPropertiesLayout.createSequentialGroup() - .addComponent(correlationPropertiesScrollPane, javax.swing.GroupLayout.PREFERRED_SIZE, 497, javax.swing.GroupLayout.PREFERRED_SIZE) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) - .addComponent(flagTaggedNotableItemsCheckbox)) - .addComponent(bnManageTypes)) - .addContainerGap(javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)) + .addGap(29, 29, 29) + .addComponent(flagTaggedNotableItemsCheckbox, javax.swing.GroupLayout.PREFERRED_SIZE, 246, javax.swing.GroupLayout.PREFERRED_SIZE)) + .addGroup(pnCorrelationPropertiesLayout.createSequentialGroup() + .addGap(18, 18, 18) + .addComponent(generateInterestingItemsLabel, javax.swing.GroupLayout.PREFERRED_SIZE, 187, javax.swing.GroupLayout.PREFERRED_SIZE))) + .addContainerGap(247, Short.MAX_VALUE)) ); pnCorrelationPropertiesLayout.setVerticalGroup( pnCorrelationPropertiesLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addGroup(javax.swing.GroupLayout.Alignment.TRAILING, pnCorrelationPropertiesLayout.createSequentialGroup() - .addGroup(pnCorrelationPropertiesLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addGroup(pnCorrelationPropertiesLayout.createSequentialGroup() - .addContainerGap() - .addComponent(flagTaggedNotableItemsCheckbox)) - .addComponent(correlationPropertiesScrollPane, javax.swing.GroupLayout.DEFAULT_SIZE, 32, Short.MAX_VALUE)) + .addComponent(correlationPropertiesScrollPane, javax.swing.GroupLayout.DEFAULT_SIZE, 32, Short.MAX_VALUE) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) .addComponent(bnManageTypes) .addGap(8, 8, 8)) + .addGroup(pnCorrelationPropertiesLayout.createSequentialGroup() + .addComponent(generateInterestingItemsLabel) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addComponent(flagTaggedNotableItemsCheckbox) + .addGap(0, 0, 0)) ); organizationPanel.setBorder(javax.swing.BorderFactory.createTitledBorder(null, org.openide.util.NbBundle.getMessage(GlobalSettingsPanel.class, "GlobalSettingsPanel.organizationPanel.border.title"), javax.swing.border.TitledBorder.DEFAULT_JUSTIFICATION, javax.swing.border.TitledBorder.DEFAULT_POSITION, new java.awt.Font("Tahoma", 0, 12))); // NOI18N @@ -288,7 +296,7 @@ public final class GlobalSettingsPanel extends IngestModuleGlobalSettingsPanel i .addGroup(organizationPanelLayout.createSequentialGroup() .addContainerGap() .addGroup(organizationPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addComponent(organizationScrollPane, javax.swing.GroupLayout.DEFAULT_SIZE, 980, Short.MAX_VALUE) + .addComponent(organizationScrollPane) .addGroup(organizationPanelLayout.createSequentialGroup() .addComponent(manageOrganizationButton) .addGap(0, 0, Short.MAX_VALUE))) @@ -363,7 +371,7 @@ public final class GlobalSettingsPanel extends IngestModuleGlobalSettingsPanel i jPanel1.setLayout(jPanel1Layout); jPanel1Layout.setHorizontalGroup( jPanel1Layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addComponent(lbCentralRepository, javax.swing.GroupLayout.DEFAULT_SIZE, 1022, Short.MAX_VALUE) + .addComponent(lbCentralRepository, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) .addGroup(jPanel1Layout.createSequentialGroup() .addGroup(jPanel1Layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addComponent(pnDatabaseConfiguration, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) @@ -635,6 +643,7 @@ public final class GlobalSettingsPanel extends IngestModuleGlobalSettingsPanel i casesPanel.setEnabled(enable && !ingestRunning); casesTextArea.setEnabled(enable && !ingestRunning); flagTaggedNotableItemsCheckbox.setEnabled(enable && !ingestRunning); + generateInterestingItemsLabel.setEnabled(enable & !ingestRunning); return true; } @@ -648,6 +657,7 @@ public final class GlobalSettingsPanel extends IngestModuleGlobalSettingsPanel i private javax.swing.JScrollPane correlationPropertiesScrollPane; private javax.swing.JTextArea correlationPropertiesTextArea; private javax.swing.JCheckBox flagTaggedNotableItemsCheckbox; + private javax.swing.JLabel generateInterestingItemsLabel; private javax.swing.JLabel ingestRunningWarningLabel; private javax.swing.JPanel jPanel1; private javax.swing.JScrollPane jScrollPane1; From 39a499c96e51165410aa9d9b20a52105f5da03c3 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Wed, 31 Oct 2018 16:41:40 -0400 Subject: [PATCH 045/129] 4305 always create interesting items if CR is enabled and ingest is not running --- .../eventlisteners/IngestEventsListener.java | 5 +- .../ingestmodule/IngestModule.java | 9 +- .../ingestmodule/IngestSettingsPanel.form | 63 ++++++++++++ .../ingestmodule/IngestSettingsPanel.java | 98 +++++++++++++++++++ .../optionspanel/Bundle.properties | 16 ++- .../optionspanel/EamDbSettingsDialog.java | 2 +- .../optionspanel/GlobalSettingsPanel.form | 37 +------ .../optionspanel/GlobalSettingsPanel.java | 32 +----- 8 files changed, 186 insertions(+), 76 deletions(-) create mode 100755 Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.form create mode 100755 Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.java diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java index 7cdf06f82c..40dcd8338a 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java @@ -47,7 +47,6 @@ import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardAttribute; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.autopsy.centralrepository.datamodel.EamDb; -import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbUtil; import org.sleuthkit.autopsy.coreutils.ThreadUtils; /** @@ -197,8 +196,8 @@ public class IngestEventsListener { } switch (IngestManager.IngestModuleEvent.valueOf(evt.getPropertyName())) { case DATA_ADDED: { - //if ingest isn't running read the flag notable items global setting otherwise use the value that existed when ingest was started - boolean flagNotable = IngestManager.getInstance().isIngestRunning() ? isFlagNotableItems() : EamDbUtil.flagNotableItems(); + //if ingest isn't running create the interesting items + boolean flagNotable = IngestManager.getInstance().isIngestRunning() ? isFlagNotableItems() : true; jobProcessingExecutor.submit(new DataAddedTask(dbManager, evt, flagNotable)); break; } diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java index e091902b85..1ee6dd8a68 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java @@ -79,8 +79,8 @@ final class IngestModule implements FileIngestModule { * * @param settings The ingest settings for the module instance. */ - IngestModule() { - flagTaggedNotableItems = EamDbUtil.flagNotableItems(); + IngestModule(IngestSettings settings) { + flagTaggedNotableItems = settings.isFlagTaggedNotableItems(); } @Override @@ -214,8 +214,9 @@ final class IngestModule implements FileIngestModule { /* * Tell the IngestEventsListener to flag notable items based on the - * global setting. This is a work around for the lack of - * an artifacts pipeline. All modules are affected by the value. While not + * current module's configuration. This is a work around for the lack of + * an artifacts pipeline. Note that this can be changed by another + * module instance. All modules are affected by the value. While not * ideal, this will be good enough until a better solution can be * posited. * diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.form b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.form new file mode 100755 index 0000000000..564031cb72 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.form @@ -0,0 +1,63 @@ + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.java new file mode 100755 index 0000000000..ed36c71287 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestSettingsPanel.java @@ -0,0 +1,98 @@ +/* + * Central Repository + * + * Copyright 2018 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.centralrepository.ingestmodule; + +import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettings; +import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettingsPanel; + +/** + * Ingest job settings panel for the Correlation Engine module. + */ +@SuppressWarnings("PMD.SingularField") // UI widgets cause lots of false positives +final class IngestSettingsPanel extends IngestModuleIngestJobSettingsPanel { + + /** + * Creates new form IngestSettingsPanel + */ + public IngestSettingsPanel(IngestSettings settings) { + initComponents(); + customizeComponents(settings); + } + + /** + * Update components with values from the ingest job settings. + * + * @param settings The ingest job settings. + */ + private void customizeComponents(IngestSettings settings) { + flagTaggedNotableItemsCheckbox.setSelected(settings.isFlagTaggedNotableItems()); + } + + @Override + public IngestModuleIngestJobSettings getSettings() { + return new IngestSettings(flagTaggedNotableItemsCheckbox.isSelected()); + } + + /** + * This method is called from within the constructor to initialize the form. + * WARNING: Do NOT modify this code. The content of this method is always + * regenerated by the Form Editor. + */ + @SuppressWarnings("unchecked") + // //GEN-BEGIN:initComponents + private void initComponents() { + + ingestSettingsLabel = new javax.swing.JLabel(); + flagTaggedNotableItemsCheckbox = new javax.swing.JCheckBox(); + + ingestSettingsLabel.setFont(new java.awt.Font("Tahoma", 1, 11)); // NOI18N + org.openide.awt.Mnemonics.setLocalizedText(ingestSettingsLabel, org.openide.util.NbBundle.getMessage(IngestSettingsPanel.class, "IngestSettingsPanel.ingestSettingsLabel.text")); // NOI18N + + org.openide.awt.Mnemonics.setLocalizedText(flagTaggedNotableItemsCheckbox, org.openide.util.NbBundle.getMessage(IngestSettingsPanel.class, "IngestSettingsPanel.flagTaggedNotableItemsCheckbox.text")); // NOI18N + + javax.swing.GroupLayout layout = new javax.swing.GroupLayout(this); + this.setLayout(layout); + layout.setHorizontalGroup( + layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addGroup(layout.createSequentialGroup() + .addContainerGap() + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addGroup(layout.createSequentialGroup() + .addGap(10, 10, 10) + .addComponent(flagTaggedNotableItemsCheckbox)) + .addComponent(ingestSettingsLabel)) + .addContainerGap(65, Short.MAX_VALUE)) + ); + layout.setVerticalGroup( + layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addGroup(layout.createSequentialGroup() + .addContainerGap() + .addComponent(ingestSettingsLabel) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) + .addComponent(flagTaggedNotableItemsCheckbox) + .addContainerGap(245, Short.MAX_VALUE)) + ); + }// //GEN-END:initComponents + + // Variables declaration - do not modify//GEN-BEGIN:variables + private javax.swing.JCheckBox flagTaggedNotableItemsCheckbox; + private javax.swing.JLabel ingestSettingsLabel; + // End of variables declaration//GEN-END:variables + +} diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/Bundle.properties b/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/Bundle.properties index 88a2150bd6..cbafa5602c 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/Bundle.properties @@ -34,7 +34,7 @@ AddNewOrganizationDialog.bnOK.text=OK AddNewOrganizationDialog.tfName.tooltip=POC Name ManageTagsDialog.okButton.text=OK ManageTagsDialog.cancelButton.text=Cancel -ManageArtifactTypesDialog.taInstructionsMsg.text=Enable one or more correlation properties to use for correlation during ingest. Note, these properties are global and impact all users of the central repository. +ManageArtifactTypesDialog.taInstructionsMsg.text=Enable one or more correlation properties to use for correlation during ingest. Note, these properties are global and impact all users of the Central Repository. EamSqliteSettingsDialog.bnOk.text=OK EamPostgresSettingsDialog.bnSave.text=Save EamDbSettingsDialog.bnDatabasePathFileOpen.text=Browse... @@ -58,10 +58,10 @@ ManageCorrelationPropertiesDialog.okButton.text=OK GlobalSettingsPanel.bnManageProperties.text=Manage Correlation Properties EamDbSettingsDialog.lbDatabaseDesc.text=Database File: EamDbSettingsDialog.lbFullDbPath.text= -GlobalSettingsPanel.cbUseCentralRepo.text=Use a central repository -GlobalSettingsPanel.organizationTextArea.text=Organization information can be tracked in the central repository. +GlobalSettingsPanel.cbUseCentralRepo.text=Use a Central Repository +GlobalSettingsPanel.organizationTextArea.text=Organization information can be tracked in the Central Repository. GlobalSettingsPanel.manageOrganizationButton.text=Manage Organizations -GlobalSettingsPanel.lbCentralRepository.text=A central repository allows you to correlate files and results between cases. +GlobalSettingsPanel.lbCentralRepository.text=A Central Repository allows you to correlate files and results between cases. GlobalSettingsPanel.pnCorrelationProperties.border.title=Correlation Properties GlobalSettingsPanel.organizationPanel.border.title=Organizations GlobalSettingsPanel.casesPanel.border.title=Case Details @@ -73,8 +73,6 @@ ShowCasesDialog.caseDetailsTable.toolTipText=Click column name to sort. Right-cl ShowCasesDialog.title=Case Details GlobalSettingsPanel.Case\ Details.AccessibleContext.accessibleName=Cases Details ShowCasesDialog.caseDetailsTable.AccessibleContext.accessibleDescription=Click column name to sort. -GlobalSettingsPanel.casesTextArea.text=Display table that lists central repository case details. -GlobalSettingsPanel.ingestRunningWarningLabel.text=Cannot make changes to central repository settings when ingest is running! -GlobalSettingsPanel.flagTaggedNotableItemsCheckbox.text=Flag items previously tagged as notable -GlobalSettingsPanel.correlationPropertiesTextArea.text=Choose which file and result properties to store in the central repository for later correlation.\n -GlobalSettingsPanel.generateInterestingItemsLabel.text=Generate interesting items to: +GlobalSettingsPanel.casesTextArea.text=Display table that lists Central Repository case details. +GlobalSettingsPanel.ingestRunningWarningLabel.text=Cannot make changes to Central Repository settings when ingest is running! +GlobalSettingsPanel.correlationPropertiesTextArea.text=Choose which file and result properties to store in the Central Repository for later correlation.\n diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/EamDbSettingsDialog.java b/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/EamDbSettingsDialog.java index c504f666e4..826e66ecbc 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/EamDbSettingsDialog.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/EamDbSettingsDialog.java @@ -102,7 +102,7 @@ public class EamDbSettingsDialog extends JDialog { @Override public String getDescription() { - return "Directories and central repository databases"; + return "Directories and Central Repository databases"; } }); cbDatabaseType.setSelectedItem(selectedPlatform); diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/GlobalSettingsPanel.form b/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/GlobalSettingsPanel.form index 24efccdc99..0a8c7dcc64 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/GlobalSettingsPanel.form +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/GlobalSettingsPanel.form @@ -67,7 +67,7 @@ - + @@ -244,21 +244,14 @@ - - - - - - - - - - + + + - + @@ -270,12 +263,6 @@ - - - - - -
@@ -326,20 +313,6 @@ - - - - - - - - - - - - - - diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/GlobalSettingsPanel.java b/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/GlobalSettingsPanel.java index ed7fa1b1d0..c8560fce10 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/GlobalSettingsPanel.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/optionspanel/GlobalSettingsPanel.java @@ -120,8 +120,6 @@ public final class GlobalSettingsPanel extends IngestModuleGlobalSettingsPanel i bnManageTypes = new javax.swing.JButton(); correlationPropertiesScrollPane = new javax.swing.JScrollPane(); correlationPropertiesTextArea = new javax.swing.JTextArea(); - flagTaggedNotableItemsCheckbox = new javax.swing.JCheckBox(); - generateInterestingItemsLabel = new javax.swing.JLabel(); organizationPanel = new javax.swing.JPanel(); manageOrganizationButton = new javax.swing.JButton(); organizationScrollPane = new javax.swing.JScrollPane(); @@ -231,27 +229,18 @@ public final class GlobalSettingsPanel extends IngestModuleGlobalSettingsPanel i correlationPropertiesTextArea.setBorder(null); correlationPropertiesScrollPane.setViewportView(correlationPropertiesTextArea); - org.openide.awt.Mnemonics.setLocalizedText(flagTaggedNotableItemsCheckbox, org.openide.util.NbBundle.getMessage(GlobalSettingsPanel.class, "GlobalSettingsPanel.flagTaggedNotableItemsCheckbox.text")); // NOI18N - - org.openide.awt.Mnemonics.setLocalizedText(generateInterestingItemsLabel, org.openide.util.NbBundle.getMessage(GlobalSettingsPanel.class, "GlobalSettingsPanel.generateInterestingItemsLabel.text")); // NOI18N - javax.swing.GroupLayout pnCorrelationPropertiesLayout = new javax.swing.GroupLayout(pnCorrelationProperties); pnCorrelationProperties.setLayout(pnCorrelationPropertiesLayout); pnCorrelationPropertiesLayout.setHorizontalGroup( pnCorrelationPropertiesLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addGroup(pnCorrelationPropertiesLayout.createSequentialGroup() .addContainerGap() - .addGroup(pnCorrelationPropertiesLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addComponent(bnManageTypes) - .addComponent(correlationPropertiesScrollPane, javax.swing.GroupLayout.PREFERRED_SIZE, 468, javax.swing.GroupLayout.PREFERRED_SIZE)) .addGroup(pnCorrelationPropertiesLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addGroup(pnCorrelationPropertiesLayout.createSequentialGroup() - .addGap(29, 29, 29) - .addComponent(flagTaggedNotableItemsCheckbox, javax.swing.GroupLayout.PREFERRED_SIZE, 246, javax.swing.GroupLayout.PREFERRED_SIZE)) - .addGroup(pnCorrelationPropertiesLayout.createSequentialGroup() - .addGap(18, 18, 18) - .addComponent(generateInterestingItemsLabel, javax.swing.GroupLayout.PREFERRED_SIZE, 187, javax.swing.GroupLayout.PREFERRED_SIZE))) - .addContainerGap(247, Short.MAX_VALUE)) + .addComponent(bnManageTypes) + .addGap(0, 0, Short.MAX_VALUE)) + .addComponent(correlationPropertiesScrollPane)) + .addContainerGap()) ); pnCorrelationPropertiesLayout.setVerticalGroup( pnCorrelationPropertiesLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) @@ -260,11 +249,6 @@ public final class GlobalSettingsPanel extends IngestModuleGlobalSettingsPanel i .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) .addComponent(bnManageTypes) .addGap(8, 8, 8)) - .addGroup(pnCorrelationPropertiesLayout.createSequentialGroup() - .addComponent(generateInterestingItemsLabel) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) - .addComponent(flagTaggedNotableItemsCheckbox) - .addGap(0, 0, 0)) ); organizationPanel.setBorder(javax.swing.BorderFactory.createTitledBorder(null, org.openide.util.NbBundle.getMessage(GlobalSettingsPanel.class, "GlobalSettingsPanel.organizationPanel.border.title"), javax.swing.border.TitledBorder.DEFAULT_JUSTIFICATION, javax.swing.border.TitledBorder.DEFAULT_POSITION, new java.awt.Font("Tahoma", 0, 12))); // NOI18N @@ -381,7 +365,7 @@ public final class GlobalSettingsPanel extends IngestModuleGlobalSettingsPanel i .addGroup(jPanel1Layout.createSequentialGroup() .addComponent(cbUseCentralRepo, javax.swing.GroupLayout.PREFERRED_SIZE, 162, javax.swing.GroupLayout.PREFERRED_SIZE) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) - .addComponent(ingestRunningWarningLabel, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)) + .addComponent(ingestRunningWarningLabel, javax.swing.GroupLayout.DEFAULT_SIZE, 844, Short.MAX_VALUE)) .addGroup(jPanel1Layout.createSequentialGroup() .addContainerGap() .addComponent(tbOops, javax.swing.GroupLayout.PREFERRED_SIZE, 974, javax.swing.GroupLayout.PREFERRED_SIZE))) @@ -466,7 +450,6 @@ public final class GlobalSettingsPanel extends IngestModuleGlobalSettingsPanel i enableAllSubComponents(false); EamDbPlatformEnum selectedPlatform = EamDbPlatformEnum.getSelectedPlatform(); cbUseCentralRepo.setSelected(EamDbUtil.useCentralRepo()); // NON-NLS - flagTaggedNotableItemsCheckbox.setSelected(EamDbUtil.flagNotableItems()); switch (selectedPlatform) { case POSTGRESQL: PostgresEamDbSettings dbSettingsPg = new PostgresEamDbSettings(); @@ -496,7 +479,6 @@ public final class GlobalSettingsPanel extends IngestModuleGlobalSettingsPanel i @Override public void store() { // Click OK or Apply on Options Panel EamDbUtil.setUseCentralRepo(cbUseCentralRepo.isSelected()); - EamDbUtil.setFlagNotableItems(flagTaggedNotableItemsCheckbox.isSelected()); } /** @@ -642,8 +624,6 @@ public final class GlobalSettingsPanel extends IngestModuleGlobalSettingsPanel i showCasesButton.setEnabled(enable && !ingestRunning); casesPanel.setEnabled(enable && !ingestRunning); casesTextArea.setEnabled(enable && !ingestRunning); - flagTaggedNotableItemsCheckbox.setEnabled(enable && !ingestRunning); - generateInterestingItemsLabel.setEnabled(enable & !ingestRunning); return true; } @@ -656,8 +636,6 @@ public final class GlobalSettingsPanel extends IngestModuleGlobalSettingsPanel i private javax.swing.JCheckBox cbUseCentralRepo; private javax.swing.JScrollPane correlationPropertiesScrollPane; private javax.swing.JTextArea correlationPropertiesTextArea; - private javax.swing.JCheckBox flagTaggedNotableItemsCheckbox; - private javax.swing.JLabel generateInterestingItemsLabel; private javax.swing.JLabel ingestRunningWarningLabel; private javax.swing.JPanel jPanel1; private javax.swing.JScrollPane jScrollPane1; From 46d3e32d6da9d0f8ea719adcda6c38e076bfde5e Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Wed, 31 Oct 2018 16:42:58 -0400 Subject: [PATCH 046/129] 4305 restore previous version of cr ingestmodulefactory --- .../ingestmodule/IngestModuleFactory.java | 39 ++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModuleFactory.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModuleFactory.java index 8d6068a408..fc5dcd9c81 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModuleFactory.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModuleFactory.java @@ -26,6 +26,8 @@ import org.sleuthkit.autopsy.ingest.IngestModuleGlobalSettingsPanel; import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettings; import org.sleuthkit.autopsy.centralrepository.optionspanel.GlobalSettingsPanel; import org.sleuthkit.autopsy.coreutils.Version; +import org.sleuthkit.autopsy.ingest.IngestModuleIngestJobSettingsPanel; +import org.sleuthkit.autopsy.ingest.NoIngestModuleIngestJobSettings; /** * Factory for Central Repository ingest modules @@ -66,7 +68,17 @@ public class IngestModuleFactory extends IngestModuleFactoryAdapter { @Override public FileIngestModule createFileIngestModule(IngestModuleIngestJobSettings settings) { - return new IngestModule(); + if (settings instanceof IngestSettings) { + return new IngestModule((IngestSettings) settings); + } + /* + * Compatibility check for older versions. + */ + if (settings instanceof NoIngestModuleIngestJobSettings) { + return new IngestModule(new IngestSettings()); + } + + throw new IllegalArgumentException("Expected settings argument to be an instance of IngestSettings"); } @Override @@ -80,5 +92,30 @@ public class IngestModuleFactory extends IngestModuleFactoryAdapter { globalOptionsPanel.load(); return globalOptionsPanel; } + + @Override + public IngestModuleIngestJobSettings getDefaultIngestJobSettings() { + return new IngestSettings(); + } + + @Override + public boolean hasIngestJobSettingsPanel() { + return true; + } + + @Override + public IngestModuleIngestJobSettingsPanel getIngestJobSettingsPanel(IngestModuleIngestJobSettings settings) { + if (settings instanceof IngestSettings) { + return new IngestSettingsPanel((IngestSettings) settings); + } + /* + * Compatibility check for older versions. + */ + if (settings instanceof NoIngestModuleIngestJobSettings) { + return new IngestSettingsPanel(new IngestSettings()); + } + + throw new IllegalArgumentException("Expected settings argument to be an instance of IngestSettings"); + } } From 077d95456703300a054801f8f77655988d02cf04 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Thu, 1 Nov 2018 12:14:41 -0400 Subject: [PATCH 047/129] 4305 fix codacy issues --- .../eventlisteners/IngestEventsListener.java | 4 ++-- .../autopsy/centralrepository/ingestmodule/IngestModule.java | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java index 40dcd8338a..2f2a35ab2a 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/eventlisteners/IngestEventsListener.java @@ -196,8 +196,8 @@ public class IngestEventsListener { } switch (IngestManager.IngestModuleEvent.valueOf(evt.getPropertyName())) { case DATA_ADDED: { - //if ingest isn't running create the interesting items - boolean flagNotable = IngestManager.getInstance().isIngestRunning() ? isFlagNotableItems() : true; + //if ingest isn't running create the interesting items otherwise use the ingest module setting to determine if we create interesting items + boolean flagNotable = !IngestManager.getInstance().isIngestRunning() || isFlagNotableItems(); jobProcessingExecutor.submit(new DataAddedTask(dbManager, evt, flagNotable)); break; } diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java index 1ee6dd8a68..7469e38bd0 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java @@ -42,7 +42,6 @@ import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationDataSource; import org.sleuthkit.autopsy.centralrepository.datamodel.EamDb; import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbPlatformEnum; import org.sleuthkit.autopsy.centralrepository.datamodel.EamArtifactUtil; -import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbUtil; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardAttribute; From c5b43a93559b4826b027ceb1e4f53a5512c28f0a Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Thu, 1 Nov 2018 15:59:55 -0400 Subject: [PATCH 048/129] Done with changes, now commenting and making it more readable, on the verge of opening a PR --- .../datamodel/AbstractAbstractFileNode.java | 473 +++++++----------- .../autopsy/datamodel/FileProperty.java | 19 +- .../autopsy/datamodel/PropertyUtil.java | 222 ++++++++ .../datamodel/SCOAndTranslationTask.java | 144 ++++++ .../datamodel/VirtualDirectoryNode.java | 2 +- 5 files changed, 552 insertions(+), 308 deletions(-) create mode 100755 Core/src/org/sleuthkit/autopsy/datamodel/PropertyUtil.java create mode 100755 Core/src/org/sleuthkit/autopsy/datamodel/SCOAndTranslationTask.java diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java index f4d1756ce6..9f5fb5cbad 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java @@ -20,6 +20,7 @@ package org.sleuthkit.autopsy.datamodel; import java.beans.PropertyChangeEvent; import java.beans.PropertyChangeListener; +import java.lang.ref.WeakReference; import java.util.ArrayList; import java.util.EnumSet; import java.util.List; @@ -29,7 +30,6 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.logging.Level; import java.util.stream.Collectors; -import org.apache.commons.io.FilenameUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; import org.openide.nodes.Children; @@ -42,29 +42,18 @@ import org.sleuthkit.autopsy.casemodule.events.CommentChangedEvent; import org.sleuthkit.autopsy.casemodule.events.ContentTagAddedEvent; import org.sleuthkit.autopsy.casemodule.events.ContentTagDeletedEvent; import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeInstance; -import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeNormalizationException; -import org.sleuthkit.autopsy.centralrepository.datamodel.EamArtifactUtil; -import org.sleuthkit.autopsy.centralrepository.datamodel.EamDb; -import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbException; -import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbUtil; import org.sleuthkit.autopsy.core.UserPreferences; import org.sleuthkit.autopsy.corecomponents.DataResultViewerTable.Score; import org.sleuthkit.autopsy.coreutils.Logger; import static org.sleuthkit.autopsy.datamodel.Bundle.*; import static org.sleuthkit.autopsy.datamodel.AbstractAbstractFileNode.AbstractFilePropertyType.*; -import org.sleuthkit.autopsy.corecomponents.DataResultViewerTable.HasCommentStatus; -import org.sleuthkit.autopsy.events.AutopsyEvent; +import org.sleuthkit.autopsy.datamodel.SCOAndTranslationTask.SCOResults; import org.sleuthkit.autopsy.ingest.IngestManager; import org.sleuthkit.autopsy.ingest.ModuleContentEvent; -import org.sleuthkit.autopsy.texttranslation.NoServiceProviderException; -import org.sleuthkit.autopsy.texttranslation.TextTranslationService; -import org.sleuthkit.autopsy.texttranslation.TranslationException; import org.sleuthkit.datamodel.AbstractFile; -import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.ContentTag; import org.sleuthkit.datamodel.TskCoreException; -import org.sleuthkit.datamodel.TskData; /** * An abstract node that encapsulates AbstractFile data @@ -77,14 +66,9 @@ public abstract class AbstractAbstractFileNode extends A @NbBundle.Messages("AbstractAbstractFileNode.addFileProperty.desc=no description") private static final String NO_DESCR = AbstractAbstractFileNode_addFileProperty_desc(); - private static final String TRANSLATION_AVAILABLE_EVENT = "TRANSLATION_AVAILABLE"; - private static final String NO_TRANSLATION = ""; - private static final Set CASE_EVENTS_OF_INTEREST = EnumSet.of(Case.Events.CURRENT_CASE, Case.Events.CONTENT_TAG_ADDED, Case.Events.CONTENT_TAG_DELETED, Case.Events.CR_COMMENT_CHANGED); - private volatile List currentProperties; - private static final ExecutorService pool; private static final Integer MAX_POOL_SIZE = 10; @@ -108,6 +92,8 @@ public abstract class AbstractAbstractFileNode extends A } static { + //Initialize this pool only once! This will be used by every instance of AAFN + //to do their heavy duty SCO column and translation updates. pool = Executors.newFixedThreadPool(MAX_POOL_SIZE); } @@ -130,13 +116,17 @@ public abstract class AbstractAbstractFileNode extends A IngestManager.getInstance().removeIngestModuleEventListener(weakPcl); Case.removeEventTypeSubscriber(CASE_EVENTS_OF_INTEREST, weakPcl); } - - //Events signals to indicate the background tasks have completed processing. - private enum NodeSpecificEvents { - TRANSLATION_AVAILABLE_EVENT, - SCORE_AVAILABLE_EVENT, - COMMENT_AVAILABLE_EVENT, - OCCURRENCES_AVAILABLE_EVENT; + + /** + * Event signals to indicate the background tasks have completed processing. + * Currently, we have two property tasks in the background: + * + * 1) Retreiving the translation of the file name 2) Getting the SCO column + * properties from the databases + */ + enum NodeSpecificEvents { + TRANSLATION_AVAILABLE, + DABABASE_CONTENT_AVAILABLE; } private final PropertyChangeListener pcl = (PropertyChangeEvent evt) -> { @@ -178,56 +168,111 @@ public abstract class AbstractAbstractFileNode extends A } else if (eventType.equals(Case.Events.CONTENT_TAG_ADDED.toString())) { ContentTagAddedEvent event = (ContentTagAddedEvent) evt; if (event.getAddedTag().getContent().equals(content)) { - updateSheet(); + //No need to do any asynchrony around these events, they are so infrequent + //and user driven that we can just keep a simple blocking approach, where we + //go out to the database ourselves! + List tags = PropertyUtil.getContentTagsFromDatabase(content); + + updateProperty(new FileProperty(SCORE.toString()) { + Pair scorePropertyAndDescription + = PropertyUtil.getScorePropertyAndDescription(content, tags); + + @Override + public Object getPropertyValue() { + return scorePropertyAndDescription.getLeft(); + } + + @Override + public String getDescription() { + return scorePropertyAndDescription.getRight(); + } + }, new FileProperty(COMMENT.toString()) { + @Override + public Object getPropertyValue() { + //Null out the correlation attribute because we are only + //concerned with changes to the content tag, not the CR! + return PropertyUtil.getCommentProperty(tags, null); + } + }); } } else if (eventType.equals(Case.Events.CONTENT_TAG_DELETED.toString())) { ContentTagDeletedEvent event = (ContentTagDeletedEvent) evt; if (event.getDeletedTagInfo().getContentID() == content.getId()) { - updateSheet(); + //No need to do any asynchrony around these events, they are so infrequent + //and user driven that we can just keep a simple blocking approach, where we + //go out to the database ourselves! + List tags = PropertyUtil.getContentTagsFromDatabase(content); + + updateProperty(new FileProperty(SCORE.toString()) { + Pair scorePropertyAndDescription + = PropertyUtil.getScorePropertyAndDescription(content, tags); + + @Override + public Object getPropertyValue() { + return scorePropertyAndDescription.getLeft(); + } + + @Override + public String getDescription() { + return scorePropertyAndDescription.getRight(); + } + }, new FileProperty(COMMENT.toString()) { + @Override + public Object getPropertyValue() { + //Null out the correlation attribute because we are only + //concerned with changes to the content tag, not the CR! + return PropertyUtil.getCommentProperty(tags, null); + } + }); } } else if (eventType.equals(Case.Events.CR_COMMENT_CHANGED.toString())) { CommentChangedEvent event = (CommentChangedEvent) evt; if (event.getContentID() == content.getId()) { - updateSheet(); + //No need to do any asynchrony around these events, they are so infrequent + //and user driven that we can just keep a simple blocking approach, where we + //go out to the database ourselves! + updateProperty(new FileProperty(COMMENT.toString()) { + @Override + public Object getPropertyValue() { + List tags = PropertyUtil.getContentTagsFromDatabase(content); + CorrelationAttributeInstance attribute = PropertyUtil.getCorrelationAttributeInstance(content); + return PropertyUtil.getCommentProperty(tags, attribute); + } + }); } - } else if (eventType.equals(NodeSpecificEvents.TRANSLATION_AVAILABLE_EVENT.toString())) { - updateProperty(TRANSLATION.toString(), new FileProperty(TRANSLATION.toString()) { + } else if (eventType.equals(NodeSpecificEvents.TRANSLATION_AVAILABLE.toString())) { + updateProperty(new FileProperty(TRANSLATION.toString()) { @Override public Object getPropertyValue() { return evt.getNewValue(); } }); - } else if(eventType.equals(NodeSpecificEvents.SCORE_AVAILABLE_EVENT.toString())) { - Pair scoreAndDescription = (Pair) evt.getNewValue(); - updateProperty(SCORE.toString(), new FileProperty(SCORE.toString()) { + } else if (eventType.equals(NodeSpecificEvents.DABABASE_CONTENT_AVAILABLE.toString())) { + SCOResults results = (SCOResults) evt.getNewValue(); + updateProperty(new FileProperty(SCORE.toString()) { @Override public Object getPropertyValue() { - return scoreAndDescription.getLeft(); + return results.getScore(); } - + @Override public String getDescription() { - return scoreAndDescription.getRight(); + return results.getScoreDescription(); } - }); - } else if(eventType.equals(NodeSpecificEvents.COMMENT_AVAILABLE_EVENT.toString())) { - updateProperty(COMMENT.toString(), new FileProperty(COMMENT.toString()) { + }, new FileProperty(COMMENT.toString()) { @Override public Object getPropertyValue() { - return evt.getNewValue(); + return results.getComment(); } - }); - } else if(eventType.equals(NodeSpecificEvents.OCCURRENCES_AVAILABLE_EVENT.toString())) { - Pair countAndDescription = (Pair) evt.getNewValue(); - updateProperty(OCCURRENCES.toString(), new FileProperty(OCCURRENCES.toString()) { + }, new FileProperty(OCCURRENCES.toString()) { @Override public Object getPropertyValue() { - return countAndDescription.getLeft(); + return results.getCount(); } - + @Override public String getDescription() { - return countAndDescription.getRight(); + return results.getCountDescription(); } }); } @@ -242,55 +287,71 @@ public abstract class AbstractAbstractFileNode extends A * unregistering of the listener in removeListeners() below. */ private final PropertyChangeListener weakPcl = WeakListeners.propertyChange(pcl, null); - - Sheet getBlankSheet() { + + /** + * Returns a blank sheet to the caller, useful for giving subclasses the + * ability to override createSheet() with their own implementation. + * + * @return + */ + protected Sheet getBlankSheet() { return super.createSheet(); } - - private synchronized void updateProperty(String propName, FileProperty newProp) { - for(int i = 0; i < currentProperties.size(); i++) { - FileProperty property = currentProperties.get(i); - if(property.getPropertyName().equals(propName)) { - currentProperties.set(i, newProp); + + /** + * Updates the values of the properties in the current property sheet with + * the new properties being passed in! Only if that property exists in the + * current sheet will it be applied. That way, we allow for subclasses to + * add their own (or omit some!) properties and we will not accidentally + * disrupt their UI. + * + * Race condition if not synchronized. Only one update should be applied at a time. + * The timing of currSheetSet.getProperties() could result in wrong/stale data + * being shown! + * + * @param newProps New file property instances to be updated in the current + * sheet. + */ + private synchronized void updateProperty(FileProperty... newProps) { + + //Refresh ONLY those properties in the sheet currently. Subclasses may have + //only added a subset of our properties or their own props! Let's keep their UI correct. + Sheet currSheet = this.getSheet(); + Sheet.Set currSheetSet = currSheet.get(Sheet.PROPERTIES); + Property[] currProps = currSheetSet.getProperties(); + + for (int i = 0; i < currProps.length; i++) { + for (FileProperty property : newProps) { + if (currProps[i].getName().equals(property.getPropertyName())) { + currProps[i] = new NodeProperty<>( + property.getPropertyName(), + property.getPropertyName(), + property.getDescription(), + property.getPropertyValue()); + } } } - - Sheet sheet = super.createSheet(); - Sheet.Set sheetSet = Sheet.createPropertiesSet(); - sheet.put(sheetSet); - - for (FileProperty property : currentProperties) { - if (property.isEnabled()) { - sheetSet.put(new NodeProperty<>( - property.getPropertyName(), - property.getPropertyName(), - property.getDescription(), - property.getPropertyValue())); - } - } - - this.setSheet(sheet); - } - - //Race condition if not synchronized. The main thread could be updating the - //sheet with blank properties, if a background task is complete, it may finish - //the updateSheet() call before the main thread. If that's the case, main thread's - //sheet will be the one to 'win' the UI and we will see stale data. Problem - //for multi-user cases if both the CR comment is updated from elsewhere and the background - //task is completed and also trying to update. - private synchronized void updateSheet() { - this.setSheet(createSheet()); + + currSheetSet.put(currProps); + currSheet.put(currSheetSet); + + //setSheet() will notify Netbeans to update this node in the UI! + this.setSheet(currSheet); } + /* + * This is called when the node is first initialized. Any new updates or changes + * happen by directly manipulating the sheet. That means we can fire off background + * events everytime this method is called and not worry about duplicated jobs! + */ @Override - protected Sheet createSheet() { - Sheet sheet = super.createSheet(); + protected synchronized Sheet createSheet() { + Sheet sheet = getBlankSheet(); Sheet.Set sheetSet = Sheet.createPropertiesSet(); sheet.put(sheetSet); - + //This will fire off fresh background tasks. List newProperties = getProperties(); - currentProperties = newProperties; //Add only the enabled properties to the sheet! for (FileProperty property : newProperties) { @@ -372,63 +433,17 @@ public abstract class AbstractAbstractFileNode extends A } /** - * Attempts translation on the file name by kicking off a background task to - * do the translation. Once the background task is done, it will fire a - * PropertyChangeEvent, which will force this node to refresh itself, thus - * updating its translated name column. + * Creates a list of properties for this file node. Each property has its + * own strategy for producing a value, its own description, name, and + * ability to be disabled. The FileProperty abstract class provides a + * wrapper for all of these characteristics. Additionally, with a return + * value of a list, any children classes of this node may reorder or omit + * any of these properties as they see fit for their use case. * - * @return The file names translation. - */ - private String getTranslatedFileName() { - //If already in complete English, don't translate. - if (content.getName().matches("^\\p{ASCII}+$")) { - return NO_TRANSLATION; - } - - //Lets fire off a background translation that will update the UI - //when it is done. - TextTranslationService tts = TextTranslationService.getInstance(); - if (tts.hasProvider()) { - //Seperate out the base and ext from the contents file name. - String base = FilenameUtils.getBaseName(content.getName()); - - try { - String translation = tts.translate(base); - String ext = FilenameUtils.getExtension(content.getName()); - - //If we have no extension, then we shouldn't add the . - String extensionDelimiter = (ext.isEmpty()) ? "" : "."; - - //Talk directly to this nodes pcl, fire an update when the translation - //is complete. - if (!translation.isEmpty()) { - return translation + extensionDelimiter + ext; - } - } catch (NoServiceProviderException noServiceEx) { - logger.log(Level.WARNING, "Translate unsuccessful because no TextTranslator " - + "implementation was provided.", noServiceEx); - } catch (TranslationException noTranslationEx) { - logger.log(Level.WARNING, "Could not successfully translate file name " - + content.getName(), noTranslationEx); - } - } - - //In the mean time, return a blank translation. - return NO_TRANSLATION; - } - - /** - * Creates a list of properties for this file node. Each property has it's own - * strategy for getting it's value, it's own description, name, and ability to be - * disabled. The FileProperty abstract class provides a wrapper for all - * of these characteristics. Additionally, with a return value of a list, any - * children classes of this node may reorder or omit any of these properties as - * they see fit for their use case. - * - * @return List of file properties associated with this file nodes content. + * @return List of file properties associated with this file node's content. */ List getProperties() { - ArrayList properties = new ArrayList<>(); + List properties = new ArrayList<>(); properties.add(new FileProperty(NAME.toString()) { @Override @@ -436,17 +451,15 @@ public abstract class AbstractAbstractFileNode extends A return getContentDisplayName(content); } }); + + //Initialize dummy place holder properties! These obviously do no work + //to get their property values, but at the bottom we kick off a background + //task that promises to update these values. + final String NO_OP = ""; properties.add(new FileProperty(TRANSLATION.toString()) { @Override public Object getPropertyValue() { - pool.submit(() -> { - weakPcl.propertyChange(new PropertyChangeEvent( - AutopsyEvent.SourceType.LOCAL.toString(), - NodeSpecificEvents.TRANSLATION_AVAILABLE_EVENT.toString(), - null, - getTranslatedFileName())); - }); - return ""; + return NO_OP; } @Override @@ -455,35 +468,16 @@ public abstract class AbstractAbstractFileNode extends A } }); - List tags = getContentTagsFromDatabase(content); - CorrelationAttributeInstance correlationAttribute = getCorrelationAttributeInstance(); properties.add(new FileProperty(SCORE.toString()) { @Override public Object getPropertyValue() { - pool.submit(() -> { - List tags = getContentTagsFromDatabase(content); - weakPcl.propertyChange(new PropertyChangeEvent( - AutopsyEvent.SourceType.LOCAL.toString(), - NodeSpecificEvents.SCORE_AVAILABLE_EVENT.toString(), - null, - getScorePropertyAndDescription(tags))); - }); - return ""; + return NO_OP; } }); properties.add(new FileProperty(COMMENT.toString()) { @Override public Object getPropertyValue() { - pool.submit(() -> { - List tags = getContentTagsFromDatabase(content); - CorrelationAttributeInstance correlationAttribute = getCorrelationAttributeInstance(); - weakPcl.propertyChange(new PropertyChangeEvent( - AutopsyEvent.SourceType.LOCAL.toString(), - NodeSpecificEvents.COMMENT_AVAILABLE_EVENT.toString(), - null, - getCommentProperty(tags, correlationAttribute))); - }); - return ""; + return NO_OP; } @Override @@ -494,15 +488,7 @@ public abstract class AbstractAbstractFileNode extends A properties.add(new FileProperty(OCCURRENCES.toString()) { @Override public Object getPropertyValue() { - pool.submit(() -> { - CorrelationAttributeInstance correlationAttribute = getCorrelationAttributeInstance(); - weakPcl.propertyChange(new PropertyChangeEvent( - AutopsyEvent.SourceType.LOCAL.toString(), - NodeSpecificEvents.OCCURRENCES_AVAILABLE_EVENT.toString(), - null, - getCountPropertyAndDescription(correlationAttribute))); - }); - return ""; + return NO_OP; } @Override @@ -631,135 +617,14 @@ public abstract class AbstractAbstractFileNode extends A } }); + //Submit the database queries ASAP! We want updated SCO columns + //without blocking the UI as soon as we can get it! Keep all weak references + //so this task doesn't block the ability of this node to be GC'd. Handle potentially + //null reference values in the Task! + pool.submit(new SCOAndTranslationTask(new WeakReference<>(content), weakPcl)); return properties; } - /** - * Get all tags from the case database that are associated with the file - * - * @return a list of tags that are associated with the file - */ - private List getContentTagsFromDatabase(AbstractFile content) { - List tags = new ArrayList<>(); - try { - tags.addAll(Case.getCurrentCaseThrows().getServices().getTagsManager().getContentTagsByContent(content)); - } catch (TskCoreException | NoCurrentCaseException ex) { - logger.log(Level.SEVERE, "Failed to get tags for content " + content.getName(), ex); - } - return tags; - } - - private CorrelationAttributeInstance getCorrelationAttributeInstance() { - CorrelationAttributeInstance correlationAttribute = null; - if (EamDbUtil.useCentralRepo()) { - correlationAttribute = EamArtifactUtil.getInstanceFromContent(content); - } - return correlationAttribute; - } - - /** - * Used by subclasses of AbstractAbstractFileNode to add the comment - * property to their sheets. - * - * @param sheetSet the modifiable Sheet.Set returned by - * Sheet.get(Sheet.PROPERTIES) - * @param tags the list of tags associated with the file - * @param attribute the correlation attribute associated with this file, - * null if central repo is not enabled - */ - @NbBundle.Messages({ - "AbstractAbstractFileNode.createSheet.comment.displayName=C"}) - private HasCommentStatus getCommentProperty(List tags, CorrelationAttributeInstance attribute) { - - HasCommentStatus status = tags.size() > 0 ? HasCommentStatus.TAG_NO_COMMENT : HasCommentStatus.NO_COMMENT; - - for (ContentTag tag : tags) { - if (!StringUtils.isBlank(tag.getComment())) { - //if the tag is null or empty or contains just white space it will indicate there is not a comment - status = HasCommentStatus.TAG_COMMENT; - break; - } - } - if (attribute != null && !StringUtils.isBlank(attribute.getComment())) { - if (status == HasCommentStatus.TAG_COMMENT) { - status = HasCommentStatus.CR_AND_TAG_COMMENTS; - } else { - status = HasCommentStatus.CR_COMMENT; - } - } - return status; - } - - /** - * Used by subclasses of AbstractAbstractFileNode to add the Score property - * to their sheets. - * - * @param sheetSet the modifiable Sheet.Set returned by - * Sheet.get(Sheet.PROPERTIES) - * @param tags the list of tags associated with the file - */ - @NbBundle.Messages({ - "AbstractAbstractFileNode.createSheet.score.displayName=S", - "AbstractAbstractFileNode.createSheet.notableFile.description=File recognized as notable.", - "AbstractAbstractFileNode.createSheet.interestingResult.description=File has interesting result associated with it.", - "AbstractAbstractFileNode.createSheet.taggedFile.description=File has been tagged.", - "AbstractAbstractFileNode.createSheet.notableTaggedFile.description=File tagged with notable tag.", - "AbstractAbstractFileNode.createSheet.noScore.description=No score"}) - private Pair getScorePropertyAndDescription(List tags) { - Score score = Score.NO_SCORE; - String description = ""; - if (content.getKnown() == TskData.FileKnown.BAD) { - score = Score.NOTABLE_SCORE; - description = Bundle.AbstractAbstractFileNode_createSheet_notableFile_description(); - } - try { - if (score == Score.NO_SCORE && !content.getArtifacts(BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT).isEmpty()) { - score = Score.INTERESTING_SCORE; - description = Bundle.AbstractAbstractFileNode_createSheet_interestingResult_description(); - } - } catch (TskCoreException ex) { - logger.log(Level.WARNING, "Error getting artifacts for file: " + content.getName(), ex); - } - if (tags.size() > 0 && (score == Score.NO_SCORE || score == Score.INTERESTING_SCORE)) { - score = Score.INTERESTING_SCORE; - description = Bundle.AbstractAbstractFileNode_createSheet_taggedFile_description(); - for (ContentTag tag : tags) { - if (tag.getName().getKnownStatus() == TskData.FileKnown.BAD) { - score = Score.NOTABLE_SCORE; - description = Bundle.AbstractAbstractFileNode_createSheet_notableTaggedFile_description(); - break; - } - } - } - return Pair.of(score, description); - } - - @NbBundle.Messages({ - "AbstractAbstractFileNode.createSheet.count.displayName=O", - "AbstractAbstractFileNode.createSheet.count.noCentralRepo.description=Central repository was not enabled when this column was populated", - "AbstractAbstractFileNode.createSheet.count.hashLookupNotRun.description=Hash lookup had not been run on this file when the column was populated", - "# {0} - occuranceCount", - "AbstractAbstractFileNode.createSheet.count.description=There were {0} datasource(s) found with occurances of the correlation value"}) - private static Pair getCountPropertyAndDescription(CorrelationAttributeInstance attribute) { - Long count = -1L; //The column renderer will not display negative values, negative value used when count unavailble to preserve sorting - String description = Bundle.AbstractAbstractFileNode_createSheet_count_noCentralRepo_description(); - try { - //don't perform the query if there is no correlation value - if (attribute != null && StringUtils.isNotBlank(attribute.getCorrelationValue())) { - count = EamDb.getInstance().getCountUniqueCaseDataSourceTuplesHavingTypeValue(attribute.getCorrelationType(), attribute.getCorrelationValue()); - description = Bundle.AbstractAbstractFileNode_createSheet_count_description(count); - } else if (attribute != null) { - description = Bundle.AbstractAbstractFileNode_createSheet_count_hashLookupNotRun_description(); - } - } catch (EamDbException ex) { - logger.log(Level.WARNING, "Error getting count of datasources with correlation attribute", ex); - } catch (CorrelationAttributeNormalizationException ex) { - logger.log(Level.WARNING, "Unable to normalize data to get count of datasources with correlation attribute", ex); - } - - return Pair.of(count, description); - } - /** * Used by subclasses of AbstractAbstractFileNode to add the tags property * to their sheets. @@ -825,7 +690,7 @@ public abstract class AbstractAbstractFileNode extends A return ""; } } - + /** * Fill map with AbstractFile properties * diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/FileProperty.java b/Core/src/org/sleuthkit/autopsy/datamodel/FileProperty.java index 0755f1b29b..e002b91c7b 100755 --- a/Core/src/org/sleuthkit/autopsy/datamodel/FileProperty.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/FileProperty.java @@ -1,7 +1,20 @@ /* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. + * Autopsy Forensic Browser + * + * Copyright 2018-2018 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.sleuthkit.autopsy.datamodel; diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/PropertyUtil.java b/Core/src/org/sleuthkit/autopsy/datamodel/PropertyUtil.java new file mode 100755 index 0000000000..f01f659627 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/datamodel/PropertyUtil.java @@ -0,0 +1,222 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2018-2018 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.datamodel; + +import java.util.ArrayList; +import java.util.List; +import java.util.logging.Level; +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.openide.util.NbBundle; +import org.sleuthkit.autopsy.casemodule.Case; +import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; +import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeInstance; +import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeNormalizationException; +import org.sleuthkit.autopsy.centralrepository.datamodel.EamArtifactUtil; +import org.sleuthkit.autopsy.centralrepository.datamodel.EamDb; +import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbException; +import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbUtil; +import org.sleuthkit.autopsy.corecomponents.DataResultViewerTable; +import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.autopsy.texttranslation.NoServiceProviderException; +import org.sleuthkit.autopsy.texttranslation.TextTranslationService; +import org.sleuthkit.autopsy.texttranslation.TranslationException; +import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.datamodel.BlackboardArtifact; +import org.sleuthkit.datamodel.ContentTag; +import org.sleuthkit.datamodel.TskCoreException; +import org.sleuthkit.datamodel.TskData; + +/** + * + * @author dsmyda + */ +class PropertyUtil { + + private static final String NO_TRANSLATION = ""; + private static final Logger logger = Logger.getLogger(PropertyUtil.class.getName()); + + @NbBundle.Messages({ + "AbstractAbstractFileNode.createSheet.count.displayName=O", + "AbstractAbstractFileNode.createSheet.count.noCentralRepo.description=Central repository was not enabled when this column was populated", + "AbstractAbstractFileNode.createSheet.count.hashLookupNotRun.description=Hash lookup had not been run on this file when the column was populated", + "# {0} - occuranceCount", + "AbstractAbstractFileNode.createSheet.count.description=There were {0} datasource(s) found with occurances of the correlation value"}) + static Pair getCountPropertyAndDescription(CorrelationAttributeInstance attribute) { + Long count = -1L; //The column renderer will not display negative values, negative value used when count unavailble to preserve sorting + String description = Bundle.AbstractAbstractFileNode_createSheet_count_noCentralRepo_description(); + try { + //don't perform the query if there is no correlation value + if (attribute != null && StringUtils.isNotBlank(attribute.getCorrelationValue())) { + count = EamDb.getInstance().getCountUniqueCaseDataSourceTuplesHavingTypeValue(attribute.getCorrelationType(), attribute.getCorrelationValue()); + description = Bundle.AbstractAbstractFileNode_createSheet_count_description(count); + } else if (attribute != null) { + description = Bundle.AbstractAbstractFileNode_createSheet_count_hashLookupNotRun_description(); + } + } catch (EamDbException ex) { + logger.log(Level.WARNING, "Error getting count of datasources with correlation attribute", ex); + } catch (CorrelationAttributeNormalizationException ex) { + logger.log(Level.WARNING, "Unable to normalize data to get count of datasources with correlation attribute", ex); + } + + return Pair.of(count, description); + } + + + /** + * Used by subclasses of AbstractAbstractFileNode to add the Score property + * to their sheets. + * + * @param sheetSet the modifiable Sheet.Set returned by + * Sheet.get(Sheet.PROPERTIES) + * @param tags the list of tags associated with the file + */ + @NbBundle.Messages({ + "AbstractAbstractFileNode.createSheet.score.displayName=S", + "AbstractAbstractFileNode.createSheet.notableFile.description=File recognized as notable.", + "AbstractAbstractFileNode.createSheet.interestingResult.description=File has interesting result associated with it.", + "AbstractAbstractFileNode.createSheet.taggedFile.description=File has been tagged.", + "AbstractAbstractFileNode.createSheet.notableTaggedFile.description=File tagged with notable tag.", + "AbstractAbstractFileNode.createSheet.noScore.description=No score"}) + static Pair getScorePropertyAndDescription(AbstractFile content, List tags) { + DataResultViewerTable.Score score = DataResultViewerTable.Score.NO_SCORE; + String description = ""; + if (content.getKnown() == TskData.FileKnown.BAD) { + score = DataResultViewerTable.Score.NOTABLE_SCORE; + description = Bundle.AbstractAbstractFileNode_createSheet_notableFile_description(); + } + try { + if (score == DataResultViewerTable.Score.NO_SCORE && !content.getArtifacts(BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT).isEmpty()) { + score = DataResultViewerTable.Score.INTERESTING_SCORE; + description = Bundle.AbstractAbstractFileNode_createSheet_interestingResult_description(); + } + } catch (TskCoreException ex) { + logger.log(Level.WARNING, "Error getting artifacts for file: " + content.getName(), ex); + } + if (tags.size() > 0 && (score == DataResultViewerTable.Score.NO_SCORE || score == DataResultViewerTable.Score.INTERESTING_SCORE)) { + score = DataResultViewerTable.Score.INTERESTING_SCORE; + description = Bundle.AbstractAbstractFileNode_createSheet_taggedFile_description(); + for (ContentTag tag : tags) { + if (tag.getName().getKnownStatus() == TskData.FileKnown.BAD) { + score = DataResultViewerTable.Score.NOTABLE_SCORE; + description = Bundle.AbstractAbstractFileNode_createSheet_notableTaggedFile_description(); + break; + } + } + } + return Pair.of(score, description); + } + + /** + * Used by subclasses of AbstractAbstractFileNode to add the comment + * property to their sheets. + * + * @param sheetSet the modifiable Sheet.Set returned by + * Sheet.get(Sheet.PROPERTIES) + * @param tags the list of tags associated with the file + * @param attribute the correlation attribute associated with this file, + * null if central repo is not enabled + */ + @NbBundle.Messages({ + "AbstractAbstractFileNode.createSheet.comment.displayName=C"}) + static DataResultViewerTable.HasCommentStatus getCommentProperty(List tags, CorrelationAttributeInstance attribute) { + + DataResultViewerTable.HasCommentStatus status = tags.size() > 0 ? DataResultViewerTable.HasCommentStatus.TAG_NO_COMMENT : DataResultViewerTable.HasCommentStatus.NO_COMMENT; + + for (ContentTag tag : tags) { + if (!StringUtils.isBlank(tag.getComment())) { + //if the tag is null or empty or contains just white space it will indicate there is not a comment + status = DataResultViewerTable.HasCommentStatus.TAG_COMMENT; + break; + } + } + if (attribute != null && !StringUtils.isBlank(attribute.getComment())) { + if (status == DataResultViewerTable.HasCommentStatus.TAG_COMMENT) { + status = DataResultViewerTable.HasCommentStatus.CR_AND_TAG_COMMENTS; + } else { + status = DataResultViewerTable.HasCommentStatus.CR_COMMENT; + } + } + return status; + } + + /** + * Attempts translation of the content name being passed in. + * + * @return The file names translation. + */ + static String getTranslatedFileName(AbstractFile content) { + //If already in complete English, don't translate. + if (content.getName().matches("^\\p{ASCII}+$")) { + return NO_TRANSLATION; + } + + TextTranslationService tts = TextTranslationService.getInstance(); + if (tts.hasProvider()) { + //Seperate out the base and ext from the contents file name. + String base = FilenameUtils.getBaseName(content.getName()); + + try { + String translation = tts.translate(base); + String ext = FilenameUtils.getExtension(content.getName()); + + //If we have no extension, then we shouldn't add the . + String extensionDelimiter = (ext.isEmpty()) ? "" : "."; + + //Talk directly to this nodes pcl, fire an update when the translation + //is complete. + if (!translation.isEmpty()) { + return translation + extensionDelimiter + ext; + } + } catch (NoServiceProviderException noServiceEx) { + logger.log(Level.WARNING, "Translate unsuccessful because no TextTranslator " + + "implementation was provided.", noServiceEx); + } catch (TranslationException noTranslationEx) { + logger.log(Level.WARNING, "Could not successfully translate file name " + + content.getName(), noTranslationEx); + } + } + + return NO_TRANSLATION; + } + + /** + * Get all tags from the case database that are associated with the file + * + * @return a list of tags that are associated with the file + */ + static List getContentTagsFromDatabase(AbstractFile content) { + List tags = new ArrayList<>(); + try { + tags.addAll(Case.getCurrentCaseThrows().getServices().getTagsManager().getContentTagsByContent(content)); + } catch (TskCoreException | NoCurrentCaseException ex) { + logger.log(Level.SEVERE, "Failed to get tags for content " + content.getName(), ex); + } + return tags; + } + + static CorrelationAttributeInstance getCorrelationAttributeInstance(AbstractFile content) { + CorrelationAttributeInstance attribute = null; + if (EamDbUtil.useCentralRepo()) { + attribute = EamArtifactUtil.getInstanceFromContent(content); + } + return attribute; + } +} diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/SCOAndTranslationTask.java b/Core/src/org/sleuthkit/autopsy/datamodel/SCOAndTranslationTask.java new file mode 100755 index 0000000000..ac8ba2d851 --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/datamodel/SCOAndTranslationTask.java @@ -0,0 +1,144 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2018-2018 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.datamodel; + +import java.beans.PropertyChangeEvent; +import java.beans.PropertyChangeListener; +import java.lang.ref.WeakReference; +import java.util.List; +import org.apache.commons.lang3.tuple.Pair; +import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeInstance; +import org.sleuthkit.autopsy.corecomponents.DataResultViewerTable; +import org.sleuthkit.autopsy.corecomponents.DataResultViewerTable.HasCommentStatus; +import org.sleuthkit.autopsy.corecomponents.DataResultViewerTable.Score; +import org.sleuthkit.autopsy.events.AutopsyEvent; +import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.datamodel.ContentTag; + + +/** + * Completes the tasks needed to populate the Score, Comment, Occurrences and Translation + * columns in the background so that the UI is not blocked while waiting for responses from the database or + * translation service. Once these events are done, it fires a PropertyChangeEvent + * to let the AbstractAbstractFileNode know it's time to update! + */ +class SCOAndTranslationTask implements Runnable { + + private final WeakReference weakContentRef; + private final PropertyChangeListener listener; + + public SCOAndTranslationTask(WeakReference weakContentRef, PropertyChangeListener listener) { + this.weakContentRef = weakContentRef; + this.listener = listener; + } + + @Override + public void run() { + try { + AbstractFile content = weakContentRef.get(); + + //Long DB queries + List tags = PropertyUtil.getContentTagsFromDatabase(content); + CorrelationAttributeInstance attribute = + PropertyUtil.getCorrelationAttributeInstance(content); + + Pair scoreAndDescription = + PropertyUtil.getScorePropertyAndDescription(content, tags); + DataResultViewerTable.HasCommentStatus comment = + PropertyUtil.getCommentProperty(tags, attribute); + Pair countAndDescription = + PropertyUtil.getCountPropertyAndDescription(attribute); + + //Load the results from the SCO column operations into a wrapper object to be passed + //back to the listener so that the node can internally update it's propertySheet. + SCOResults results = new SCOResults( + scoreAndDescription.getLeft(), + scoreAndDescription.getRight(), + comment, + countAndDescription.getLeft(), + countAndDescription.getRight() + ); + + listener.propertyChange(new PropertyChangeEvent( + AutopsyEvent.SourceType.LOCAL.toString(), + AbstractAbstractFileNode.NodeSpecificEvents.DABABASE_CONTENT_AVAILABLE.toString(), + null, + results)); + + //Once we've got the SCO columns, then lets fire the translation result. + //Updating of this column is significantly lower priority than + //getting results to the SCO columns! + listener.propertyChange(new PropertyChangeEvent( + AutopsyEvent.SourceType.LOCAL.toString(), + AbstractAbstractFileNode.NodeSpecificEvents.TRANSLATION_AVAILABLE.toString(), + null, + PropertyUtil.getTranslatedFileName(content))); + } catch (NullPointerException ex) { + //If we are here, that means our weakPcl or content pointer has gone stale (aka + //has been garbage collected). There's no recovery. Netbeans has + //GC'd the node because its not useful to the user anymore. No need + //to log. Fail out fast to keep the thread pool rollin! + } + } + + /** + * Wrapper around data obtained from doing the Score, Comment, and Occurrences + * tasks. This object will be accessed by the AAFN to update it's state. + */ + final class SCOResults { + + private final Score score; + private final String scoreDescription; + + private final HasCommentStatus comment; + + private final Long count; + private final String countDescription; + + public SCOResults(Score score, String scoreDescription, + HasCommentStatus comment, Long count, + String countDescription) { + this.score = score; + this.scoreDescription = scoreDescription; + this.comment = comment; + this.count = count; + this.countDescription = countDescription; + } + + public Score getScore() { + return score; + } + + public String getScoreDescription() { + return scoreDescription; + } + + public HasCommentStatus getComment() { + return comment; + } + + public Long getCount() { + return count; + } + + public String getCountDescription() { + return countDescription; + } + } +} diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/VirtualDirectoryNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/VirtualDirectoryNode.java index 0c3bf4b48a..8907493015 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/VirtualDirectoryNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/VirtualDirectoryNode.java @@ -115,7 +115,7 @@ public class VirtualDirectoryNode extends SpecialDirectoryNode { } catch (SQLException | TskCoreException | NoCurrentCaseException ex) { logger.log(Level.SEVERE, "Failed to get device id for the following image: " + this.content.getId(), ex); } - + return sheet; } //Otherwise default to the AAFN createSheet method. From af3c213568a483559283061e913420341e6449be Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Thu, 1 Nov 2018 16:53:09 -0400 Subject: [PATCH 049/129] Realized that FileProperty is redundant, all we want is the ability to see if its enabled, so I refactored and did some light renaming too --- .../datamodel/AbstractAbstractFileNode.java | 505 +++++++++--------- .../autopsy/datamodel/FileProperty.java | 44 +- .../datamodel/SCOAndTranslationTask.java | 18 +- ...tyUtil.java => SCOAndTranslationUtil.java} | 7 +- 4 files changed, 260 insertions(+), 314 deletions(-) rename Core/src/org/sleuthkit/autopsy/datamodel/{PropertyUtil.java => SCOAndTranslationUtil.java} (98%) diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java index 9f5fb5cbad..f0375851ea 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java @@ -171,28 +171,21 @@ public abstract class AbstractAbstractFileNode extends A //No need to do any asynchrony around these events, they are so infrequent //and user driven that we can just keep a simple blocking approach, where we //go out to the database ourselves! - List tags = PropertyUtil.getContentTagsFromDatabase(content); - - updateProperty(new FileProperty(SCORE.toString()) { - Pair scorePropertyAndDescription - = PropertyUtil.getScorePropertyAndDescription(content, tags); - - @Override - public Object getPropertyValue() { - return scorePropertyAndDescription.getLeft(); - } - - @Override - public String getDescription() { - return scorePropertyAndDescription.getRight(); - } - }, new FileProperty(COMMENT.toString()) { - @Override - public Object getPropertyValue() { - //Null out the correlation attribute because we are only - //concerned with changes to the content tag, not the CR! - return PropertyUtil.getCommentProperty(tags, null); - } + List tags = SCOAndTranslationUtil.getContentTagsFromDatabase(content); + Pair scorePropertyAndDescription + = SCOAndTranslationUtil.getScorePropertyAndDescription(content, tags); + updateProperty(new FileProperty( + new NodeProperty<>( + SCORE.toString(), + SCORE.toString(), + scorePropertyAndDescription.getRight(), + scorePropertyAndDescription.getLeft())), + new FileProperty( + new NodeProperty<>( + COMMENT.toString(), + COMMENT.toString(), + NO_DESCR, + SCOAndTranslationUtil.getCommentProperty(tags, null))) { }); } } else if (eventType.equals(Case.Events.CONTENT_TAG_DELETED.toString())) { @@ -201,28 +194,21 @@ public abstract class AbstractAbstractFileNode extends A //No need to do any asynchrony around these events, they are so infrequent //and user driven that we can just keep a simple blocking approach, where we //go out to the database ourselves! - List tags = PropertyUtil.getContentTagsFromDatabase(content); - - updateProperty(new FileProperty(SCORE.toString()) { - Pair scorePropertyAndDescription - = PropertyUtil.getScorePropertyAndDescription(content, tags); - - @Override - public Object getPropertyValue() { - return scorePropertyAndDescription.getLeft(); - } - - @Override - public String getDescription() { - return scorePropertyAndDescription.getRight(); - } - }, new FileProperty(COMMENT.toString()) { - @Override - public Object getPropertyValue() { - //Null out the correlation attribute because we are only - //concerned with changes to the content tag, not the CR! - return PropertyUtil.getCommentProperty(tags, null); - } + List tags = SCOAndTranslationUtil.getContentTagsFromDatabase(content); + Pair scorePropertyAndDescription + = SCOAndTranslationUtil.getScorePropertyAndDescription(content, tags); + updateProperty(new FileProperty( + new NodeProperty<>( + SCORE.toString(), + SCORE.toString(), + scorePropertyAndDescription.getRight(), + scorePropertyAndDescription.getLeft())), + new FileProperty( + new NodeProperty<>( + COMMENT.toString(), + COMMENT.toString(), + NO_DESCR, + SCOAndTranslationUtil.getCommentProperty(tags, null))) { }); } } else if (eventType.equals(Case.Events.CR_COMMENT_CHANGED.toString())) { @@ -231,68 +217,71 @@ public abstract class AbstractAbstractFileNode extends A //No need to do any asynchrony around these events, they are so infrequent //and user driven that we can just keep a simple blocking approach, where we //go out to the database ourselves! - updateProperty(new FileProperty(COMMENT.toString()) { - @Override - public Object getPropertyValue() { - List tags = PropertyUtil.getContentTagsFromDatabase(content); - CorrelationAttributeInstance attribute = PropertyUtil.getCorrelationAttributeInstance(content); - return PropertyUtil.getCommentProperty(tags, attribute); - } + List tags = SCOAndTranslationUtil.getContentTagsFromDatabase(content); + CorrelationAttributeInstance attribute = + SCOAndTranslationUtil.getCorrelationAttributeInstance(content); + updateProperty(new FileProperty( + new NodeProperty<>( + COMMENT.toString(), + COMMENT.toString(), + NO_DESCR, + SCOAndTranslationUtil.getCommentProperty(tags, attribute))) { }); } } else if (eventType.equals(NodeSpecificEvents.TRANSLATION_AVAILABLE.toString())) { - updateProperty(new FileProperty(TRANSLATION.toString()) { + updateProperty(new FileProperty( + new NodeProperty<>(TRANSLATION.toString(), + TRANSLATION.toString(), + NO_DESCR, + evt.getNewValue())) { @Override - public Object getPropertyValue() { - return evt.getNewValue(); + public boolean isEnabled() { + return UserPreferences.displayTranslationFileNames(); } }); } else if (eventType.equals(NodeSpecificEvents.DABABASE_CONTENT_AVAILABLE.toString())) { SCOResults results = (SCOResults) evt.getNewValue(); - updateProperty(new FileProperty(SCORE.toString()) { - @Override - public Object getPropertyValue() { - return results.getScore(); - } - - @Override - public String getDescription() { - return results.getScoreDescription(); - } - }, new FileProperty(COMMENT.toString()) { - @Override - public Object getPropertyValue() { - return results.getComment(); - } - }, new FileProperty(OCCURRENCES.toString()) { - @Override - public Object getPropertyValue() { - return results.getCount(); - } - - @Override - public String getDescription() { - return results.getCountDescription(); - } + updateProperty(new FileProperty( + new NodeProperty<>( + SCORE.toString(), + SCORE.toString(), + results.getScoreDescription(), + results.getScore())), + new FileProperty( + new NodeProperty<>( + COMMENT.toString(), + COMMENT.toString(), + NO_DESCR, + results.getComment())), + new FileProperty( + new NodeProperty<>( + OCCURRENCES.toString(), + OCCURRENCES.toString(), + results.getCountDescription(), + results.getCount())) { + @Override + public boolean isEnabled() { + return !UserPreferences.hideCentralRepoCommentsAndOccurrences(); + } }); } }; - - /** - * We pass a weak reference wrapper around the listener to the event - * publisher. This allows Netbeans to delete the node when the user - * navigates to another part of the tree (previously, nodes were not being - * deleted because the event publisher was holding onto a strong reference - * to the listener. We need to hold onto the weak reference here to support - * unregistering of the listener in removeListeners() below. - */ + /** + * We pass a weak reference wrapper around the listener to the event + * publisher. This allows Netbeans to delete the node when the user + * navigates to another part of the tree (previously, nodes were not + * being deleted because the event publisher was holding onto a + * strong reference to the listener. We need to hold onto the weak + * reference here to support unregistering of the listener in + * removeListeners() below. + */ private final PropertyChangeListener weakPcl = WeakListeners.propertyChange(pcl, null); /** - * Returns a blank sheet to the caller, useful for giving subclasses the + * Returns a blank sheet to the caller, useful for giving subclasses the * ability to override createSheet() with their own implementation. - * - * @return + * + * @return */ protected Sheet getBlankSheet() { return super.createSheet(); @@ -305,9 +294,9 @@ public abstract class AbstractAbstractFileNode extends A * add their own (or omit some!) properties and we will not accidentally * disrupt their UI. * - * Race condition if not synchronized. Only one update should be applied at a time. - * The timing of currSheetSet.getProperties() could result in wrong/stale data - * being shown! + * Race condition if not synchronized. Only one update should be applied at + * a time. The timing of currSheetSet.getProperties() could result in + * wrong/stale data being shown! * * @param newProps New file property instances to be updated in the current * sheet. @@ -322,12 +311,8 @@ public abstract class AbstractAbstractFileNode extends A for (int i = 0; i < currProps.length; i++) { for (FileProperty property : newProps) { - if (currProps[i].getName().equals(property.getPropertyName())) { - currProps[i] = new NodeProperty<>( - property.getPropertyName(), - property.getPropertyName(), - property.getDescription(), - property.getPropertyValue()); + if (currProps[i].getName().equals(property.getProperty().getName()) && property.isEnabled()) { + currProps[i] = property.getProperty(); } } } @@ -340,9 +325,10 @@ public abstract class AbstractAbstractFileNode extends A } /* - * This is called when the node is first initialized. Any new updates or changes - * happen by directly manipulating the sheet. That means we can fire off background - * events everytime this method is called and not worry about duplicated jobs! + * This is called when the node is first initialized. Any new updates or + * changes happen by directly manipulating the sheet. That means we can fire + * off background events everytime this method is called and not worry about + * duplicated jobs! */ @Override protected synchronized Sheet createSheet() { @@ -356,11 +342,7 @@ public abstract class AbstractAbstractFileNode extends A //Add only the enabled properties to the sheet! for (FileProperty property : newProperties) { if (property.isEnabled()) { - sheetSet.put(new NodeProperty<>( - property.getPropertyName(), - property.getPropertyName(), - property.getDescription(), - property.getPropertyValue())); + sheetSet.put(property.getProperty()); } } @@ -444,183 +426,176 @@ public abstract class AbstractAbstractFileNode extends A */ List getProperties() { List properties = new ArrayList<>(); + properties.add(new FileProperty( + new NodeProperty<>( + NAME.toString(), + NAME.toString(), + NO_DESCR, + getContentDisplayName(content)))); - properties.add(new FileProperty(NAME.toString()) { - @Override - public Object getPropertyValue() { - return getContentDisplayName(content); - } - }); - //Initialize dummy place holder properties! These obviously do no work //to get their property values, but at the bottom we kick off a background //task that promises to update these values. final String NO_OP = ""; - properties.add(new FileProperty(TRANSLATION.toString()) { - @Override - public Object getPropertyValue() { - return NO_OP; - } - + properties.add(new FileProperty( + new NodeProperty<>(TRANSLATION.toString(), + TRANSLATION.toString(), + NO_DESCR, + NO_OP)) { @Override public boolean isEnabled() { return UserPreferences.displayTranslationFileNames(); } }); - - properties.add(new FileProperty(SCORE.toString()) { - @Override - public Object getPropertyValue() { - return NO_OP; - } + properties.add(new FileProperty( + new NodeProperty<>( + SCORE.toString(), + SCORE.toString(), + NO_DESCR, + NO_OP))); + properties.add(new FileProperty( + new NodeProperty<>( + COMMENT.toString(), + COMMENT.toString(), + NO_DESCR, + NO_OP)) { }); - properties.add(new FileProperty(COMMENT.toString()) { - @Override - public Object getPropertyValue() { - return NO_OP; - } + properties.add(new FileProperty( + new NodeProperty<>( + OCCURRENCES.toString(), + OCCURRENCES.toString(), + NO_DESCR, + NO_OP)) { @Override public boolean isEnabled() { return !UserPreferences.hideCentralRepoCommentsAndOccurrences(); } }); - properties.add(new FileProperty(OCCURRENCES.toString()) { - @Override - public Object getPropertyValue() { - return NO_OP; - } - - @Override - public boolean isEnabled() { - return !UserPreferences.hideCentralRepoCommentsAndOccurrences(); - } - }); - properties.add(new FileProperty(LOCATION.toString()) { - @Override - public Object getPropertyValue() { - return getContentPath(content); - } - }); - properties.add(new FileProperty(MOD_TIME.toString()) { - @Override - public Object getPropertyValue() { - return ContentUtils.getStringTime(content.getMtime(), content); - } - }); - properties.add(new FileProperty(CHANGED_TIME.toString()) { - @Override - public Object getPropertyValue() { - return ContentUtils.getStringTime(content.getCtime(), content); - } - }); - properties.add(new FileProperty(ACCESS_TIME.toString()) { - @Override - public Object getPropertyValue() { - return ContentUtils.getStringTime(content.getAtime(), content); - } - }); - properties.add(new FileProperty(CREATED_TIME.toString()) { - @Override - public Object getPropertyValue() { - return ContentUtils.getStringTime(content.getCrtime(), content); - } - }); - properties.add(new FileProperty(SIZE.toString()) { - @Override - public Object getPropertyValue() { - return content.getSize(); - } - }); - properties.add(new FileProperty(FLAGS_DIR.toString()) { - @Override - public Object getPropertyValue() { - return content.getDirFlagAsString(); - } - }); - properties.add(new FileProperty(FLAGS_META.toString()) { - @Override - public Object getPropertyValue() { - return content.getMetaFlagsAsString(); - } - }); - properties.add(new FileProperty(MODE.toString()) { - @Override - public Object getPropertyValue() { - return content.getModesAsString(); - } - }); - properties.add(new FileProperty(USER_ID.toString()) { - @Override - public Object getPropertyValue() { - return content.getUid(); - } - }); - properties.add(new FileProperty(GROUP_ID.toString()) { - @Override - public Object getPropertyValue() { - return content.getGid(); - } - }); - properties.add(new FileProperty(META_ADDR.toString()) { - @Override - public Object getPropertyValue() { - return content.getMetaAddr(); - } - }); - properties.add(new FileProperty(ATTR_ADDR.toString()) { - @Override - public Object getPropertyValue() { - return content.getAttrType().getValue() + "-" + content.getAttributeId(); - } - }); - properties.add(new FileProperty(TYPE_DIR.toString()) { - @Override - public Object getPropertyValue() { - return content.getDirType().getLabel(); - } - }); - properties.add(new FileProperty(TYPE_META.toString()) { - @Override - public Object getPropertyValue() { - return content.getMetaType().toString(); - } - }); - properties.add(new FileProperty(KNOWN.toString()) { - @Override - public Object getPropertyValue() { - return content.getKnown().getName(); - } - }); - properties.add(new FileProperty(MD5HASH.toString()) { - @Override - public Object getPropertyValue() { - return StringUtils.defaultString(content.getMd5Hash()); - } - }); - properties.add(new FileProperty(ObjectID.toString()) { - @Override - public Object getPropertyValue() { - return content.getId(); - } - }); - properties.add(new FileProperty(MIMETYPE.toString()) { - @Override - public Object getPropertyValue() { - return StringUtils.defaultString(content.getMIMEType()); - } - }); - properties.add(new FileProperty(EXTENSION.toString()) { - @Override - public Object getPropertyValue() { - return content.getNameExtension(); - } - }); + properties.add(new FileProperty( + new NodeProperty<>( + LOCATION.toString(), + LOCATION.toString(), + NO_DESCR, + getContentPath(content)))); + properties.add(new FileProperty( + new NodeProperty<>( + MOD_TIME.toString(), + MOD_TIME.toString(), + NO_DESCR, + ContentUtils.getStringTime(content.getMtime(), content)))); + properties.add(new FileProperty( + new NodeProperty<>( + CHANGED_TIME.toString(), + CHANGED_TIME.toString(), + NO_DESCR, + ContentUtils.getStringTime(content.getCtime(), content)))); + properties.add(new FileProperty( + new NodeProperty<>( + ACCESS_TIME.toString(), + ACCESS_TIME.toString(), + NO_DESCR, + ContentUtils.getStringTime(content.getAtime(), content)))); + properties.add(new FileProperty( + new NodeProperty<>( + CREATED_TIME.toString(), + CREATED_TIME.toString(), + NO_DESCR, + ContentUtils.getStringTime(content.getCrtime(), content)))); + properties.add(new FileProperty( + new NodeProperty<>( + SIZE.toString(), + SIZE.toString(), + NO_DESCR, + StringUtils.defaultString(content.getMIMEType())))); + properties.add(new FileProperty( + new NodeProperty<>( + FLAGS_DIR.toString(), + FLAGS_DIR.toString(), + NO_DESCR, + content.getSize()))); + properties.add(new FileProperty( + new NodeProperty<>( + FLAGS_META.toString(), + FLAGS_META.toString(), + NO_DESCR, + content.getMetaFlagsAsString()))); + properties.add(new FileProperty( + new NodeProperty<>( + MODE.toString(), + MODE.toString(), + NO_DESCR, + content.getModesAsString()))); + properties.add(new FileProperty( + new NodeProperty<>( + USER_ID.toString(), + USER_ID.toString(), + NO_DESCR, + content.getUid()))); + properties.add(new FileProperty( + new NodeProperty<>( + GROUP_ID.toString(), + GROUP_ID.toString(), + NO_DESCR, + content.getGid()))); + properties.add(new FileProperty( + new NodeProperty<>( + META_ADDR.toString(), + META_ADDR.toString(), + NO_DESCR, + content.getMetaAddr()))); + properties.add(new FileProperty( + new NodeProperty<>( + ATTR_ADDR.toString(), + ATTR_ADDR.toString(), + NO_DESCR, + content.getAttrType().getValue() + "-" + content.getAttributeId()))); + properties.add(new FileProperty( + new NodeProperty<>( + TYPE_DIR.toString(), + TYPE_DIR.toString(), + NO_DESCR, + content.getDirType().getLabel()))); + properties.add(new FileProperty( + new NodeProperty<>( + TYPE_META.toString(), + TYPE_META.toString(), + NO_DESCR, + content.getMetaType().toString()))); + properties.add(new FileProperty( + new NodeProperty<>( + KNOWN.toString(), + KNOWN.toString(), + NO_DESCR, + content.getKnown().getName()))); + properties.add(new FileProperty( + new NodeProperty<>( + MD5HASH.toString(), + MD5HASH.toString(), + NO_DESCR, + StringUtils.defaultString(content.getMd5Hash())))); + properties.add(new FileProperty( + new NodeProperty<>( + ObjectID.toString(), + ObjectID.toString(), + NO_DESCR, + content.getId()))); + properties.add(new FileProperty( + new NodeProperty<>( + MIMETYPE.toString(), + MIMETYPE.toString(), + NO_DESCR, + StringUtils.defaultString(content.getMIMEType())))); + properties.add(new FileProperty( + new NodeProperty<>( + EXTENSION.toString(), + EXTENSION.toString(), + NO_DESCR, + content.getNameExtension()))); //Submit the database queries ASAP! We want updated SCO columns //without blocking the UI as soon as we can get it! Keep all weak references - //so this task doesn't block the ability of this node to be GC'd. Handle potentially - //null reference values in the Task! + //so this task doesn't block the ability of this node to be GC'd. pool.submit(new SCOAndTranslationTask(new WeakReference<>(content), weakPcl)); return properties; } diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/FileProperty.java b/Core/src/org/sleuthkit/autopsy/datamodel/FileProperty.java index e002b91c7b..35bd0d5681 100755 --- a/Core/src/org/sleuthkit/autopsy/datamodel/FileProperty.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/FileProperty.java @@ -19,34 +19,22 @@ package org.sleuthkit.autopsy.datamodel; /** - * - * @author dsmyda + * Wraps a NodeProperty to extend its capability to also hold */ //No modifier means its is only package usable (not part of public api) -abstract class FileProperty { +class FileProperty { - private String description = ""; - private final String PROPERTY_NAME; + private final NodeProperty prop; - public FileProperty(String propertyName) { - PROPERTY_NAME = propertyName; + public FileProperty(NodeProperty prop) { + this.prop = prop; } - /** - * - * @param content - * @return - */ - public abstract Object getPropertyValue(); - - /** - * - * @return - */ - public String getPropertyName(){ - return PROPERTY_NAME; + public NodeProperty getProperty() { + return prop; } + /** * * @return @@ -54,20 +42,4 @@ abstract class FileProperty { public boolean isEnabled() { return true; } - - /* - * - */ - public void setDescription(String description) { - this.description = description; - } - - /** - * - * @param content - * @return - */ - public String getDescription() { - return description; - } } diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/SCOAndTranslationTask.java b/Core/src/org/sleuthkit/autopsy/datamodel/SCOAndTranslationTask.java index ac8ba2d851..4421707359 100755 --- a/Core/src/org/sleuthkit/autopsy/datamodel/SCOAndTranslationTask.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/SCOAndTranslationTask.java @@ -54,16 +54,16 @@ class SCOAndTranslationTask implements Runnable { AbstractFile content = weakContentRef.get(); //Long DB queries - List tags = PropertyUtil.getContentTagsFromDatabase(content); + List tags = SCOAndTranslationUtil.getContentTagsFromDatabase(content); CorrelationAttributeInstance attribute = - PropertyUtil.getCorrelationAttributeInstance(content); + SCOAndTranslationUtil.getCorrelationAttributeInstance(content); Pair scoreAndDescription = - PropertyUtil.getScorePropertyAndDescription(content, tags); + SCOAndTranslationUtil.getScorePropertyAndDescription(content, tags); DataResultViewerTable.HasCommentStatus comment = - PropertyUtil.getCommentProperty(tags, attribute); + SCOAndTranslationUtil.getCommentProperty(tags, attribute); Pair countAndDescription = - PropertyUtil.getCountPropertyAndDescription(attribute); + SCOAndTranslationUtil.getCountPropertyAndDescription(attribute); //Load the results from the SCO column operations into a wrapper object to be passed //back to the listener so that the node can internally update it's propertySheet. @@ -85,10 +85,10 @@ class SCOAndTranslationTask implements Runnable { //Updating of this column is significantly lower priority than //getting results to the SCO columns! listener.propertyChange(new PropertyChangeEvent( - AutopsyEvent.SourceType.LOCAL.toString(), - AbstractAbstractFileNode.NodeSpecificEvents.TRANSLATION_AVAILABLE.toString(), - null, - PropertyUtil.getTranslatedFileName(content))); + AutopsyEvent.SourceType.LOCAL.toString(), + AbstractAbstractFileNode.NodeSpecificEvents.TRANSLATION_AVAILABLE.toString(), + null, + SCOAndTranslationUtil.getTranslatedFileName(content))); } catch (NullPointerException ex) { //If we are here, that means our weakPcl or content pointer has gone stale (aka //has been garbage collected). There's no recovery. Netbeans has diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/PropertyUtil.java b/Core/src/org/sleuthkit/autopsy/datamodel/SCOAndTranslationUtil.java similarity index 98% rename from Core/src/org/sleuthkit/autopsy/datamodel/PropertyUtil.java rename to Core/src/org/sleuthkit/autopsy/datamodel/SCOAndTranslationUtil.java index f01f659627..9f3ce5fdd0 100755 --- a/Core/src/org/sleuthkit/autopsy/datamodel/PropertyUtil.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/SCOAndTranslationUtil.java @@ -45,13 +45,12 @@ import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskData; /** - * - * @author dsmyda + * */ -class PropertyUtil { +class SCOAndTranslationUtil { private static final String NO_TRANSLATION = ""; - private static final Logger logger = Logger.getLogger(PropertyUtil.class.getName()); + private static final Logger logger = Logger.getLogger(SCOAndTranslationUtil.class.getName()); @NbBundle.Messages({ "AbstractAbstractFileNode.createSheet.count.displayName=O", From a9df5391364e9f6c2c6d13392f8c517a582f844c Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Fri, 2 Nov 2018 09:04:12 -0400 Subject: [PATCH 050/129] Refactor of file names and methods --- .../datamodel/AbstractAbstractFileNode.java | 427 +++++++----------- ...TranslationUtil.java => FileNodeUtil.java} | 26 +- .../datamodel/SCOAndTranslationTask.java | 12 +- ...perty.java => ToggleableNodeProperty.java} | 28 +- 4 files changed, 220 insertions(+), 273 deletions(-) rename Core/src/org/sleuthkit/autopsy/datamodel/{SCOAndTranslationUtil.java => FileNodeUtil.java} (93%) rename Core/src/org/sleuthkit/autopsy/datamodel/{FileProperty.java => ToggleableNodeProperty.java} (51%) diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java index f0375851ea..6d08aa874d 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java @@ -22,7 +22,10 @@ import java.beans.PropertyChangeEvent; import java.beans.PropertyChangeListener; import java.lang.ref.WeakReference; import java.util.ArrayList; +import java.util.Arrays; import java.util.EnumSet; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -171,21 +174,18 @@ public abstract class AbstractAbstractFileNode extends A //No need to do any asynchrony around these events, they are so infrequent //and user driven that we can just keep a simple blocking approach, where we //go out to the database ourselves! - List tags = SCOAndTranslationUtil.getContentTagsFromDatabase(content); + List tags = FileNodeUtil.getContentTagsFromDatabase(content); Pair scorePropertyAndDescription - = SCOAndTranslationUtil.getScorePropertyAndDescription(content, tags); - updateProperty(new FileProperty( - new NodeProperty<>( - SCORE.toString(), + = FileNodeUtil.getScorePropertyAndDescription(content, tags); + updateProperty( + new ToggleableNodeProperty( SCORE.toString(), scorePropertyAndDescription.getRight(), - scorePropertyAndDescription.getLeft())), - new FileProperty( - new NodeProperty<>( - COMMENT.toString(), - COMMENT.toString(), - NO_DESCR, - SCOAndTranslationUtil.getCommentProperty(tags, null))) { + scorePropertyAndDescription.getLeft()), + new ToggleableNodeProperty( + COMMENT.toString(), + NO_DESCR, + FileNodeUtil.getCommentProperty(tags, null)) { }); } } else if (eventType.equals(Case.Events.CONTENT_TAG_DELETED.toString())) { @@ -194,22 +194,19 @@ public abstract class AbstractAbstractFileNode extends A //No need to do any asynchrony around these events, they are so infrequent //and user driven that we can just keep a simple blocking approach, where we //go out to the database ourselves! - List tags = SCOAndTranslationUtil.getContentTagsFromDatabase(content); + List tags = FileNodeUtil.getContentTagsFromDatabase(content); Pair scorePropertyAndDescription - = SCOAndTranslationUtil.getScorePropertyAndDescription(content, tags); - updateProperty(new FileProperty( - new NodeProperty<>( - SCORE.toString(), + = FileNodeUtil.getScorePropertyAndDescription(content, tags); + updateProperty( + new ToggleableNodeProperty( SCORE.toString(), scorePropertyAndDescription.getRight(), - scorePropertyAndDescription.getLeft())), - new FileProperty( - new NodeProperty<>( - COMMENT.toString(), - COMMENT.toString(), - NO_DESCR, - SCOAndTranslationUtil.getCommentProperty(tags, null))) { - }); + scorePropertyAndDescription.getLeft()), + new ToggleableNodeProperty( + COMMENT.toString(), + NO_DESCR, + FileNodeUtil.getCommentProperty(tags, null)) + ); } } else if (eventType.equals(Case.Events.CR_COMMENT_CHANGED.toString())) { CommentChangedEvent event = (CommentChangedEvent) evt; @@ -217,23 +214,21 @@ public abstract class AbstractAbstractFileNode extends A //No need to do any asynchrony around these events, they are so infrequent //and user driven that we can just keep a simple blocking approach, where we //go out to the database ourselves! - List tags = SCOAndTranslationUtil.getContentTagsFromDatabase(content); - CorrelationAttributeInstance attribute = - SCOAndTranslationUtil.getCorrelationAttributeInstance(content); - updateProperty(new FileProperty( - new NodeProperty<>( - COMMENT.toString(), + List tags = FileNodeUtil.getContentTagsFromDatabase(content); + CorrelationAttributeInstance attribute + = FileNodeUtil.getCorrelationAttributeInstance(content); + updateProperty( + new ToggleableNodeProperty( COMMENT.toString(), NO_DESCR, - SCOAndTranslationUtil.getCommentProperty(tags, attribute))) { - }); + FileNodeUtil.getCommentProperty(tags, attribute))); } } else if (eventType.equals(NodeSpecificEvents.TRANSLATION_AVAILABLE.toString())) { - updateProperty(new FileProperty( - new NodeProperty<>(TRANSLATION.toString(), + updateProperty( + new ToggleableNodeProperty( TRANSLATION.toString(), NO_DESCR, - evt.getNewValue())) { + evt.getNewValue()) { @Override public boolean isEnabled() { return UserPreferences.displayTranslationFileNames(); @@ -241,40 +236,34 @@ public abstract class AbstractAbstractFileNode extends A }); } else if (eventType.equals(NodeSpecificEvents.DABABASE_CONTENT_AVAILABLE.toString())) { SCOResults results = (SCOResults) evt.getNewValue(); - updateProperty(new FileProperty( - new NodeProperty<>( - SCORE.toString(), + updateProperty( + new ToggleableNodeProperty( SCORE.toString(), results.getScoreDescription(), - results.getScore())), - new FileProperty( - new NodeProperty<>( - COMMENT.toString(), - COMMENT.toString(), - NO_DESCR, - results.getComment())), - new FileProperty( - new NodeProperty<>( - OCCURRENCES.toString(), - OCCURRENCES.toString(), - results.getCountDescription(), - results.getCount())) { - @Override - public boolean isEnabled() { - return !UserPreferences.hideCentralRepoCommentsAndOccurrences(); - } + results.getScore()), + new ToggleableNodeProperty( + COMMENT.toString(), + NO_DESCR, + results.getComment()), + new ToggleableNodeProperty( + OCCURRENCES.toString(), + results.getCountDescription(), + results.getCount()) { + @Override + public boolean isEnabled() { + return !UserPreferences.hideCentralRepoCommentsAndOccurrences(); + } }); } }; - /** - * We pass a weak reference wrapper around the listener to the event - * publisher. This allows Netbeans to delete the node when the user - * navigates to another part of the tree (previously, nodes were not - * being deleted because the event publisher was holding onto a - * strong reference to the listener. We need to hold onto the weak - * reference here to support unregistering of the listener in - * removeListeners() below. - */ + /** + * We pass a weak reference wrapper around the listener to the event + * publisher. This allows Netbeans to delete the node when the user + * navigates to another part of the tree (previously, nodes were not being + * deleted because the event publisher was holding onto a strong reference + * to the listener. We need to hold onto the weak reference here to support + * unregistering of the listener in removeListeners() below. + */ private final PropertyChangeListener weakPcl = WeakListeners.propertyChange(pcl, null); /** @@ -301,19 +290,24 @@ public abstract class AbstractAbstractFileNode extends A * @param newProps New file property instances to be updated in the current * sheet. */ - private synchronized void updateProperty(FileProperty... newProps) { + private synchronized void updateProperty(ToggleableNodeProperty... newProps) { //Refresh ONLY those properties in the sheet currently. Subclasses may have //only added a subset of our properties or their own props! Let's keep their UI correct. Sheet currSheet = this.getSheet(); Sheet.Set currSheetSet = currSheet.get(Sheet.PROPERTIES); Property[] currProps = currSheetSet.getProperties(); + + Map newPropsMap = new HashMap<>(); + for(ToggleableNodeProperty property: newProps) { + newPropsMap.put(property.getName(), property); + } for (int i = 0; i < currProps.length; i++) { - for (FileProperty property : newProps) { - if (currProps[i].getName().equals(property.getProperty().getName()) && property.isEnabled()) { - currProps[i] = property.getProperty(); - } + String currentPropertyName = currProps[i].getName(); + if (newPropsMap.containsKey(currentPropertyName) && + newPropsMap.get(currentPropertyName).isEnabled()) { + currProps[i] = newPropsMap.get(currentPropertyName); } } @@ -337,12 +331,12 @@ public abstract class AbstractAbstractFileNode extends A sheet.put(sheetSet); //This will fire off fresh background tasks. - List newProperties = getProperties(); + List newProperties = getProperties(); //Add only the enabled properties to the sheet! - for (FileProperty property : newProperties) { + for (ToggleableNodeProperty property : newProperties) { if (property.isEnabled()) { - sheetSet.put(property.getProperty()); + sheetSet.put(property); } } @@ -417,181 +411,132 @@ public abstract class AbstractAbstractFileNode extends A /** * Creates a list of properties for this file node. Each property has its * own strategy for producing a value, its own description, name, and - * ability to be disabled. The FileProperty abstract class provides a - * wrapper for all of these characteristics. Additionally, with a return - * value of a list, any children classes of this node may reorder or omit - * any of these properties as they see fit for their use case. + * ability to be disabled. The ToggleableNodeProperty abstract class + * provides a wrapper for all of these characteristics. Additionally, with a + * return value of a list, any children classes of this node may reorder or + * omit any of these properties as they see fit for their use case. * * @return List of file properties associated with this file node's content. */ - List getProperties() { - List properties = new ArrayList<>(); - properties.add(new FileProperty( - new NodeProperty<>( - NAME.toString(), - NAME.toString(), - NO_DESCR, - getContentDisplayName(content)))); + List getProperties() { + List properties = new ArrayList<>(); + properties.add(new ToggleableNodeProperty( + NAME.toString(), + NO_DESCR, + FileNodeUtil.getContentDisplayName(content))); //Initialize dummy place holder properties! These obviously do no work //to get their property values, but at the bottom we kick off a background //task that promises to update these values. final String NO_OP = ""; - properties.add(new FileProperty( - new NodeProperty<>(TRANSLATION.toString(), - TRANSLATION.toString(), - NO_DESCR, - NO_OP)) { + properties.add(new ToggleableNodeProperty( + TRANSLATION.toString(), + NO_DESCR, + NO_OP) { @Override public boolean isEnabled() { return UserPreferences.displayTranslationFileNames(); } }); - properties.add(new FileProperty( - new NodeProperty<>( - SCORE.toString(), - SCORE.toString(), - NO_DESCR, - NO_OP))); - properties.add(new FileProperty( - new NodeProperty<>( - COMMENT.toString(), - COMMENT.toString(), - NO_DESCR, - NO_OP)) { + properties.add(new ToggleableNodeProperty( + SCORE.toString(), + NO_DESCR, + NO_OP)); + properties.add(new ToggleableNodeProperty( + COMMENT.toString(), + NO_DESCR, + NO_OP) { }); - properties.add(new FileProperty( - new NodeProperty<>( - OCCURRENCES.toString(), - OCCURRENCES.toString(), - NO_DESCR, - NO_OP)) { + properties.add(new ToggleableNodeProperty( + OCCURRENCES.toString(), + NO_DESCR, + NO_OP) { @Override public boolean isEnabled() { return !UserPreferences.hideCentralRepoCommentsAndOccurrences(); } }); - properties.add(new FileProperty( - new NodeProperty<>( - LOCATION.toString(), - LOCATION.toString(), - NO_DESCR, - getContentPath(content)))); - properties.add(new FileProperty( - new NodeProperty<>( - MOD_TIME.toString(), - MOD_TIME.toString(), - NO_DESCR, - ContentUtils.getStringTime(content.getMtime(), content)))); - properties.add(new FileProperty( - new NodeProperty<>( - CHANGED_TIME.toString(), - CHANGED_TIME.toString(), - NO_DESCR, - ContentUtils.getStringTime(content.getCtime(), content)))); - properties.add(new FileProperty( - new NodeProperty<>( - ACCESS_TIME.toString(), - ACCESS_TIME.toString(), - NO_DESCR, - ContentUtils.getStringTime(content.getAtime(), content)))); - properties.add(new FileProperty( - new NodeProperty<>( - CREATED_TIME.toString(), - CREATED_TIME.toString(), - NO_DESCR, - ContentUtils.getStringTime(content.getCrtime(), content)))); - properties.add(new FileProperty( - new NodeProperty<>( - SIZE.toString(), - SIZE.toString(), - NO_DESCR, - StringUtils.defaultString(content.getMIMEType())))); - properties.add(new FileProperty( - new NodeProperty<>( - FLAGS_DIR.toString(), - FLAGS_DIR.toString(), - NO_DESCR, - content.getSize()))); - properties.add(new FileProperty( - new NodeProperty<>( - FLAGS_META.toString(), - FLAGS_META.toString(), - NO_DESCR, - content.getMetaFlagsAsString()))); - properties.add(new FileProperty( - new NodeProperty<>( - MODE.toString(), - MODE.toString(), - NO_DESCR, - content.getModesAsString()))); - properties.add(new FileProperty( - new NodeProperty<>( - USER_ID.toString(), - USER_ID.toString(), - NO_DESCR, - content.getUid()))); - properties.add(new FileProperty( - new NodeProperty<>( - GROUP_ID.toString(), - GROUP_ID.toString(), - NO_DESCR, - content.getGid()))); - properties.add(new FileProperty( - new NodeProperty<>( - META_ADDR.toString(), - META_ADDR.toString(), - NO_DESCR, - content.getMetaAddr()))); - properties.add(new FileProperty( - new NodeProperty<>( - ATTR_ADDR.toString(), - ATTR_ADDR.toString(), - NO_DESCR, - content.getAttrType().getValue() + "-" + content.getAttributeId()))); - properties.add(new FileProperty( - new NodeProperty<>( - TYPE_DIR.toString(), - TYPE_DIR.toString(), - NO_DESCR, - content.getDirType().getLabel()))); - properties.add(new FileProperty( - new NodeProperty<>( - TYPE_META.toString(), - TYPE_META.toString(), - NO_DESCR, - content.getMetaType().toString()))); - properties.add(new FileProperty( - new NodeProperty<>( - KNOWN.toString(), - KNOWN.toString(), - NO_DESCR, - content.getKnown().getName()))); - properties.add(new FileProperty( - new NodeProperty<>( - MD5HASH.toString(), - MD5HASH.toString(), - NO_DESCR, - StringUtils.defaultString(content.getMd5Hash())))); - properties.add(new FileProperty( - new NodeProperty<>( - ObjectID.toString(), - ObjectID.toString(), - NO_DESCR, - content.getId()))); - properties.add(new FileProperty( - new NodeProperty<>( - MIMETYPE.toString(), - MIMETYPE.toString(), - NO_DESCR, - StringUtils.defaultString(content.getMIMEType())))); - properties.add(new FileProperty( - new NodeProperty<>( - EXTENSION.toString(), - EXTENSION.toString(), - NO_DESCR, - content.getNameExtension()))); + properties.add(new ToggleableNodeProperty( + LOCATION.toString(), + NO_DESCR, + FileNodeUtil.getContentPath(content))); + properties.add(new ToggleableNodeProperty( + MOD_TIME.toString(), + NO_DESCR, + ContentUtils.getStringTime(content.getMtime(), content))); + properties.add(new ToggleableNodeProperty( + CHANGED_TIME.toString(), + NO_DESCR, + ContentUtils.getStringTime(content.getCtime(), content))); + properties.add(new ToggleableNodeProperty( + ACCESS_TIME.toString(), + NO_DESCR, + ContentUtils.getStringTime(content.getAtime(), content))); + properties.add(new ToggleableNodeProperty( + CREATED_TIME.toString(), + NO_DESCR, + ContentUtils.getStringTime(content.getCrtime(), content))); + properties.add(new ToggleableNodeProperty( + SIZE.toString(), + NO_DESCR, + StringUtils.defaultString(content.getMIMEType()))); + properties.add(new ToggleableNodeProperty( + FLAGS_DIR.toString(), + NO_DESCR, + content.getSize())); + properties.add(new ToggleableNodeProperty( + FLAGS_META.toString(), + NO_DESCR, + content.getMetaFlagsAsString())); + properties.add(new ToggleableNodeProperty( + MODE.toString(), + NO_DESCR, + content.getModesAsString())); + properties.add(new ToggleableNodeProperty( + USER_ID.toString(), + NO_DESCR, + content.getUid())); + properties.add(new ToggleableNodeProperty( + GROUP_ID.toString(), + NO_DESCR, + content.getGid())); + properties.add(new ToggleableNodeProperty( + META_ADDR.toString(), + NO_DESCR, + content.getMetaAddr())); + properties.add(new ToggleableNodeProperty( + ATTR_ADDR.toString(), + NO_DESCR, + content.getAttrType().getValue() + "-" + content.getAttributeId())); + properties.add(new ToggleableNodeProperty( + TYPE_DIR.toString(), + NO_DESCR, + content.getDirType().getLabel())); + properties.add(new ToggleableNodeProperty( + TYPE_META.toString(), + NO_DESCR, + content.getMetaType().toString())); + properties.add(new ToggleableNodeProperty( + KNOWN.toString(), + NO_DESCR, + content.getKnown().getName())); + properties.add(new ToggleableNodeProperty( + MD5HASH.toString(), + NO_DESCR, + StringUtils.defaultString(content.getMd5Hash()))); + properties.add(new ToggleableNodeProperty( + ObjectID.toString(), + NO_DESCR, + content.getId())); + properties.add(new ToggleableNodeProperty( + MIMETYPE.toString(), + NO_DESCR, + StringUtils.defaultString(content.getMIMEType()))); + properties.add(new ToggleableNodeProperty( + EXTENSION.toString(), + NO_DESCR, + content.getNameExtension())); //Submit the database queries ASAP! We want updated SCO columns //without blocking the UI as soon as we can get it! Keep all weak references @@ -624,28 +569,6 @@ public abstract class AbstractAbstractFileNode extends A .collect(Collectors.joining(", ")))); } - private static String getContentPath(AbstractFile file) { - try { - return file.getUniquePath(); - } catch (TskCoreException ex) { - logger.log(Level.SEVERE, "Except while calling Content.getUniquePath() on " + file, ex); //NON-NLS - return ""; //NON-NLS - } - } - - static String getContentDisplayName(AbstractFile file) { - String name = file.getName(); - switch (name) { - case "..": - return DirectoryNode.DOTDOTDIR; - - case ".": - return DirectoryNode.DOTDIR; - default: - return name; - } - } - /** * Gets a comma-separated values list of the names of the hash sets * currently identified as including a given file. @@ -675,8 +598,8 @@ public abstract class AbstractAbstractFileNode extends A */ @Deprecated static public void fillPropertyMap(Map map, AbstractFile content) { - map.put(NAME.toString(), getContentDisplayName(content)); - map.put(LOCATION.toString(), getContentPath(content)); + map.put(NAME.toString(), FileNodeUtil.getContentDisplayName(content)); + map.put(LOCATION.toString(), FileNodeUtil.getContentPath(content)); map.put(MOD_TIME.toString(), ContentUtils.getStringTime(content.getMtime(), content)); map.put(CHANGED_TIME.toString(), ContentUtils.getStringTime(content.getCtime(), content)); map.put(ACCESS_TIME.toString(), ContentUtils.getStringTime(content.getAtime(), content)); diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/SCOAndTranslationUtil.java b/Core/src/org/sleuthkit/autopsy/datamodel/FileNodeUtil.java similarity index 93% rename from Core/src/org/sleuthkit/autopsy/datamodel/SCOAndTranslationUtil.java rename to Core/src/org/sleuthkit/autopsy/datamodel/FileNodeUtil.java index 9f3ce5fdd0..08ba208b95 100755 --- a/Core/src/org/sleuthkit/autopsy/datamodel/SCOAndTranslationUtil.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/FileNodeUtil.java @@ -47,10 +47,10 @@ import org.sleuthkit.datamodel.TskData; /** * */ -class SCOAndTranslationUtil { +class FileNodeUtil { private static final String NO_TRANSLATION = ""; - private static final Logger logger = Logger.getLogger(SCOAndTranslationUtil.class.getName()); + private static final Logger logger = Logger.getLogger(FileNodeUtil.class.getName()); @NbBundle.Messages({ "AbstractAbstractFileNode.createSheet.count.displayName=O", @@ -218,4 +218,26 @@ class SCOAndTranslationUtil { } return attribute; } + + static String getContentPath(AbstractFile file) { + try { + return file.getUniquePath(); + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, "Except while calling Content.getUniquePath() on " + file, ex); //NON-NLS + return ""; //NON-NLS + } + } + + static String getContentDisplayName(AbstractFile file) { + String name = file.getName(); + switch (name) { + case "..": + return DirectoryNode.DOTDOTDIR; + + case ".": + return DirectoryNode.DOTDIR; + default: + return name; + } + } } diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/SCOAndTranslationTask.java b/Core/src/org/sleuthkit/autopsy/datamodel/SCOAndTranslationTask.java index 4421707359..43dfa783d7 100755 --- a/Core/src/org/sleuthkit/autopsy/datamodel/SCOAndTranslationTask.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/SCOAndTranslationTask.java @@ -54,16 +54,16 @@ class SCOAndTranslationTask implements Runnable { AbstractFile content = weakContentRef.get(); //Long DB queries - List tags = SCOAndTranslationUtil.getContentTagsFromDatabase(content); + List tags = FileNodeUtil.getContentTagsFromDatabase(content); CorrelationAttributeInstance attribute = - SCOAndTranslationUtil.getCorrelationAttributeInstance(content); + FileNodeUtil.getCorrelationAttributeInstance(content); Pair scoreAndDescription = - SCOAndTranslationUtil.getScorePropertyAndDescription(content, tags); + FileNodeUtil.getScorePropertyAndDescription(content, tags); DataResultViewerTable.HasCommentStatus comment = - SCOAndTranslationUtil.getCommentProperty(tags, attribute); + FileNodeUtil.getCommentProperty(tags, attribute); Pair countAndDescription = - SCOAndTranslationUtil.getCountPropertyAndDescription(attribute); + FileNodeUtil.getCountPropertyAndDescription(attribute); //Load the results from the SCO column operations into a wrapper object to be passed //back to the listener so that the node can internally update it's propertySheet. @@ -88,7 +88,7 @@ class SCOAndTranslationTask implements Runnable { AutopsyEvent.SourceType.LOCAL.toString(), AbstractAbstractFileNode.NodeSpecificEvents.TRANSLATION_AVAILABLE.toString(), null, - SCOAndTranslationUtil.getTranslatedFileName(content))); + FileNodeUtil.getTranslatedFileName(content))); } catch (NullPointerException ex) { //If we are here, that means our weakPcl or content pointer has gone stale (aka //has been garbage collected). There's no recovery. Netbeans has diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/FileProperty.java b/Core/src/org/sleuthkit/autopsy/datamodel/ToggleableNodeProperty.java similarity index 51% rename from Core/src/org/sleuthkit/autopsy/datamodel/FileProperty.java rename to Core/src/org/sleuthkit/autopsy/datamodel/ToggleableNodeProperty.java index 35bd0d5681..4e65576dcb 100755 --- a/Core/src/org/sleuthkit/autopsy/datamodel/FileProperty.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/ToggleableNodeProperty.java @@ -19,25 +19,27 @@ package org.sleuthkit.autopsy.datamodel; /** - * Wraps a NodeProperty to extend its capability to also hold + * Adds the functionality of enabling and disabling a NodeProperty (column in the UI). */ +class ToggleableNodeProperty extends NodeProperty { -//No modifier means its is only package usable (not part of public api) -class FileProperty { - - private final NodeProperty prop; - - public FileProperty(NodeProperty prop) { - this.prop = prop; - } - - public NodeProperty getProperty() { - return prop; + /** + * Wraps the super constructor. In our use cases, we want the name and display + * name of the column to be the exact same, so to avoid redundancy we accept the name + * just once and pass it twice to the NodeProperty. + * + * @param name Name of the property to be displayed + * @param desc Description of the property when hovering over the column + * @param value Value to be displayed in that column + */ + public ToggleableNodeProperty(String name, String desc, Object value) { + super(name, name, desc, value); } /** * - * @return + * + * @return boolean denoting the availiability of this property. True by default. */ public boolean isEnabled() { return true; From 2bfec7517f792001239e0517a5dcd676f462885d Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Fri, 2 Nov 2018 09:31:23 -0400 Subject: [PATCH 051/129] Tested these refactors and everything is working correctly. --- .../autopsy/datamodel/AbstractAbstractFileNode.java | 10 ++++++---- .../autopsy/datamodel/AbstractFsContentNode.java | 2 +- .../autopsy/datamodel/ToggleableNodeProperty.java | 3 ++- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java index 6d08aa874d..19d46616d3 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java @@ -22,10 +22,8 @@ import java.beans.PropertyChangeEvent; import java.beans.PropertyChangeListener; import java.lang.ref.WeakReference; import java.util.ArrayList; -import java.util.Arrays; import java.util.EnumSet; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -177,6 +175,8 @@ public abstract class AbstractAbstractFileNode extends A List tags = FileNodeUtil.getContentTagsFromDatabase(content); Pair scorePropertyAndDescription = FileNodeUtil.getScorePropertyAndDescription(content, tags); + CorrelationAttributeInstance attribute = + FileNodeUtil.getCorrelationAttributeInstance(content); updateProperty( new ToggleableNodeProperty( SCORE.toString(), @@ -185,7 +185,7 @@ public abstract class AbstractAbstractFileNode extends A new ToggleableNodeProperty( COMMENT.toString(), NO_DESCR, - FileNodeUtil.getCommentProperty(tags, null)) { + FileNodeUtil.getCommentProperty(tags, attribute)) { }); } } else if (eventType.equals(Case.Events.CONTENT_TAG_DELETED.toString())) { @@ -197,6 +197,8 @@ public abstract class AbstractAbstractFileNode extends A List tags = FileNodeUtil.getContentTagsFromDatabase(content); Pair scorePropertyAndDescription = FileNodeUtil.getScorePropertyAndDescription(content, tags); + CorrelationAttributeInstance attribute = + FileNodeUtil.getCorrelationAttributeInstance(content); updateProperty( new ToggleableNodeProperty( SCORE.toString(), @@ -205,7 +207,7 @@ public abstract class AbstractAbstractFileNode extends A new ToggleableNodeProperty( COMMENT.toString(), NO_DESCR, - FileNodeUtil.getCommentProperty(tags, null)) + FileNodeUtil.getCommentProperty(tags, attribute)) ); } } else if (eventType.equals(Case.Events.CR_COMMENT_CHANGED.toString())) { diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractFsContentNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractFsContentNode.java index 816502dd8d..559acb451f 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractFsContentNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractFsContentNode.java @@ -51,7 +51,7 @@ public abstract class AbstractFsContentNode extends Abst */ AbstractFsContentNode(T content, boolean directoryBrowseMode) { super(content); - this.setDisplayName(AbstractAbstractFileNode.getContentDisplayName(content)); + this.setDisplayName(FileNodeUtil.getContentDisplayName(content)); this.directoryBrowseMode = directoryBrowseMode; } diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/ToggleableNodeProperty.java b/Core/src/org/sleuthkit/autopsy/datamodel/ToggleableNodeProperty.java index 4e65576dcb..78e2b237f5 100755 --- a/Core/src/org/sleuthkit/autopsy/datamodel/ToggleableNodeProperty.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/ToggleableNodeProperty.java @@ -37,7 +37,8 @@ class ToggleableNodeProperty extends NodeProperty { } /** - * + * Allows a property to be either enabled or disabled. When creating a sheet, + * this method is used to filter out from displaying in the UI. * * @return boolean denoting the availiability of this property. True by default. */ From 4dcf88fc2c72e380ebd04187ccc35cfd1aca88f3 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Fri, 2 Nov 2018 09:47:24 -0400 Subject: [PATCH 052/129] Fixed Cr node instance to depend on AAFN --- .../CaseDBCommonAttributeInstanceNode.java | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CaseDBCommonAttributeInstanceNode.java b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CaseDBCommonAttributeInstanceNode.java index 0b528239a2..b7d1e77f3c 100644 --- a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CaseDBCommonAttributeInstanceNode.java +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CaseDBCommonAttributeInstanceNode.java @@ -22,6 +22,8 @@ import java.util.Arrays; import java.util.HashSet; import java.util.Set; import org.openide.nodes.Sheet; +import org.openide.util.NbBundle; +import org.sleuthkit.autopsy.datamodel.AbstractAbstractFileNode; import org.sleuthkit.autopsy.datamodel.DisplayableItemNodeVisitor; import org.sleuthkit.autopsy.datamodel.FileNode; import org.sleuthkit.autopsy.datamodel.NodeProperty; @@ -74,14 +76,16 @@ public class CaseDBCommonAttributeInstanceNode extends FileNode { protected Sheet createSheet() { Sheet sheet = super.createSheet(); Sheet.Set sheetSet = sheet.get(Sheet.PROPERTIES); - Set keepProps = new HashSet<>(Arrays.asList("S", - "C", "O", "Mime Type")); + Set keepProps = new HashSet<>(Arrays.asList( + NbBundle.getMessage(AbstractAbstractFileNode.class, "AbstractAbstractFileNode.createSheet.score.name"), + NbBundle.getMessage(AbstractAbstractFileNode.class, "AbstractAbstractFileNode.createSheet.comment.name"), + NbBundle.getMessage(AbstractAbstractFileNode.class, "AbstractAbstractFileNode.createSheet.count.name"), + NbBundle.getMessage(AbstractAbstractFileNode.class, "AbstractAbstractFileNode.mimeType"))); for(Property p : sheetSet.getProperties()) { - if(keepProps.contains(p.getName())){ - continue; + if(!keepProps.contains(p.getName())){ + sheetSet.remove(p.getName()); } - sheetSet.remove(p.getName()); } final String NO_DESCR = Bundle.CommonFilesSearchResultsViewerTable_noDescText(); From 12adee375ed1153f753f83cc849844e5d297e027 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Fri, 2 Nov 2018 09:58:39 -0400 Subject: [PATCH 053/129] Did the same for the MessageContentViewer --- .../contentviewers/MessageContentViewer.java | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/contentviewers/MessageContentViewer.java b/Core/src/org/sleuthkit/autopsy/contentviewers/MessageContentViewer.java index 4210d2eb92..bf319bde0c 100644 --- a/Core/src/org/sleuthkit/autopsy/contentviewers/MessageContentViewer.java +++ b/Core/src/org/sleuthkit/autopsy/contentviewers/MessageContentViewer.java @@ -44,6 +44,7 @@ import org.sleuthkit.autopsy.corecomponentinterfaces.DataContentViewer; import org.sleuthkit.autopsy.corecomponents.DataResultPanel; import org.sleuthkit.autopsy.corecomponents.TableFilterNode; import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.autopsy.datamodel.AbstractAbstractFileNode; import org.sleuthkit.autopsy.datamodel.FileNode; import org.sleuthkit.autopsy.datamodel.NodeProperty; import org.sleuthkit.autopsy.directorytree.DataResultFilterNode; @@ -723,16 +724,21 @@ public class MessageContentViewer extends javax.swing.JPanel implements DataCont @Override protected Sheet createSheet() { Sheet sheet = super.createSheet(); - Set keepProps = new HashSet<>(Arrays.asList("Name" , "S", - "C", "O", "Size", "Mime Type", "Known")); + Set keepProps = new HashSet<>(Arrays.asList( + NbBundle.getMessage(AbstractAbstractFileNode.class, "AbstractAbstractFileNode.nameColLbl"), + NbBundle.getMessage(AbstractAbstractFileNode.class, "AbstractAbstractFileNode.createSheet.score.name"), + NbBundle.getMessage(AbstractAbstractFileNode.class, "AbstractAbstractFileNode.createSheet.comment.name"), + NbBundle.getMessage(AbstractAbstractFileNode.class, "AbstractAbstractFileNode.createSheet.count.name"), + NbBundle.getMessage(AbstractAbstractFileNode.class, "AbstractAbstractFileNode.sizeColLbl"), + NbBundle.getMessage(AbstractAbstractFileNode.class, "AbstractAbstractFileNode.mimeType"), + NbBundle.getMessage(AbstractAbstractFileNode.class, "AbstractAbstractFileNode.knownColLbl"))); //Remove all other props except for the ones above Sheet.Set sheetSet = sheet.get(Sheet.PROPERTIES); for(Property p : sheetSet.getProperties()) { - if(keepProps.contains(p.getName())){ - continue; + if(!keepProps.contains(p.getName())){ + sheetSet.remove(p.getName()); } - sheetSet.remove(p.getName()); } return sheet; From 70cb50b3597f9156b04e90f8ad44fa1fa21161ff Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Fri, 2 Nov 2018 10:01:57 -0400 Subject: [PATCH 054/129] Realized I added an unnecessary method to public API --- .../autopsy/datamodel/AbstractAbstractFileNode.java | 12 +----------- .../autopsy/datamodel/VirtualDirectoryNode.java | 2 +- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java index 19d46616d3..a13bc3515b 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java @@ -268,16 +268,6 @@ public abstract class AbstractAbstractFileNode extends A */ private final PropertyChangeListener weakPcl = WeakListeners.propertyChange(pcl, null); - /** - * Returns a blank sheet to the caller, useful for giving subclasses the - * ability to override createSheet() with their own implementation. - * - * @return - */ - protected Sheet getBlankSheet() { - return super.createSheet(); - } - /** * Updates the values of the properties in the current property sheet with * the new properties being passed in! Only if that property exists in the @@ -328,7 +318,7 @@ public abstract class AbstractAbstractFileNode extends A */ @Override protected synchronized Sheet createSheet() { - Sheet sheet = getBlankSheet(); + Sheet sheet = new Sheet(); Sheet.Set sheetSet = Sheet.createPropertiesSet(); sheet.put(sheetSet); diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/VirtualDirectoryNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/VirtualDirectoryNode.java index 8907493015..19e8950d33 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/VirtualDirectoryNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/VirtualDirectoryNode.java @@ -75,7 +75,7 @@ public class VirtualDirectoryNode extends SpecialDirectoryNode { protected Sheet createSheet() { //Do a special strategy for virtual directories.. if(this.content.isDataSource()){ - Sheet sheet = getBlankSheet(); + Sheet sheet = new Sheet(); Sheet.Set sheetSet = Sheet.createPropertiesSet(); sheet.put(sheetSet); From 805f002bfc33a75cb1bc426249501b03b6fe27fc Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Fri, 2 Nov 2018 10:15:51 -0400 Subject: [PATCH 055/129] Added name back in --- .../commonfilesearch/CaseDBCommonAttributeInstanceNode.java | 1 + 1 file changed, 1 insertion(+) diff --git a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CaseDBCommonAttributeInstanceNode.java b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CaseDBCommonAttributeInstanceNode.java index b7d1e77f3c..504aa45b97 100644 --- a/Core/src/org/sleuthkit/autopsy/commonfilesearch/CaseDBCommonAttributeInstanceNode.java +++ b/Core/src/org/sleuthkit/autopsy/commonfilesearch/CaseDBCommonAttributeInstanceNode.java @@ -77,6 +77,7 @@ public class CaseDBCommonAttributeInstanceNode extends FileNode { Sheet sheet = super.createSheet(); Sheet.Set sheetSet = sheet.get(Sheet.PROPERTIES); Set keepProps = new HashSet<>(Arrays.asList( + NbBundle.getMessage(AbstractAbstractFileNode.class, "AbstractAbstractFileNode.nameColLbl"), NbBundle.getMessage(AbstractAbstractFileNode.class, "AbstractAbstractFileNode.createSheet.score.name"), NbBundle.getMessage(AbstractAbstractFileNode.class, "AbstractAbstractFileNode.createSheet.comment.name"), NbBundle.getMessage(AbstractAbstractFileNode.class, "AbstractAbstractFileNode.createSheet.count.name"), From 79d886bbb54800343fe904ab206cc25f478e9d0e Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Fri, 2 Nov 2018 10:23:17 -0400 Subject: [PATCH 056/129] Added final comments --- .../autopsy/datamodel/AbstractAbstractFileNode.java | 11 +++++------ .../org/sleuthkit/autopsy/datamodel/FileNodeUtil.java | 3 ++- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java index a13bc3515b..85a6099f73 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java @@ -401,12 +401,11 @@ public abstract class AbstractAbstractFileNode extends A } /** - * Creates a list of properties for this file node. Each property has its - * own strategy for producing a value, its own description, name, and - * ability to be disabled. The ToggleableNodeProperty abstract class - * provides a wrapper for all of these characteristics. Additionally, with a - * return value of a list, any children classes of this node may reorder or - * omit any of these properties as they see fit for their use case. + * Creates a list of properties for this file node. ToggleableNodeProperty + * is a subclass of NodeProperty, with the added functionality of being able to be + * enabled and disabled. Disabled properties don't get added to the sheet! + * Additionally, with a return value of a list, any children classes of this + * node may reorder or omit any of these properties as they see fit for their use case. * * @return List of file properties associated with this file node's content. */ diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/FileNodeUtil.java b/Core/src/org/sleuthkit/autopsy/datamodel/FileNodeUtil.java index 08ba208b95..6eb2bd18c7 100755 --- a/Core/src/org/sleuthkit/autopsy/datamodel/FileNodeUtil.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/FileNodeUtil.java @@ -45,7 +45,8 @@ import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskData; /** - * + * Utility class for getting common data about an AbstractFile, such as content tags + * correlation attributes, content paths and SCO values, to name a few. */ class FileNodeUtil { From 3c0384e0e0043e1257a261bc036356698c703db8 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Fri, 2 Nov 2018 10:30:45 -0400 Subject: [PATCH 057/129] Fixed warnings in log --- .../org/sleuthkit/autopsy/datamodel/ToggleableNodeProperty.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/ToggleableNodeProperty.java b/Core/src/org/sleuthkit/autopsy/datamodel/ToggleableNodeProperty.java index 78e2b237f5..5272c8b205 100755 --- a/Core/src/org/sleuthkit/autopsy/datamodel/ToggleableNodeProperty.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/ToggleableNodeProperty.java @@ -21,7 +21,7 @@ package org.sleuthkit.autopsy.datamodel; /** * Adds the functionality of enabling and disabling a NodeProperty (column in the UI). */ -class ToggleableNodeProperty extends NodeProperty { +class ToggleableNodeProperty extends NodeProperty { /** * Wraps the super constructor. In our use cases, we want the name and display From 767f90b4b59390481ff99847b8cd11841129e2ea Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Fri, 2 Nov 2018 11:46:17 -0400 Subject: [PATCH 058/129] Added a final note about synchonization --- .../sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java index 85a6099f73..7e207eb281 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java @@ -406,6 +406,9 @@ public abstract class AbstractAbstractFileNode extends A * enabled and disabled. Disabled properties don't get added to the sheet! * Additionally, with a return value of a list, any children classes of this * node may reorder or omit any of these properties as they see fit for their use case. + * + * Note: subclasses that use this, please synchronize your createSheet method, so that the + * updates don't come in while you haven't finished creating your sheet. * * @return List of file properties associated with this file node's content. */ From c18fe62f908bbb84544e8228edd37498032b4e29 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Fri, 2 Nov 2018 12:15:11 -0400 Subject: [PATCH 059/129] Removed stack trace on warning log --- Core/src/org/sleuthkit/autopsy/datamodel/FileNodeUtil.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/FileNodeUtil.java b/Core/src/org/sleuthkit/autopsy/datamodel/FileNodeUtil.java index 6eb2bd18c7..64261c56b0 100755 --- a/Core/src/org/sleuthkit/autopsy/datamodel/FileNodeUtil.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/FileNodeUtil.java @@ -187,10 +187,10 @@ class FileNodeUtil { } } catch (NoServiceProviderException noServiceEx) { logger.log(Level.WARNING, "Translate unsuccessful because no TextTranslator " - + "implementation was provided.", noServiceEx); + + "implementation was provided.", noServiceEx.getMessage()); } catch (TranslationException noTranslationEx) { logger.log(Level.WARNING, "Could not successfully translate file name " - + content.getName(), noTranslationEx); + + content.getName(), noTranslationEx.getMessage()); } } From 559a963a0bbbc51aa74984dce249d03ca13facf6 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dgrove" Date: Mon, 5 Nov 2018 02:41:50 -0500 Subject: [PATCH 060/129] Added handling of NSArray data. --- .../autopsy/contentviewers/PListViewer.java | 31 +++++++++++++------ 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/contentviewers/PListViewer.java b/Core/src/org/sleuthkit/autopsy/contentviewers/PListViewer.java index 506084f0c3..dd73b6f6ca 100644 --- a/Core/src/org/sleuthkit/autopsy/contentviewers/PListViewer.java +++ b/Core/src/org/sleuthkit/autopsy/contentviewers/PListViewer.java @@ -74,7 +74,7 @@ class PListViewer extends javax.swing.JPanel implements FileTypeViewer, Explorer private final Outline outline; private ExplorerManager explorerManager; - private NSDictionary rootDict; + private NSObject rootDict; /** * Creates new form PListViewer @@ -415,22 +415,35 @@ class PListViewer extends javax.swing.JPanel implements FileTypeViewer, Explorer } /** - * Parses given binary stream and extracts Plist key/value + * Parses given binary stream and extracts Plist key/value. * - * @param plistbytes + * @param plistbytes The byte array containing the Plist data. * * @return list of PropKeyValue */ private List parsePList(final byte[] plistbytes) throws IOException, PropertyListFormatException, ParseException, ParserConfigurationException, SAXException { final List plist = new ArrayList<>(); - rootDict = (NSDictionary) PropertyListParser.parse(plistbytes); + rootDict = PropertyListParser.parse(plistbytes); - final String[] keys = rootDict.allKeys(); - for (final String key : keys) { - final PropKeyValue pkv = parseProperty(key, rootDict.objectForKey(key)); - if (null != pkv) { - plist.add(pkv); + /* + * Parse the data if the root is an NSArray or NSDictionary. Anything + * else is unexpected and will be ignored. + */ + if (rootDict instanceof NSArray) { + for (int i=0; i < ((NSArray)rootDict).count(); i++) { + final PropKeyValue pkv = parseProperty("", ((NSArray)rootDict).objectAtIndex(i)); + if (null != pkv) { + plist.add(pkv); + } + } + } else if (rootDict instanceof NSDictionary) { + final String[] keys = ((NSDictionary)rootDict).allKeys(); + for (final String key : keys) { + final PropKeyValue pkv = parseProperty(key, ((NSDictionary)rootDict).objectForKey(key)); + if (null != pkv) { + plist.add(pkv); + } } } From 8d38f28d081796d8b3526427ead712e418590dcb Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Mon, 5 Nov 2018 16:18:28 -0500 Subject: [PATCH 061/129] Initial commit of all the work. Need to test it and think about edge cases a bit logner --- .../autopsy/core/AutopsySQLiteException.java | 21 + .../autopsy/core/SQLiteTableReader.java | 540 ++++++++++++++++++ 2 files changed, 561 insertions(+) create mode 100755 Core/src/org/sleuthkit/autopsy/core/AutopsySQLiteException.java create mode 100755 Core/src/org/sleuthkit/autopsy/core/SQLiteTableReader.java diff --git a/Core/src/org/sleuthkit/autopsy/core/AutopsySQLiteException.java b/Core/src/org/sleuthkit/autopsy/core/AutopsySQLiteException.java new file mode 100755 index 0000000000..3e41a23ece --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/core/AutopsySQLiteException.java @@ -0,0 +1,21 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ +package org.sleuthkit.autopsy.core; + +/** + * + * @author dsmyda + */ +public class AutopsySQLiteException extends Exception { + + public AutopsySQLiteException(String msg, Throwable ex) { + super(msg, ex); + } + + public AutopsySQLiteException(Throwable ex) { + super(ex); + } +} diff --git a/Core/src/org/sleuthkit/autopsy/core/SQLiteTableReader.java b/Core/src/org/sleuthkit/autopsy/core/SQLiteTableReader.java new file mode 100755 index 0000000000..b46834715c --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/core/SQLiteTableReader.java @@ -0,0 +1,540 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2018-2018 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.sleuthkit.autopsy.core; + +import java.io.File; +import java.io.IOException; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.function.BooleanSupplier; +import java.util.function.Consumer; +import org.sleuthkit.autopsy.casemodule.Case; +import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; +import org.sleuthkit.autopsy.casemodule.services.FileManager; +import org.sleuthkit.autopsy.casemodule.services.Services; +import org.sleuthkit.autopsy.datamodel.ContentUtils; +import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.datamodel.SleuthkitCase; +import org.sleuthkit.datamodel.TskCoreException; + +/** + * + * @author dsmyda + */ +public class SQLiteTableReader implements AutoCloseable { + + /** + * + */ + public static class Builder { + + private final AbstractFile file; + private Consumer onMetaDataAction; + private Consumer onStringAction; + private Consumer onLongAction; + private Consumer onIntegerAction; + private Consumer onFloatAction; + private Consumer onBlobAction; + private Consumer forAllAction; + + /** + * Creates a SQLiteTableReaderBuilder for this abstract file. + * + * @param file + */ + public Builder(AbstractFile file) { + this.file = file; + } + + /** + * Specify a function to handle MetaData parsing. The MetaData object + * will be parsed before any contents are read from the table. + * + * @param action + * + * @return + */ + public Builder onMetadata(Consumer action) { + this.onMetaDataAction = action; + return this; + } + + /** + * Specify a function to do on receiving a database entry that is type + * String. + * + * @param action + * + * @return + */ + public Builder onString(Consumer action) { + this.onStringAction = action; + return this; + } + + /** + * Specify a function to do on receiving a database entry that is type + * Integer. + * + * @param action + * + * @return + */ + public Builder onInteger(Consumer action) { + this.onIntegerAction = action; + return this; + } + + /** + * Specify a function to do on receiving a database entry that is type + * Real. + * + * @param action + * + * @return + */ + public Builder onFloat(Consumer action) { + this.onFloatAction = action; + return this; + } + + /** + * + * @param action + * @return + */ + public Builder onLong(Consumer action) { + this.onLongAction = action; + return this; + } + + /** + * + * @param action + * @return + */ + public Builder onBlob(Consumer action) { + this.onBlobAction = action; + return this; + } + + /** + * Specify a function to do for any database entry, regardless of type. + * + * @param action + * + * @return + */ + public Builder forAll(Consumer action) { + this.forAllAction = action; + return this; + } + + /** + * Pass all params to the SQLTableStream so that it can iterate through + * the table + * + * @return + */ + public SQLiteTableReader build() { + return new SQLiteTableReader( + file, + onMetaDataAction, + onStringAction, + onIntegerAction, + onLongAction, + onFloatAction, + onBlobAction, + forAllAction + ); + } + } + + private final AbstractFile file; + + private Connection conn; + private PreparedStatement statement; + private ResultSet queryResults; + + private final Consumer onMetaDataAction; + private final Consumer onStringAction; + private final Consumer onIntegerAction; + private final Consumer onLongAction; + private final Consumer onFloatAction; + private final Consumer onBlobAction; + private final Consumer forAllAction; + + //Iteration state variables + private Integer currColumnCount; + private boolean unfinishedRowState = false; + + private boolean isFinished; + private boolean hasOpened; + + private final BooleanSupplier alwaysFalseCondition = () -> {return false;}; + + /** + * Initialize a new table stream given the parameters passed in from the + * StreamBuilder above. + */ + private SQLiteTableReader(AbstractFile file, + Consumer metaDataAction, + Consumer stringAction, + Consumer integerAction, + Consumer longAction, + Consumer floatAction, + Consumer blobAction, + Consumer forAllAction) { + + this.onMetaDataAction = checkNonNull(metaDataAction); + this.onStringAction = checkNonNull(stringAction); + this.onIntegerAction = checkNonNull(integerAction); + this.onLongAction = checkNonNull(longAction); + this.onFloatAction = checkNonNull(floatAction); + this.onBlobAction = checkNonNull(blobAction); + this.forAllAction = checkNonNull(forAllAction); + + this.file = file; + } + + /** + * + * @param + * @param action + * @return + */ + private Consumer checkNonNull(Consumer action) { + if (Objects.nonNull(action)) { + return action; + } + + //No-op lambda, keep from NPE or having to check during iteration + //if action == null. + return (NO_OP) -> {}; + } + + /** + * Get table names from database + * + * @return + * @throws org.sleuthkit.autopsy.core.AutopsySQLiteException + * + */ + public List getTableNames() throws AutopsySQLiteException { + ensureOpen(); + + List tableNames = new ArrayList<>(); + + try (ResultSet tableNameResult = conn.createStatement() + .executeQuery("SELECT name FROM sqlite_master " + + " WHERE type= 'table' ")) { + while (tableNameResult.next()) { + tableNames.add(tableNameResult.getString("name")); //NON-NLS + } + } catch (SQLException ex) { + throw new AutopsySQLiteException(ex); + } + + return tableNames; + } + + /** + * + * @param tableName + * @return + * @throws org.sleuthkit.autopsy.core.AutopsySQLiteException + */ + public int getRowCount(String tableName) throws AutopsySQLiteException { + ensureOpen(); + + try (ResultSet countResult = conn.createStatement() + .executeQuery("SELECT count (*) as count FROM " + + "\"" + tableName + "\"")) { + return countResult.getInt("count"); + } catch (SQLException ex) { + throw new AutopsySQLiteException(ex); + } + } + + /** + * + * @param tableName + * @throws org.sleuthkit.autopsy.core.AutopsySQLiteException + */ + public void read(String tableName) throws AutopsySQLiteException { + readHelper("SELECT * FROM \"" + tableName +"\"", alwaysFalseCondition); + } + + /** + * Read x number of rows (limit), starting from row number y (offset) in + * table z (tableName). + * + * @param tableName + * @param limit + * @param offset + * @throws org.sleuthkit.autopsy.core.AutopsySQLiteException + * + */ + public void read(String tableName, int limit, int offset) throws AutopsySQLiteException { + readHelper("SELECT * FROM \"" + tableName +"\" LIMIT " + limit + + " OFFSET " + offset, alwaysFalseCondition); + } + + /** + * Iterate through the table stopping if we are done, an exception is + * thrown, or the condition is false! + * + * @param tableName + * @param condition + * @throws org.sleuthkit.autopsy.core.AutopsySQLiteException + * + */ + public void read(String tableName, BooleanSupplier condition) throws AutopsySQLiteException { + readHelper("SELECT * FROM \"" + tableName + "\"", condition); + } + + /** + * Iterate through the entire table calling the correct function given the + * datatype. Only stop when there is nothing left to read or a SQLException + * is thrown. + * + * @param tableName + * + * @throws org.sleuthkit.autopsy.core.AutopsySQLiteException + */ + private void readHelper(String query, BooleanSupplier condition) throws AutopsySQLiteException { + try { + if(!hasOpened) { + openResultSet(query); + } + + isFinished = false; + + ResultSetMetaData metaData = queryResults.getMetaData(); + this.onMetaDataAction.accept(metaData); + + int columnCount = metaData.getColumnCount(); + + while (unfinishedRowState || queryResults.next()) { + if (!unfinishedRowState) { + currColumnCount = 1; + } + + for (; currColumnCount <= columnCount; currColumnCount++) { + + if (condition.getAsBoolean()) { + unfinishedRowState = true; + return; + } + + Object item = queryResults.getObject(currColumnCount); + if(item instanceof String) { + this.onStringAction.accept((String) item); + } else if(item instanceof Integer) { + this.onIntegerAction.accept((Integer) item); + } else if(item instanceof Double) { + this.onFloatAction.accept((Double) item); + } else if(item instanceof Long) { + this.onLongAction.accept((Long) item); + } else if(item instanceof byte[]) { + this.onBlobAction.accept((byte[]) item); + } + + this.forAllAction.accept(item); + } + + unfinishedRowState = false; + } + + isFinished = true; + closeResultSet(); + } catch (SQLException ex) { + throw new AutopsySQLiteException(ex); + } + } + + /** + * + * @throws org.sleuthkit.autopsy.core.AutopsySQLiteException + */ + private void ensureOpen() throws AutopsySQLiteException { + if (Objects.isNull(conn)) { + try { + String localDiskPath = writeAbstractFileToLocalDisk(file, file.getId()); + findAndCopySQLiteMetaFile(file); + conn = DriverManager.getConnection("jdbc:sqlite:" + localDiskPath); + } catch (NoCurrentCaseException | TskCoreException | IOException | SQLException ex) { + throw new AutopsySQLiteException(ex); + } + } + } + + /** + * Overloaded implementation of + * {@link #findAndCopySQLiteMetaFile(AbstractFile, String) findAndCopySQLiteMetaFile} + * , automatically tries to copy -wal and -shm files without needing to know + * their existence. + * + * @param sqliteFile file which has -wal and -shm meta files + * + * @throws NoCurrentCaseException Case has been closed. + * @throws TskCoreException fileManager cannot find AbstractFile + * files. + * @throws IOException Issue during writing to file. + */ + private void findAndCopySQLiteMetaFile(AbstractFile sqliteFile) + throws NoCurrentCaseException, TskCoreException, IOException { + + findAndCopySQLiteMetaFile(sqliteFile, sqliteFile.getName() + "-wal"); + findAndCopySQLiteMetaFile(sqliteFile, sqliteFile.getName() + "-shm"); + } + + /** + * Searches for a meta file associated with the give SQLite database. If + * found, it copies this file into the temp directory of the current case. + * + * @param sqliteFile file being processed + * @param metaFileName name of meta file to look for + * + * @throws NoCurrentCaseException Case has been closed. + * @throws TskCoreException fileManager cannot find AbstractFile + * files. + * @throws IOException Issue during writing to file. + */ + private void findAndCopySQLiteMetaFile(AbstractFile sqliteFile, + String metaFileName) throws NoCurrentCaseException, TskCoreException, IOException { + + Case openCase = Case.getCurrentCaseThrows(); + SleuthkitCase sleuthkitCase = openCase.getSleuthkitCase(); + Services services = new Services(sleuthkitCase); + FileManager fileManager = services.getFileManager(); + + List metaFiles = fileManager.findFiles( + sqliteFile.getDataSource(), metaFileName, + sqliteFile.getParent().getName()); + + if (metaFiles != null) { + for (AbstractFile metaFile : metaFiles) { + writeAbstractFileToLocalDisk(metaFile, sqliteFile.getId()); + } + } + } + + /** + * Copies the file contents into a unique path in the current case temp + * directory. + * + * @param file AbstractFile from the data source + * + * @return The path of the file on disk + * + * @throws IOException Exception writing file contents + * @throws NoCurrentCaseException Current case closed during file copying + */ + private String writeAbstractFileToLocalDisk(AbstractFile file, long id) + throws IOException, NoCurrentCaseException { + + String localDiskPath = Case.getCurrentCaseThrows().getTempDirectory() + + File.separator + id + file.getName(); + File localDatabaseFile = new File(localDiskPath); + if (!localDatabaseFile.exists()) { + ContentUtils.writeToFile(file, localDatabaseFile); + } + return localDiskPath; + } + + /** + * + * @param query + * @throws SQLException + */ + private void openResultSet(String query) throws AutopsySQLiteException { + ensureOpen(); + + try { + statement = conn.prepareStatement(query); + // statement.setFetchSize(300); + + queryResults = statement.executeQuery(); + hasOpened = true; + } catch (SQLException ex) { + throw new AutopsySQLiteException(ex); + } + } + + /** + * + */ + private void closeResultSet() { + try { + if(Objects.nonNull(statement)) { + statement.close(); + } + if(Objects.nonNull(queryResults)) { + queryResults.close(); + } + hasOpened = false; + } catch (SQLException ex) { + //Do nothing, can't close.. tried our best. + } + } + + /** + * Closes all connections with the database. + */ + @Override + public void close() { + try { + closeResultSet(); + if(Objects.nonNull(conn)) { + conn.close(); + } + } catch (SQLException ex) { + //Do nothing, can't close.. tried our best. + } + } + + /** + * Checks if there is still work to do on the result set. + * + * @return boolean + */ + public boolean isFinished() { + return isFinished; + } + + /** + * Last ditch effort to close the connections. + * + * @throws Throwable + */ + @Override + public void finalize() throws Throwable { + super.finalize(); + close(); + } +} \ No newline at end of file From b7bb8f19f78a1356bb25f9b655863b2be542bc3b Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Tue, 6 Nov 2018 12:06:45 -0500 Subject: [PATCH 062/129] Renames are not final, added some things that were either necessary or very NTH in order to use this class --- .../AutopsySQLiteException.java | 2 +- .../SQLiteTableStream.java} | 157 ++++++++++++------ 2 files changed, 110 insertions(+), 49 deletions(-) rename Core/src/org/sleuthkit/autopsy/{core => coreutils}/AutopsySQLiteException.java (91%) rename Core/src/org/sleuthkit/autopsy/{core/SQLiteTableReader.java => coreutils/SQLiteTableStream.java} (74%) diff --git a/Core/src/org/sleuthkit/autopsy/core/AutopsySQLiteException.java b/Core/src/org/sleuthkit/autopsy/coreutils/AutopsySQLiteException.java similarity index 91% rename from Core/src/org/sleuthkit/autopsy/core/AutopsySQLiteException.java rename to Core/src/org/sleuthkit/autopsy/coreutils/AutopsySQLiteException.java index 3e41a23ece..7bc9adb727 100755 --- a/Core/src/org/sleuthkit/autopsy/core/AutopsySQLiteException.java +++ b/Core/src/org/sleuthkit/autopsy/coreutils/AutopsySQLiteException.java @@ -3,7 +3,7 @@ * To change this template file, choose Tools | Templates * and open the template in the editor. */ -package org.sleuthkit.autopsy.core; +package org.sleuthkit.autopsy.coreutils; /** * diff --git a/Core/src/org/sleuthkit/autopsy/core/SQLiteTableReader.java b/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableStream.java similarity index 74% rename from Core/src/org/sleuthkit/autopsy/core/SQLiteTableReader.java rename to Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableStream.java index b46834715c..2095ac6f77 100755 --- a/Core/src/org/sleuthkit/autopsy/core/SQLiteTableReader.java +++ b/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableStream.java @@ -17,7 +17,7 @@ * limitations under the License. */ -package org.sleuthkit.autopsy.core; +package org.sleuthkit.autopsy.coreutils; import java.io.File; import java.io.IOException; @@ -45,7 +45,7 @@ import org.sleuthkit.datamodel.TskCoreException; * * @author dsmyda */ -public class SQLiteTableReader implements AutoCloseable { +public class SQLiteTableStream implements AutoCloseable { /** * @@ -54,12 +54,12 @@ public class SQLiteTableReader implements AutoCloseable { private final AbstractFile file; private Consumer onMetaDataAction; - private Consumer onStringAction; - private Consumer onLongAction; - private Consumer onIntegerAction; - private Consumer onFloatAction; - private Consumer onBlobAction; - private Consumer forAllAction; + private Consumer> onStringAction; + private Consumer> onLongAction; + private Consumer> onIntegerAction; + private Consumer> onFloatAction; + private Consumer> onBlobAction; + private Consumer> forAllAction; /** * Creates a SQLiteTableReaderBuilder for this abstract file. @@ -91,7 +91,7 @@ public class SQLiteTableReader implements AutoCloseable { * * @return */ - public Builder onString(Consumer action) { + public Builder onString(Consumer> action) { this.onStringAction = action; return this; } @@ -104,7 +104,7 @@ public class SQLiteTableReader implements AutoCloseable { * * @return */ - public Builder onInteger(Consumer action) { + public Builder onInteger(Consumer> action) { this.onIntegerAction = action; return this; } @@ -117,7 +117,7 @@ public class SQLiteTableReader implements AutoCloseable { * * @return */ - public Builder onFloat(Consumer action) { + public Builder onFloat(Consumer> action) { this.onFloatAction = action; return this; } @@ -127,7 +127,7 @@ public class SQLiteTableReader implements AutoCloseable { * @param action * @return */ - public Builder onLong(Consumer action) { + public Builder onLong(Consumer> action) { this.onLongAction = action; return this; } @@ -137,7 +137,7 @@ public class SQLiteTableReader implements AutoCloseable { * @param action * @return */ - public Builder onBlob(Consumer action) { + public Builder onBlob(Consumer> action) { this.onBlobAction = action; return this; } @@ -149,7 +149,7 @@ public class SQLiteTableReader implements AutoCloseable { * * @return */ - public Builder forAll(Consumer action) { + public Builder forAll(Consumer> action) { this.forAllAction = action; return this; } @@ -160,8 +160,8 @@ public class SQLiteTableReader implements AutoCloseable { * * @return */ - public SQLiteTableReader build() { - return new SQLiteTableReader( + public SQLiteTableStream build() { + return new SQLiteTableStream( file, onMetaDataAction, onStringAction, @@ -181,12 +181,12 @@ public class SQLiteTableReader implements AutoCloseable { private ResultSet queryResults; private final Consumer onMetaDataAction; - private final Consumer onStringAction; - private final Consumer onIntegerAction; - private final Consumer onLongAction; - private final Consumer onFloatAction; - private final Consumer onBlobAction; - private final Consumer forAllAction; + private final Consumer> onStringAction; + private final Consumer> onLongAction; + private final Consumer> onIntegerAction; + private final Consumer> onFloatAction; + private final Consumer> onBlobAction; + private final Consumer> forAllAction; //Iteration state variables private Integer currColumnCount; @@ -201,14 +201,14 @@ public class SQLiteTableReader implements AutoCloseable { * Initialize a new table stream given the parameters passed in from the * StreamBuilder above. */ - private SQLiteTableReader(AbstractFile file, + private SQLiteTableStream(AbstractFile file, Consumer metaDataAction, - Consumer stringAction, - Consumer integerAction, - Consumer longAction, - Consumer floatAction, - Consumer blobAction, - Consumer forAllAction) { + Consumer> stringAction, + Consumer> integerAction, + Consumer> longAction, + Consumer> floatAction, + Consumer> blobAction, + Consumer> forAllAction) { this.onMetaDataAction = checkNonNull(metaDataAction); this.onStringAction = checkNonNull(stringAction); @@ -220,13 +220,7 @@ public class SQLiteTableReader implements AutoCloseable { this.file = file; } - - /** - * - * @param - * @param action - * @return - */ + private Consumer checkNonNull(Consumer action) { if (Objects.nonNull(action)) { return action; @@ -241,7 +235,7 @@ public class SQLiteTableReader implements AutoCloseable { * Get table names from database * * @return - * @throws org.sleuthkit.autopsy.core.AutopsySQLiteException + * @throws org.sleuthkit.autopsy.coreutils.AutopsySQLiteException * */ public List getTableNames() throws AutopsySQLiteException { @@ -266,7 +260,7 @@ public class SQLiteTableReader implements AutoCloseable { * * @param tableName * @return - * @throws org.sleuthkit.autopsy.core.AutopsySQLiteException + * @throws org.sleuthkit.autopsy.coreutils.AutopsySQLiteException */ public int getRowCount(String tableName) throws AutopsySQLiteException { ensureOpen(); @@ -279,11 +273,23 @@ public class SQLiteTableReader implements AutoCloseable { throw new AutopsySQLiteException(ex); } } + + public int getColumnCount(String tableName) throws AutopsySQLiteException { + ensureOpen(); + + try (ResultSet columnCount = conn.createStatement() + .executeQuery("SELECT * FROM " + + "\"" + tableName + "\"")) { + return columnCount.getMetaData().getColumnCount(); + } catch (SQLException ex) { + throw new AutopsySQLiteException(ex); + } + } /** * * @param tableName - * @throws org.sleuthkit.autopsy.core.AutopsySQLiteException + * @throws org.sleuthkit.autopsy.coreutils.AutopsySQLiteException */ public void read(String tableName) throws AutopsySQLiteException { readHelper("SELECT * FROM \"" + tableName +"\"", alwaysFalseCondition); @@ -296,7 +302,7 @@ public class SQLiteTableReader implements AutoCloseable { * @param tableName * @param limit * @param offset - * @throws org.sleuthkit.autopsy.core.AutopsySQLiteException + * @throws org.sleuthkit.autopsy.coreutils.AutopsySQLiteException * */ public void read(String tableName, int limit, int offset) throws AutopsySQLiteException { @@ -310,7 +316,7 @@ public class SQLiteTableReader implements AutoCloseable { * * @param tableName * @param condition - * @throws org.sleuthkit.autopsy.core.AutopsySQLiteException + * @throws org.sleuthkit.autopsy.coreutils.AutopsySQLiteException * */ public void read(String tableName, BooleanSupplier condition) throws AutopsySQLiteException { @@ -351,20 +357,45 @@ public class SQLiteTableReader implements AutoCloseable { return; } + //getObject automatically instiantiates the correct java data type Object item = queryResults.getObject(currColumnCount); if(item instanceof String) { - this.onStringAction.accept((String) item); + this.onStringAction.accept(new ResultState<>( + (String) item, + metaData.getColumnName(currColumnCount), + currColumnCount) + ); } else if(item instanceof Integer) { - this.onIntegerAction.accept((Integer) item); + this.onIntegerAction.accept(new ResultState<>( + (Integer) item, + metaData.getColumnName(currColumnCount), + currColumnCount) + ); } else if(item instanceof Double) { - this.onFloatAction.accept((Double) item); + this.onFloatAction.accept(new ResultState<>( + (Double) item, + metaData.getColumnName(currColumnCount), + currColumnCount) + ); } else if(item instanceof Long) { - this.onLongAction.accept((Long) item); + this.onLongAction.accept(new ResultState<>( + (Long) item, + metaData.getColumnName(currColumnCount), + currColumnCount) + ); } else if(item instanceof byte[]) { - this.onBlobAction.accept((byte[]) item); + this.onBlobAction.accept(new ResultState<>( + (byte[]) item, + metaData.getColumnName(currColumnCount), + currColumnCount) + ); } - this.forAllAction.accept(item); + this.forAllAction.accept(new ResultState<>( + (Object) item, + metaData.getColumnName(currColumnCount), + currColumnCount) + ); } unfinishedRowState = false; @@ -384,10 +415,12 @@ public class SQLiteTableReader implements AutoCloseable { private void ensureOpen() throws AutopsySQLiteException { if (Objects.isNull(conn)) { try { + Class.forName("org.sqlite.JDBC"); //NON-NLS String localDiskPath = writeAbstractFileToLocalDisk(file, file.getId()); findAndCopySQLiteMetaFile(file); conn = DriverManager.getConnection("jdbc:sqlite:" + localDiskPath); - } catch (NoCurrentCaseException | TskCoreException | IOException | SQLException ex) { + } catch (NoCurrentCaseException | TskCoreException | IOException | + ClassNotFoundException | SQLException ex) { throw new AutopsySQLiteException(ex); } } @@ -537,4 +570,32 @@ public class SQLiteTableReader implements AutoCloseable { super.finalize(); close(); } + + /** + * + * @param + */ + public class ResultState { + private final T value; + private final String columnName; + private final int columnIndex; + + private ResultState(T value, String columnName, int columnIndex) { + this.value = value; + this.columnName = columnName; + this.columnIndex = columnIndex; + } + + public T getValue() { + return value; + } + + public String getColumnName() { + return columnName; + } + + public Integer getIndex() { + return columnIndex; + } + } } \ No newline at end of file From 86ebf78dfbacd1c450ef4b8bdb30691ab9650ca2 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Tue, 6 Nov 2018 15:40:27 -0500 Subject: [PATCH 063/129] Fixed stream for read with condition, will be coming back around to refactor --- .../autopsy/coreutils/SQLiteTableStream.java | 86 +++++++++++-------- 1 file changed, 50 insertions(+), 36 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableStream.java b/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableStream.java index 2095ac6f77..a455e7bfeb 100755 --- a/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableStream.java +++ b/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableStream.java @@ -54,12 +54,12 @@ public class SQLiteTableStream implements AutoCloseable { private final AbstractFile file; private Consumer onMetaDataAction; - private Consumer> onStringAction; - private Consumer> onLongAction; - private Consumer> onIntegerAction; - private Consumer> onFloatAction; - private Consumer> onBlobAction; - private Consumer> forAllAction; + private Consumer> onStringAction; + private Consumer> onLongAction; + private Consumer> onIntegerAction; + private Consumer> onFloatAction; + private Consumer> onBlobAction; + private Consumer> forAllAction; /** * Creates a SQLiteTableReaderBuilder for this abstract file. @@ -91,7 +91,7 @@ public class SQLiteTableStream implements AutoCloseable { * * @return */ - public Builder onString(Consumer> action) { + public Builder onString(Consumer> action) { this.onStringAction = action; return this; } @@ -104,7 +104,7 @@ public class SQLiteTableStream implements AutoCloseable { * * @return */ - public Builder onInteger(Consumer> action) { + public Builder onInteger(Consumer> action) { this.onIntegerAction = action; return this; } @@ -117,7 +117,7 @@ public class SQLiteTableStream implements AutoCloseable { * * @return */ - public Builder onFloat(Consumer> action) { + public Builder onFloat(Consumer> action) { this.onFloatAction = action; return this; } @@ -127,7 +127,7 @@ public class SQLiteTableStream implements AutoCloseable { * @param action * @return */ - public Builder onLong(Consumer> action) { + public Builder onLong(Consumer> action) { this.onLongAction = action; return this; } @@ -137,7 +137,7 @@ public class SQLiteTableStream implements AutoCloseable { * @param action * @return */ - public Builder onBlob(Consumer> action) { + public Builder onBlob(Consumer> action) { this.onBlobAction = action; return this; } @@ -149,7 +149,7 @@ public class SQLiteTableStream implements AutoCloseable { * * @return */ - public Builder forAll(Consumer> action) { + public Builder forAll(Consumer> action) { this.forAllAction = action; return this; } @@ -181,12 +181,12 @@ public class SQLiteTableStream implements AutoCloseable { private ResultSet queryResults; private final Consumer onMetaDataAction; - private final Consumer> onStringAction; - private final Consumer> onLongAction; - private final Consumer> onIntegerAction; - private final Consumer> onFloatAction; - private final Consumer> onBlobAction; - private final Consumer> forAllAction; + private final Consumer> onStringAction; + private final Consumer> onLongAction; + private final Consumer> onIntegerAction; + private final Consumer> onFloatAction; + private final Consumer> onBlobAction; + private final Consumer> forAllAction; //Iteration state variables private Integer currColumnCount; @@ -194,6 +194,7 @@ public class SQLiteTableStream implements AutoCloseable { private boolean isFinished; private boolean hasOpened; + private String prevTableName; private final BooleanSupplier alwaysFalseCondition = () -> {return false;}; @@ -203,12 +204,12 @@ public class SQLiteTableStream implements AutoCloseable { */ private SQLiteTableStream(AbstractFile file, Consumer metaDataAction, - Consumer> stringAction, - Consumer> integerAction, - Consumer> longAction, - Consumer> floatAction, - Consumer> blobAction, - Consumer> forAllAction) { + Consumer> stringAction, + Consumer> integerAction, + Consumer> longAction, + Consumer> floatAction, + Consumer> blobAction, + Consumer> forAllAction) { this.onMetaDataAction = checkNonNull(metaDataAction); this.onStringAction = checkNonNull(stringAction); @@ -320,7 +321,19 @@ public class SQLiteTableStream implements AutoCloseable { * */ public void read(String tableName, BooleanSupplier condition) throws AutopsySQLiteException { - readHelper("SELECT * FROM \"" + tableName + "\"", condition); + if(Objects.nonNull(prevTableName)) { + if(prevTableName.equals(tableName)) { + readHelper("SELECT * FROM \"" + tableName + "\"", condition); + } else { + prevTableName = tableName; + closeResultSet(); + readHelper("SELECT * FROM \"" + tableName + "\"", condition); + } + } else { + prevTableName = tableName; + closeResultSet(); + readHelper("SELECT * FROM \"" + tableName + "\"", condition); + } } /** @@ -336,13 +349,13 @@ public class SQLiteTableStream implements AutoCloseable { try { if(!hasOpened) { openResultSet(query); + ResultSetMetaData metaData = queryResults.getMetaData(); + this.onMetaDataAction.accept(metaData); } isFinished = false; ResultSetMetaData metaData = queryResults.getMetaData(); - this.onMetaDataAction.accept(metaData); - int columnCount = metaData.getColumnCount(); while (unfinishedRowState || queryResults.next()) { @@ -360,38 +373,38 @@ public class SQLiteTableStream implements AutoCloseable { //getObject automatically instiantiates the correct java data type Object item = queryResults.getObject(currColumnCount); if(item instanceof String) { - this.onStringAction.accept(new ResultState<>( + this.onStringAction.accept(new State<>( (String) item, metaData.getColumnName(currColumnCount), currColumnCount) ); } else if(item instanceof Integer) { - this.onIntegerAction.accept(new ResultState<>( + this.onIntegerAction.accept(new State<>( (Integer) item, metaData.getColumnName(currColumnCount), currColumnCount) ); } else if(item instanceof Double) { - this.onFloatAction.accept(new ResultState<>( + this.onFloatAction.accept(new State<>( (Double) item, metaData.getColumnName(currColumnCount), currColumnCount) ); } else if(item instanceof Long) { - this.onLongAction.accept(new ResultState<>( + this.onLongAction.accept(new State<>( (Long) item, metaData.getColumnName(currColumnCount), currColumnCount) ); } else if(item instanceof byte[]) { - this.onBlobAction.accept(new ResultState<>( + this.onBlobAction.accept(new State<>( (byte[]) item, metaData.getColumnName(currColumnCount), currColumnCount) ); } - this.forAllAction.accept(new ResultState<>( + this.forAllAction.accept(new State<>( (Object) item, metaData.getColumnName(currColumnCount), currColumnCount) @@ -404,6 +417,8 @@ public class SQLiteTableStream implements AutoCloseable { isFinished = true; closeResultSet(); } catch (SQLException ex) { + closeResultSet(); + isFinished = true; throw new AutopsySQLiteException(ex); } } @@ -510,7 +525,6 @@ public class SQLiteTableStream implements AutoCloseable { try { statement = conn.prepareStatement(query); - // statement.setFetchSize(300); queryResults = statement.executeQuery(); hasOpened = true; @@ -575,12 +589,12 @@ public class SQLiteTableStream implements AutoCloseable { * * @param */ - public class ResultState { + public class State { private final T value; private final String columnName; private final int columnIndex; - private ResultState(T value, String columnName, int columnIndex) { + private State(T value, String columnName, int columnIndex) { this.value = value; this.columnName = columnName; this.columnIndex = columnIndex; From 0a8d6d80c4c56b2e623166071e4d48dde749c2de Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Tue, 6 Nov 2018 16:10:31 -0500 Subject: [PATCH 064/129] 4354 initial changes to remove unnecessary constructors from CorrelationAttributeInstance --- .../DataContentViewerOtherCases.java | 7 +++--- .../CorrelationAttributeInstance.java | 23 ++----------------- .../ingestmodule/IngestModule.java | 3 +-- .../AnnotationsContentViewer.java | 3 +-- 4 files changed, 7 insertions(+), 29 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/contentviewer/DataContentViewerOtherCases.java b/Core/src/org/sleuthkit/autopsy/centralrepository/contentviewer/DataContentViewerOtherCases.java index 0080ade4fb..076e52bb93 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/contentviewer/DataContentViewerOtherCases.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/contentviewer/DataContentViewerOtherCases.java @@ -430,8 +430,7 @@ public class DataContentViewerOtherCases extends JPanel implements DataContentVi CorrelationCase corCase = EamDb.getInstance().getCase(Case.getCurrentCase()); try { ret.add(new CorrelationAttributeInstance( - md5, - aType, + aType, md5, corCase, CorrelationDataSource.fromTSKDataSource(corCase, file.getDataSource()), file.getParentPath() + file.getName(), @@ -458,8 +457,8 @@ public class DataContentViewerOtherCases extends JPanel implements DataContentVi .filter(attrType -> attrType.getId() == CorrelationAttributeInstance.FILES_TYPE_ID) .findAny() .get(); - - ret.add(new CorrelationAttributeInstance(fileAttributeType, md5)); + //The Central Repository is not enabled + ret.add(new CorrelationAttributeInstance(fileAttributeType, md5, null, null, "", "", TskData.FileKnown.UNKNOWN)); } catch (EamDbException ex) { LOGGER.log(Level.SEVERE, "Error connecting to DB", ex); // NON-NLS } catch (CorrelationAttributeNormalizationException ex) { diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeInstance.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeInstance.java index 20845ff447..263f6051aa 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeInstance.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeInstance.java @@ -48,38 +48,19 @@ public class CorrelationAttributeInstance implements Serializable { private String filePath; private String comment; private TskData.FileKnown knownStatus; + private long objectId; public CorrelationAttributeInstance( - String correlationValue, CorrelationAttributeInstance.Type correlationType, - CorrelationCase eamCase, - CorrelationDataSource eamDataSource, - String filePath - ) throws EamDbException, CorrelationAttributeNormalizationException { - this(correlationType, correlationValue, -1, eamCase, eamDataSource, filePath, null, TskData.FileKnown.UNKNOWN); - } - - public CorrelationAttributeInstance( String correlationValue, - CorrelationAttributeInstance.Type correlationType, CorrelationCase eamCase, CorrelationDataSource eamDataSource, String filePath, String comment, - TskData.FileKnown knownStatus - ) throws EamDbException, CorrelationAttributeNormalizationException { + TskData.FileKnown knownStatus) throws EamDbException, CorrelationAttributeNormalizationException { this(correlationType, correlationValue, -1, eamCase, eamDataSource, filePath, comment, knownStatus); } - public CorrelationAttributeInstance( - Type correlationType, - String correlationValue, - CorrelationCase correlationCase, - CorrelationDataSource fromTSKDataSource, - String string) throws EamDbException, CorrelationAttributeNormalizationException { - this(correlationType, correlationValue, -1, correlationCase, fromTSKDataSource, string, "", TskData.FileKnown.UNKNOWN); - } - /** * NOTE: Only used for when EamDB is NOT enabled. * diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java index 7469e38bd0..8ac7acd2cd 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java @@ -152,8 +152,7 @@ final class IngestModule implements FileIngestModule { // insert this file into the central repository try { CorrelationAttributeInstance cefi = new CorrelationAttributeInstance( - md5, - filesType, + filesType, md5, eamCase, eamDataSource, abstractFile.getParentPath() + abstractFile.getName(), diff --git a/Core/src/org/sleuthkit/autopsy/contentviewers/AnnotationsContentViewer.java b/Core/src/org/sleuthkit/autopsy/contentviewers/AnnotationsContentViewer.java index d88b5ac2fb..583dfca703 100755 --- a/Core/src/org/sleuthkit/autopsy/contentviewers/AnnotationsContentViewer.java +++ b/Core/src/org/sleuthkit/autopsy/contentviewers/AnnotationsContentViewer.java @@ -208,8 +208,7 @@ public class AnnotationsContentViewer extends javax.swing.JPanel implements Data if (attributeType.getId() == CorrelationAttributeInstance.FILES_TYPE_ID) { CorrelationCase correlationCase = EamDb.getInstance().getCase(Case.getCurrentCase()); instancesList.add(new CorrelationAttributeInstance( - md5, - attributeType, + attributeType, md5, correlationCase, CorrelationDataSource.fromTSKDataSource(correlationCase, sourceFile.getDataSource()), sourceFile.getParentPath() + sourceFile.getName(), From be7e9675a57b77039c5f52e9e3c6251bcb1dd701 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Tue, 6 Nov 2018 16:16:17 -0500 Subject: [PATCH 065/129] 4354 additional changes to remove unnecessary constructors from CorrelationAttributeInstance --- .../datamodel/CorrelationAttributeInstance.java | 11 ----------- .../centralrepository/datamodel/EamArtifactUtil.java | 6 ++++-- 2 files changed, 4 insertions(+), 13 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeInstance.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeInstance.java index 263f6051aa..d7b2e3cf6b 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeInstance.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeInstance.java @@ -48,7 +48,6 @@ public class CorrelationAttributeInstance implements Serializable { private String filePath; private String comment; private TskData.FileKnown knownStatus; - private long objectId; public CorrelationAttributeInstance( CorrelationAttributeInstance.Type correlationType, @@ -61,16 +60,6 @@ public class CorrelationAttributeInstance implements Serializable { this(correlationType, correlationValue, -1, eamCase, eamDataSource, filePath, comment, knownStatus); } - /** - * NOTE: Only used for when EamDB is NOT enabled. - * - * @param aType CorrelationAttributeInstance.Type - * @param value correlation value - */ - public CorrelationAttributeInstance(Type aType, String value) throws EamDbException, CorrelationAttributeNormalizationException { - this(aType, value, -1, null, null, "", "", TskData.FileKnown.UNKNOWN); - } - CorrelationAttributeInstance( Type type, String value, diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactUtil.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactUtil.java index aca0471345..5785355cd9 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactUtil.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactUtil.java @@ -207,8 +207,8 @@ public class EamArtifactUtil { correlationCase = EamDb.getInstance().newCase(Case.getCurrentCaseThrows()); } return new CorrelationAttributeInstance( - value, correlationType, + value, correlationCase, CorrelationDataSource.fromTSKDataSource(correlationCase, bbSourceFile.getDataSource()), bbSourceFile.getParentPath() + bbSourceFile.getName(), @@ -324,7 +324,9 @@ public class EamArtifactUtil { af.getMd5Hash(), correlationCase, CorrelationDataSource.fromTSKDataSource(correlationCase, af.getDataSource()), - af.getParentPath() + af.getName()); + af.getParentPath() + af.getName(), + "", + TskData.FileKnown.UNKNOWN); } catch (TskCoreException | EamDbException | CorrelationAttributeNormalizationException ex) { logger.log(Level.SEVERE, "Error making correlation attribute.", ex); From 1e7d9616b567032d39534d9dca4be2d5a1625467 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Tue, 6 Nov 2018 17:23:05 -0500 Subject: [PATCH 066/129] 4354 changes to add the files object id to correlationAttributeInstance --- .../DataContentViewerOtherCases.java | 8 +- .../datamodel/AbstractSqlEamDb.java | 100 ++++++++++++++++-- .../CorrelationAttributeInstance.java | 10 +- .../datamodel/EamArtifactUtil.java | 74 ++++++++++++- .../centralrepository/datamodel/EamDb.java | 5 + .../datamodel/PostgresEamDbSettings.java | 33 ++++-- .../datamodel/SqliteEamDbSettings.java | 37 +++++-- .../ingestmodule/IngestModule.java | 5 +- .../AnnotationsContentViewer.java | 85 ++++++++------- 9 files changed, 274 insertions(+), 83 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/contentviewer/DataContentViewerOtherCases.java b/Core/src/org/sleuthkit/autopsy/centralrepository/contentviewer/DataContentViewerOtherCases.java index 076e52bb93..191d1c3bc6 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/contentviewer/DataContentViewerOtherCases.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/contentviewer/DataContentViewerOtherCases.java @@ -430,12 +430,14 @@ public class DataContentViewerOtherCases extends JPanel implements DataContentVi CorrelationCase corCase = EamDb.getInstance().getCase(Case.getCurrentCase()); try { ret.add(new CorrelationAttributeInstance( - aType, md5, + aType, + md5, corCase, CorrelationDataSource.fromTSKDataSource(corCase, file.getDataSource()), file.getParentPath() + file.getName(), "", - file.getKnown())); + file.getKnown(), + file.getId())); } catch (CorrelationAttributeNormalizationException ex) { LOGGER.log(Level.INFO, String.format("Unable to check create CorrelationAttribtueInstance for value %s and type %s.", md5, aType.toString()), ex); } @@ -458,7 +460,7 @@ public class DataContentViewerOtherCases extends JPanel implements DataContentVi .findAny() .get(); //The Central Repository is not enabled - ret.add(new CorrelationAttributeInstance(fileAttributeType, md5, null, null, "", "", TskData.FileKnown.UNKNOWN)); + ret.add(new CorrelationAttributeInstance(fileAttributeType, md5, null, null, "", "", TskData.FileKnown.UNKNOWN, this.file.getId())); } catch (EamDbException ex) { LOGGER.log(Level.SEVERE, "Error connecting to DB", ex); // NON-NLS } catch (CorrelationAttributeNormalizationException ex) { diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java index e239f0b987..f57c27444f 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java @@ -900,6 +900,8 @@ abstract class AbstractSqlEamDb implements EamDb { + ".id," + tableName + ".value," + + tableName + + ".object_id," + " cases.case_name, cases.case_uid, data_sources.id AS data_source_id, data_sources.name, device_id, file_path, known_status, comment, data_sources.case_id FROM " + tableName + " LEFT JOIN cases ON " @@ -963,6 +965,8 @@ abstract class AbstractSqlEamDb implements EamDb { + ".id, " + tableName + ".value," + + tableName + + ".object_id," + " cases.case_name, cases.case_uid, data_sources.id AS data_source_id, data_sources.name, device_id, file_path, known_status, comment, data_sources.case_id FROM " + tableName + " LEFT JOIN cases ON " @@ -1439,6 +1443,68 @@ abstract class AbstractSqlEamDb implements EamDb { } } + /** + * Find a correlation attribute in the Central Repository database given the + * instance type, case, data source, value, and file path. + * + * @param type The type of instance. + * @param correlationCase The case tied to the instance. + * @param correlationDataSource The data source tied to the instance. + * @param value The value tied to the instance. + * @param filePath The file path tied to the instance. + * + * @return The correlation attribute if it exists; otherwise null. + * + * @throws EamDbException + */ + @Override + public CorrelationAttributeInstance getCorrelationAttributeInstance(CorrelationAttributeInstance.Type type, CorrelationCase correlationCase, + CorrelationDataSource correlationDataSource, long objectID) throws EamDbException, CorrelationAttributeNormalizationException { + + if (correlationCase == null) { + throw new EamDbException("Correlation case is null"); + } + + Connection conn = connect(); + + PreparedStatement preparedStatement = null; + ResultSet resultSet = null; + CorrelationAttributeInstance correlationAttributeInstance = null; + + try { + + String tableName = EamDbUtil.correlationTypeToInstanceTableName(type); + String sql + = "SELECT id, value, file_path, known_status, comment FROM " + + tableName + + " WHERE case_id=?" + + " AND object_id=?"; + + preparedStatement = conn.prepareStatement(sql); + preparedStatement.setInt(1, correlationCase.getID()); + preparedStatement.setInt(2, (int) objectID); + resultSet = preparedStatement.executeQuery(); + if (resultSet.next()) { + int instanceId = resultSet.getInt(1); + String value = resultSet.getString(2); + String filePath = resultSet.getString(3); + int knownStatus = resultSet.getInt(4); + String comment = resultSet.getString(5); + + correlationAttributeInstance = new CorrelationAttributeInstance(type, value, + instanceId, correlationCase, correlationDataSource, filePath, comment, TskData.FileKnown.valueOf((byte) knownStatus), objectID); + } + } catch (SQLException ex) { + throw new EamDbException("Error getting notable artifact instances.", ex); // NON-NLS + } finally { + EamDbUtil.closeStatement(preparedStatement); + EamDbUtil.closeResultSet(resultSet); + EamDbUtil.closeConnection(conn); + } + + return correlationAttributeInstance; + } + /** * Find a correlation attribute in the Central Repository database given the * instance type, case, data source, value, and file path. @@ -1495,9 +1561,9 @@ abstract class AbstractSqlEamDb implements EamDb { int instanceId = resultSet.getInt(1); int knownStatus = resultSet.getInt(2); String comment = resultSet.getString(3); - + //null objectId used because we only fall back to using this method when objec correlationAttributeInstance = new CorrelationAttributeInstance(type, value, - instanceId, correlationCase, correlationDataSource, filePath, comment, TskData.FileKnown.valueOf((byte) knownStatus)); + instanceId, correlationCase, correlationDataSource, filePath, comment, TskData.FileKnown.valueOf((byte) knownStatus), null); } } catch (SQLException ex) { throw new EamDbException("Error getting notable artifact instances.", ex); // NON-NLS @@ -1637,6 +1703,8 @@ abstract class AbstractSqlEamDb implements EamDb { + ".id, " + tableName + ".value, " + + tableName + + ".object_id," + "cases.case_name, cases.case_uid, data_sources.id AS data_source_id, data_sources.name, device_id, file_path, known_status, comment, data_sources.case_id FROM " + tableName + " LEFT JOIN cases ON " @@ -1694,7 +1762,7 @@ abstract class AbstractSqlEamDb implements EamDb { String tableName = EamDbUtil.correlationTypeToInstanceTableName(aType); String sql - = "SELECT cases.case_name, cases.case_uid, data_sources.name, device_id, file_path, known_status, comment, data_sources.case_id, id, value FROM " + = "SELECT cases.case_name, cases.case_uid, data_sources.name, device_id, file_path, known_status, comment, data_sources.case_id, id, value, object_id FROM " + tableName + " LEFT JOIN cases ON " + tableName @@ -2960,11 +3028,10 @@ abstract class AbstractSqlEamDb implements EamDb { resultSet.getString("poc_phone")); } - CorrelationCase eamCase = new CorrelationCase(resultSet.getInt("case_id"), resultSet.getString("case_uid"), eamOrg, resultSet.getString("case_name"), - resultSet.getString("creation_date"), resultSet.getString("case_number"), resultSet.getString("examiner_name"), + CorrelationCase eamCase = new CorrelationCase(resultSet.getInt("case_id"), resultSet.getString("case_uid"), eamOrg, resultSet.getString("case_name"), + resultSet.getString("creation_date"), resultSet.getString("case_number"), resultSet.getString("examiner_name"), resultSet.getString("examiner_email"), resultSet.getString("examiner_phone"), resultSet.getString("notes")); - return eamCase; } @@ -3021,8 +3088,8 @@ abstract class AbstractSqlEamDb implements EamDb { new CorrelationDataSource(resultSet.getInt("case_id"), resultSet.getInt("data_source_id"), resultSet.getString("device_id"), resultSet.getString("name")), resultSet.getString("file_path"), resultSet.getString("comment"), - TskData.FileKnown.valueOf(resultSet.getByte("known_status")) - ); + TskData.FileKnown.valueOf(resultSet.getByte("known_status")), + resultSet.getLong("object_id")); } private EamOrganization getEamOrganizationFromResultSet(ResultSet resultSet) throws SQLException { @@ -3119,7 +3186,7 @@ abstract class AbstractSqlEamDb implements EamDb { logger.log(Level.INFO, "Central Repository is of newer version than software creates"); return; } - + // Update from 1.0 to 1.1 if (dbSchemaVersion.compareTo(new CaseDbSchemaVersionNumber(1, 1)) < 0) { statement.execute("ALTER TABLE reference_sets ADD COLUMN known_status INTEGER;"); //NON-NLS @@ -3133,13 +3200,15 @@ abstract class AbstractSqlEamDb implements EamDb { } //Update to 1.2 if (dbSchemaVersion.compareTo(new CaseDbSchemaVersionNumber(1, 2)) < 0) { - //update central repository to be able to store new correlation attributes + EamDbPlatformEnum selectedPlatform = EamDbPlatformEnum.getSelectedPlatform(); + final String addObjectIdColumnTemplate = "ALTER TABLE %s ADD COLUMN object_id INTEGER;"; final String addSsidTableTemplate; final String addCaseIdIndexTemplate; final String addDataSourceIdIndexTemplate; final String addValueIndexTemplate; final String addKnownStatusIndexTemplate; + final String addObjectIdIndexTemplate; //get the data base specific code for creating a new _instance table switch (selectedPlatform) { case POSTGRESQL: @@ -3148,6 +3217,7 @@ abstract class AbstractSqlEamDb implements EamDb { addDataSourceIdIndexTemplate = PostgresEamDbSettings.getAddDataSourceIdIndexTemplate(); addValueIndexTemplate = PostgresEamDbSettings.getAddValueIndexTemplate(); addKnownStatusIndexTemplate = PostgresEamDbSettings.getAddKnownStatusIndexTemplate(); + addObjectIdIndexTemplate = PostgresEamDbSettings.getAddObjectIdIndexTemplate(); break; case SQLITE: addSsidTableTemplate = SqliteEamDbSettings.getCreateArtifactInstancesTableTemplate(); @@ -3155,10 +3225,19 @@ abstract class AbstractSqlEamDb implements EamDb { addDataSourceIdIndexTemplate = SqliteEamDbSettings.getAddDataSourceIdIndexTemplate(); addValueIndexTemplate = SqliteEamDbSettings.getAddValueIndexTemplate(); addKnownStatusIndexTemplate = SqliteEamDbSettings.getAddKnownStatusIndexTemplate(); + addObjectIdIndexTemplate = SqliteEamDbSettings.getAddObjectIdIndexTemplate(); break; default: throw new EamDbException("Currently selected database platform \"" + selectedPlatform.name() + "\" can not be upgraded."); } + String instance_type_dbname; + //add object_id column to existing _instances tables + for (CorrelationAttributeInstance.Type type : CorrelationAttributeInstance.getDefaultCorrelationTypes()) { + instance_type_dbname = EamDbUtil.correlationTypeToInstanceTableName(type); + statement.execute(String.format(addObjectIdColumnTemplate, instance_type_dbname)); //NON-NLS + statement.execute(String.format(addObjectIdIndexTemplate, instance_type_dbname, instance_type_dbname)); + } + //update central repository to be able to store new correlation attributes final String wirelessNetworsDbTableName = "wireless_networks"; final String wirelessNetworksTableInstanceName = wirelessNetworsDbTableName + "_instances"; //add the wireless_networks attribute to the correlation_types table @@ -3177,6 +3256,7 @@ abstract class AbstractSqlEamDb implements EamDb { statement.execute(String.format(addDataSourceIdIndexTemplate, wirelessNetworksTableInstanceName, wirelessNetworksTableInstanceName)); statement.execute(String.format(addValueIndexTemplate, wirelessNetworksTableInstanceName, wirelessNetworksTableInstanceName)); statement.execute(String.format(addKnownStatusIndexTemplate, wirelessNetworksTableInstanceName, wirelessNetworksTableInstanceName)); + statement.execute(String.format(addObjectIdIndexTemplate, wirelessNetworksTableInstanceName, wirelessNetworksTableInstanceName)); } if (!updateSchemaVersion(conn)) { diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeInstance.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeInstance.java index d7b2e3cf6b..86a904d368 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeInstance.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeInstance.java @@ -48,6 +48,7 @@ public class CorrelationAttributeInstance implements Serializable { private String filePath; private String comment; private TskData.FileKnown knownStatus; + private Long objectId; public CorrelationAttributeInstance( CorrelationAttributeInstance.Type correlationType, @@ -56,8 +57,9 @@ public class CorrelationAttributeInstance implements Serializable { CorrelationDataSource eamDataSource, String filePath, String comment, - TskData.FileKnown knownStatus) throws EamDbException, CorrelationAttributeNormalizationException { - this(correlationType, correlationValue, -1, eamCase, eamDataSource, filePath, comment, knownStatus); + TskData.FileKnown knownStatus, + long fileObjectId) throws EamDbException, CorrelationAttributeNormalizationException { + this(correlationType, correlationValue, -1, eamCase, eamDataSource, filePath, comment, knownStatus, fileObjectId); } CorrelationAttributeInstance( @@ -68,7 +70,8 @@ public class CorrelationAttributeInstance implements Serializable { CorrelationDataSource eamDataSource, String filePath, String comment, - TskData.FileKnown knownStatus + TskData.FileKnown knownStatus, + Long fileObjectId ) throws EamDbException, CorrelationAttributeNormalizationException { if (filePath == null) { throw new EamDbException("file path is null"); @@ -83,6 +86,7 @@ public class CorrelationAttributeInstance implements Serializable { this.filePath = filePath.toLowerCase(); this.comment = comment; this.knownStatus = knownStatus; + this.objectId = fileObjectId; } public Boolean equals(CorrelationAttributeInstance otherInstance) { diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactUtil.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactUtil.java index 5785355cd9..66b4f565f3 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactUtil.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactUtil.java @@ -213,8 +213,8 @@ public class EamArtifactUtil { CorrelationDataSource.fromTSKDataSource(correlationCase, bbSourceFile.getDataSource()), bbSourceFile.getParentPath() + bbSourceFile.getName(), "", - TskData.FileKnown.UNKNOWN - ); + TskData.FileKnown.UNKNOWN, + bbSourceFile.getId()); } catch (TskCoreException | EamDbException | CorrelationAttributeNormalizationException ex) { logger.log(Level.SEVERE, "Error creating artifact instance.", ex); // NON-NLS @@ -232,7 +232,7 @@ public class EamArtifactUtil { * * @return The new CorrelationAttribute, or null if retrieval failed. */ - public static CorrelationAttributeInstance getInstanceFromContent(Content content) { + public static CorrelationAttributeInstance getInstanceFromContent2(Content content) { if (!(content instanceof AbstractFile)) { return null; @@ -281,6 +281,70 @@ public class EamArtifactUtil { return correlationAttributeInstance; } + /** + * Retrieve CorrelationAttribute from the given Content. + * + * @param content The content object + * + * @return The new CorrelationAttribute, or null if retrieval failed. + */ + public static CorrelationAttributeInstance getInstanceFromContent(Content content) { + + if (!(content instanceof AbstractFile)) { + return null; + } + + final AbstractFile file = (AbstractFile) content; + + if (!isSupportedAbstractFileType(file)) { + return null; + } + + CorrelationAttributeInstance.Type type; + CorrelationCase correlationCase; + CorrelationDataSource correlationDataSource; + + try { + type = EamDb.getInstance().getCorrelationTypeById(CorrelationAttributeInstance.FILES_TYPE_ID); + correlationCase = EamDb.getInstance().getCase(Case.getCurrentCaseThrows()); + if (null == correlationCase) { + //if the correlationCase is not in the Central repo then attributes generated in relation to it will not be + return null; + } + correlationDataSource = CorrelationDataSource.fromTSKDataSource(correlationCase, file.getDataSource()); + } catch (TskCoreException | EamDbException ex) { + logger.log(Level.SEVERE, "Error retrieving correlation attribute.", ex); + return null; + } catch (NoCurrentCaseException ex) { + logger.log(Level.SEVERE, "Case is closed.", ex); + return null; + } + + CorrelationAttributeInstance correlationAttributeInstance; + try { + correlationAttributeInstance = EamDb.getInstance().getCorrelationAttributeInstance(type, correlationCase, correlationDataSource, content.getId()); + } catch (EamDbException | CorrelationAttributeNormalizationException ex) { + logger.log(Level.WARNING, String.format( + "Correlation attribute could not be retrieved for '%s' (id=%d): %s", + content.getName(), content.getId(), ex.getMessage())); + return null; + } + if (correlationAttributeInstance == null) { + String value = file.getMd5Hash(); + String filePath = (file.getParentPath() + file.getName()).toLowerCase(); + try { + correlationAttributeInstance = EamDb.getInstance().getCorrelationAttributeInstance(type, correlationCase, correlationDataSource, value, filePath); + } catch (EamDbException | CorrelationAttributeNormalizationException ex) { + logger.log(Level.WARNING, String.format( + "Correlation attribute could not be retrieved for '%s' (id=%d): %s", + content.getName(), content.getId(), ex.getMessage())); + return null; + } + } + + return correlationAttributeInstance; + } + /** * Create an EamArtifact from the given Content. Will return null if an * artifact can not be created - this is not necessarily an error case, it @@ -319,6 +383,7 @@ public class EamArtifactUtil { if (null == correlationCase) { correlationCase = EamDb.getInstance().newCase(Case.getCurrentCaseThrows()); } + return new CorrelationAttributeInstance( filesType, af.getMd5Hash(), @@ -326,7 +391,8 @@ public class EamArtifactUtil { CorrelationDataSource.fromTSKDataSource(correlationCase, af.getDataSource()), af.getParentPath() + af.getName(), "", - TskData.FileKnown.UNKNOWN); + TskData.FileKnown.UNKNOWN, + af.getId()); } catch (TskCoreException | EamDbException | CorrelationAttributeNormalizationException ex) { logger.log(Level.SEVERE, "Error making correlation attribute.", ex); diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDb.java index 418181d8ab..dc1420ff5e 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDb.java @@ -355,11 +355,16 @@ public interface EamDb { * * @return The correlation attribute if it exists; otherwise null. * + * @deprecated - included to support Central Reposities version 1,1 and older * @throws EamDbException */ + @Deprecated CorrelationAttributeInstance getCorrelationAttributeInstance(CorrelationAttributeInstance.Type type, CorrelationCase correlationCase, CorrelationDataSource correlationDataSource, String value, String filePath) throws EamDbException, CorrelationAttributeNormalizationException; + + CorrelationAttributeInstance getCorrelationAttributeInstance(CorrelationAttributeInstance.Type type, CorrelationCase correlationCase, + CorrelationDataSource correlationDataSource, long objectID) throws EamDbException, CorrelationAttributeNormalizationException; /** * Sets an eamArtifact instance to the given known status. If eamArtifact * exists, it is updated. If eamArtifact does not exist nothing happens diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDbSettings.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDbSettings.java index 6ee454f915..a3e3ba196a 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDbSettings.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDbSettings.java @@ -394,11 +394,11 @@ public final class PostgresEamDbSettings { String createArtifactInstancesTableTemplate = getCreateArtifactInstancesTableTemplate(); - String instancesIdx1 = getAddCaseIdIndexTemplate(); - String instancesIdx2 = getAddDataSourceIdIndexTemplate(); - - String instancesIdx3 = getAddValueIndexTemplate(); - String instancesIdx4 = getAddKnownStatusIndexTemplate(); + String instancesCaseIdIdx = getAddCaseIdIndexTemplate(); + String instancesDatasourceIdIdx = getAddDataSourceIdIndexTemplate(); + String instancesValueIdx = getAddValueIndexTemplate(); + String instancesKnownStatusIdx = getAddKnownStatusIndexTemplate(); + String instancesObjectIdIdx = getAddObjectIdIndexTemplate(); StringBuilder createDbInfoTable = new StringBuilder(); createDbInfoTable.append("CREATE TABLE IF NOT EXISTS db_info ("); @@ -443,10 +443,11 @@ public final class PostgresEamDbSettings { instance_type_dbname = EamDbUtil.correlationTypeToInstanceTableName(type); stmt.execute(String.format(createArtifactInstancesTableTemplate, instance_type_dbname, instance_type_dbname)); - stmt.execute(String.format(instancesIdx1, instance_type_dbname, instance_type_dbname)); - stmt.execute(String.format(instancesIdx2, instance_type_dbname, instance_type_dbname)); - stmt.execute(String.format(instancesIdx3, instance_type_dbname, instance_type_dbname)); - stmt.execute(String.format(instancesIdx4, instance_type_dbname, instance_type_dbname)); + stmt.execute(String.format(instancesCaseIdIdx, instance_type_dbname, instance_type_dbname)); + stmt.execute(String.format(instancesDatasourceIdIdx, instance_type_dbname, instance_type_dbname)); + stmt.execute(String.format(instancesValueIdx, instance_type_dbname, instance_type_dbname)); + stmt.execute(String.format(instancesKnownStatusIdx, instance_type_dbname, instance_type_dbname)); + stmt.execute(String.format(instancesObjectIdIdx, instance_type_dbname, instance_type_dbname)); // FUTURE: allow more than the FILES type if (type.getId() == CorrelationAttributeInstance.FILES_TYPE_ID) { @@ -486,6 +487,7 @@ public final class PostgresEamDbSettings { createArtifactInstancesTableTemplate.append("file_path text NOT NULL,"); createArtifactInstancesTableTemplate.append("known_status integer NOT NULL,"); createArtifactInstancesTableTemplate.append("comment text,"); + createArtifactInstancesTableTemplate.append("object_id integer,"); createArtifactInstancesTableTemplate.append("CONSTRAINT %s_multi_unique_ UNIQUE (data_source_id, value, file_path),"); createArtifactInstancesTableTemplate.append("foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL,"); createArtifactInstancesTableTemplate.append("foreign key (data_source_id) references data_sources(id) ON UPDATE SET NULL ON DELETE SET NULL"); @@ -545,6 +547,19 @@ public final class PostgresEamDbSettings { return "CREATE INDEX IF NOT EXISTS %s_value_known_status ON %s (value, known_status)"; } + /** + * Get the template for creating an index on the object_id column of an + * instance table. %s will exist in the template where the name of the new + * table will be addedd. + * + * @return a String which is a template for adding an index to the object_id + * column of a _instances table + */ + static String getAddObjectIdIndexTemplate() { + // Each "%s" will be replaced with the relevant TYPE_instances table name. + return "CREATE INDEX IF NOT EXISTS %s_object_id ON %s (object_id)"; + } + public boolean insertDefaultDatabaseContent() { Connection conn = getEphemeralConnection(false); if (null == conn) { diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDbSettings.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDbSettings.java index 615e49e523..2ead0cd4f3 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDbSettings.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDbSettings.java @@ -337,12 +337,12 @@ public final class SqliteEamDbSettings { String createArtifactInstancesTableTemplate = getCreateArtifactInstancesTableTemplate(); - String instancesIdx1 = getAddCaseIdIndexTemplate(); - String instancesIdx2 = getAddDataSourceIdIndexTemplate(); - - String instancesIdx3 = getAddValueIndexTemplate(); - String instancesIdx4 = getAddKnownStatusIndexTemplate(); - + String instancesCaseIdIdx = getAddCaseIdIndexTemplate(); + String instancesDatasourceIdIdx = getAddDataSourceIdIndexTemplate(); + String instancesValueIdx = getAddValueIndexTemplate(); + String instancesKnownStatusIdx = getAddKnownStatusIndexTemplate(); + String instancesObjectIdIdx = getAddObjectIdIndexTemplate(); + StringBuilder createDbInfoTable = new StringBuilder(); createDbInfoTable.append("CREATE TABLE IF NOT EXISTS db_info ("); createDbInfoTable.append("id integer primary key NOT NULL,"); @@ -392,10 +392,11 @@ public final class SqliteEamDbSettings { instance_type_dbname = EamDbUtil.correlationTypeToInstanceTableName(type); stmt.execute(String.format(createArtifactInstancesTableTemplate, instance_type_dbname, instance_type_dbname)); - stmt.execute(String.format(instancesIdx1, instance_type_dbname, instance_type_dbname)); - stmt.execute(String.format(instancesIdx2, instance_type_dbname, instance_type_dbname)); - stmt.execute(String.format(instancesIdx3, instance_type_dbname, instance_type_dbname)); - stmt.execute(String.format(instancesIdx4, instance_type_dbname, instance_type_dbname)); + stmt.execute(String.format(instancesCaseIdIdx, instance_type_dbname, instance_type_dbname)); + stmt.execute(String.format(instancesDatasourceIdIdx, instance_type_dbname, instance_type_dbname)); + stmt.execute(String.format(instancesValueIdx, instance_type_dbname, instance_type_dbname)); + stmt.execute(String.format(instancesKnownStatusIdx, instance_type_dbname, instance_type_dbname)); + stmt.execute(String.format(instancesObjectIdIdx, instance_type_dbname, instance_type_dbname)); // FUTURE: allow more than the FILES type if (type.getId() == CorrelationAttributeInstance.FILES_TYPE_ID) { @@ -415,7 +416,7 @@ public final class SqliteEamDbSettings { } return true; } - + /** * Get the template String for creating a new _instances table in a Sqlite * central repository. %s will exist in the template where the name of the @@ -434,6 +435,7 @@ public final class SqliteEamDbSettings { createArtifactInstancesTableTemplate.append("file_path text NOT NULL,"); createArtifactInstancesTableTemplate.append("known_status integer NOT NULL,"); createArtifactInstancesTableTemplate.append("comment text,"); + createArtifactInstancesTableTemplate.append("object_id integer,"); createArtifactInstancesTableTemplate.append("CONSTRAINT %s_multi_unique UNIQUE(data_source_id, value, file_path) ON CONFLICT IGNORE,"); createArtifactInstancesTableTemplate.append("foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL,"); createArtifactInstancesTableTemplate.append("foreign key (data_source_id) references data_sources(id) ON UPDATE SET NULL ON DELETE SET NULL"); @@ -493,6 +495,19 @@ public final class SqliteEamDbSettings { return "CREATE INDEX IF NOT EXISTS %s_value_known_status ON %s (value, known_status)"; } + /** + * Get the template for creating an index on the object_id column of an + * instance table. %s will exist in the template where the name of the new + * table will be addedd. + * + * @return a String which is a template for adding an index to the object_id + * column of a _instances table + */ + static String getAddObjectIdIndexTemplate() { + // Each "%s" will be replaced with the relevant TYPE_instances table name. + return "CREATE INDEX IF NOT EXISTS %s_object_id ON %s (object_id)"; + } + public boolean insertDefaultDatabaseContent() { Connection conn = getEphemeralConnection(); if (null == conn) { diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java index 8ac7acd2cd..62a10aebb2 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/ingestmodule/IngestModule.java @@ -152,13 +152,14 @@ final class IngestModule implements FileIngestModule { // insert this file into the central repository try { CorrelationAttributeInstance cefi = new CorrelationAttributeInstance( - filesType, md5, + filesType, + md5, eamCase, eamDataSource, abstractFile.getParentPath() + abstractFile.getName(), null, TskData.FileKnown.UNKNOWN // NOTE: Known status in the CR is based on tagging, not hashes like the Case Database. - ); +, abstractFile.getId()); dbManager.addAttributeInstanceBulk(cefi); } catch (EamDbException ex) { logger.log(Level.SEVERE, "Error adding artifact to bulk artifacts.", ex); // NON-NLS diff --git a/Core/src/org/sleuthkit/autopsy/contentviewers/AnnotationsContentViewer.java b/Core/src/org/sleuthkit/autopsy/contentviewers/AnnotationsContentViewer.java index 583dfca703..72343ec616 100755 --- a/Core/src/org/sleuthkit/autopsy/contentviewers/AnnotationsContentViewer.java +++ b/Core/src/org/sleuthkit/autopsy/contentviewers/AnnotationsContentViewer.java @@ -67,19 +67,19 @@ public class AnnotationsContentViewer extends javax.swing.JPanel implements Data initComponents(); Utilities.configureTextPaneAsHtml(jTextPane1); } - + @Override public void setNode(Node node) { if ((node == null) || (!isSupported(node))) { resetComponent(); return; } - + StringBuilder html = new StringBuilder(); - + BlackboardArtifact artifact = node.getLookup().lookup(BlackboardArtifact.class); Content sourceFile = null; - + try { if (artifact != null) { /* @@ -100,32 +100,32 @@ public class AnnotationsContentViewer extends javax.swing.JPanel implements Data "Exception while trying to retrieve a Content instance from the BlackboardArtifact '%s' (id=%d).", artifact.getDisplayName(), artifact.getArtifactID()), ex); } - + if (artifact != null) { populateTagData(html, artifact, sourceFile); } else { populateTagData(html, sourceFile); } - + if (sourceFile instanceof AbstractFile) { populateCentralRepositoryData(html, artifact, (AbstractFile) sourceFile); } - + setText(html.toString()); jTextPane1.setCaretPosition(0); } - + /** * Populate the "Selected Item" sections with tag data for the supplied * content. - * + * * @param html The HTML text to update. * @param content Selected content. */ private void populateTagData(StringBuilder html, Content content) { try { SleuthkitCase tskCase = Case.getCurrentCaseThrows().getSleuthkitCase(); - + startSection(html, "Selected Item"); List fileTagsList = tskCase.getContentTagsByContent(content); if (fileTagsList.isEmpty()) { @@ -142,11 +142,11 @@ public class AnnotationsContentViewer extends javax.swing.JPanel implements Data logger.log(Level.SEVERE, "Exception while getting tags from the case database.", ex); //NON-NLS } } - + /** * Populate the "Selected Item" and "Source File" sections with tag data for * a supplied artifact. - * + * * @param html The HTML text to update. * @param artifact A selected artifact. * @param sourceFile The source content of the selected artifact. @@ -154,7 +154,7 @@ public class AnnotationsContentViewer extends javax.swing.JPanel implements Data private void populateTagData(StringBuilder html, BlackboardArtifact artifact, Content sourceFile) { try { SleuthkitCase tskCase = Case.getCurrentCaseThrows().getSleuthkitCase(); - + startSection(html, "Selected Item"); List artifactTagsList = tskCase.getBlackboardArtifactTagsByArtifact(artifact); if (artifactTagsList.isEmpty()) { @@ -165,7 +165,7 @@ public class AnnotationsContentViewer extends javax.swing.JPanel implements Data } } endSection(html); - + if (sourceFile != null) { startSection(html, "Source File"); List fileTagsList = tskCase.getContentTagsByContent(sourceFile); @@ -184,10 +184,10 @@ public class AnnotationsContentViewer extends javax.swing.JPanel implements Data logger.log(Level.SEVERE, "Exception while getting tags from the case database.", ex); //NON-NLS } } - + /** * Populate the "Central Repository Comments" section with data. - * + * * @param html The HTML text to update. * @param artifact A selected artifact (can be null). * @param sourceFile A selected file, or a source file of the selected @@ -208,22 +208,24 @@ public class AnnotationsContentViewer extends javax.swing.JPanel implements Data if (attributeType.getId() == CorrelationAttributeInstance.FILES_TYPE_ID) { CorrelationCase correlationCase = EamDb.getInstance().getCase(Case.getCurrentCase()); instancesList.add(new CorrelationAttributeInstance( - attributeType, md5, + attributeType, + md5, correlationCase, CorrelationDataSource.fromTSKDataSource(correlationCase, sourceFile.getDataSource()), sourceFile.getParentPath() + sourceFile.getName(), "", - sourceFile.getKnown())); + sourceFile.getKnown(), + sourceFile.getId())); break; } } } boolean commentDataFound = false; - + for (CorrelationAttributeInstance instance : instancesList) { - List correlatedInstancesList = - EamDb.getInstance().getArtifactInstancesByTypeValue(instance.getCorrelationType(), instance.getCorrelationValue()); + List correlatedInstancesList + = EamDb.getInstance().getArtifactInstancesByTypeValue(instance.getCorrelationType(), instance.getCorrelationValue()); for (CorrelationAttributeInstance correlatedInstance : correlatedInstancesList) { if (correlatedInstance.getComment() != null && correlatedInstance.getComment().isEmpty() == false) { commentDataFound = true; @@ -231,7 +233,7 @@ public class AnnotationsContentViewer extends javax.swing.JPanel implements Data } } } - + if (commentDataFound == false) { addMessage(html, "There is no comment data for the selected content in the Central Repository."); } @@ -246,16 +248,16 @@ public class AnnotationsContentViewer extends javax.swing.JPanel implements Data /** * Set the text of the text panel. - * + * * @param text The text to set to the text panel. */ private void setText(String text) { jTextPane1.setText("" + text + ""); //NON-NLS } - + /** * Start a new data section. - * + * * @param html The HTML text to add the section to. * @param sectionName The name of the section. */ @@ -264,10 +266,10 @@ public class AnnotationsContentViewer extends javax.swing.JPanel implements Data .append(sectionName) .append("


"); //NON-NLS } - + /** * Add a message. - * + * * @param html The HTML text to add the message to. * @param message The message text. */ @@ -276,10 +278,10 @@ public class AnnotationsContentViewer extends javax.swing.JPanel implements Data .append(message) .append("


"); //NON-NLS } - + /** * Add a data table containing information about a tag. - * + * * @param html The HTML text to add the table to. * @param tag The tag whose information will be used to populate the table. */ @@ -295,11 +297,11 @@ public class AnnotationsContentViewer extends javax.swing.JPanel implements Data addRow(html, Bundle.AnnotationsContentViewer_tagEntryDataLabel_comment(), formatHtmlString(tag.getComment())); endTable(html); } - + /** * Add a data table containing information about a correlation attribute * instance in the Central Repository. - * + * * @param html The HTML text to add the table to. * @param attributeInstance The attribute instance whose information will be * used to populate the table. @@ -318,10 +320,10 @@ public class AnnotationsContentViewer extends javax.swing.JPanel implements Data addRow(html, Bundle.AnnotationsContentViewer_centralRepositoryEntryDataLabel_path(), attributeInstance.getFilePath()); endTable(html); } - + /** * Start a data table. - * + * * @param html The HTML text to add the table to. */ private void startTable(StringBuilder html) { @@ -330,7 +332,7 @@ public class AnnotationsContentViewer extends javax.swing.JPanel implements Data /** * Add a data row to a table. - * + * * @param html The HTML text to add the row to. * @param key The key for the left column of the data row. * @param value The value for the right column of the data row. @@ -342,10 +344,10 @@ public class AnnotationsContentViewer extends javax.swing.JPanel implements Data html.append(value); html.append(""); //NON-NLS } - + /** * End a data table. - * + * * @param html The HTML text on which to end a table. */ private void endTable(StringBuilder html) { @@ -354,18 +356,19 @@ public class AnnotationsContentViewer extends javax.swing.JPanel implements Data /** * End a data section. - * + * * @param html The HTML text on which to end a section. */ private void endSection(StringBuilder html) { html.append("
"); //NON-NLS } - + /** * Apply escape sequence to special characters. Line feed and carriage * return character combinations will be converted to HTML line breaks. - * + * * @param text The text to format. + * * @return The formatted text. */ private String formatHtmlString(String text) { @@ -427,7 +430,7 @@ public class AnnotationsContentViewer extends javax.swing.JPanel implements Data @Override public boolean isSupported(Node node) { BlackboardArtifact artifact = node.getLookup().lookup(BlackboardArtifact.class); - + try { if (artifact != null) { if (artifact.getSleuthkitCase().getAbstractFileById(artifact.getObjectID()) != null) { @@ -443,7 +446,7 @@ public class AnnotationsContentViewer extends javax.swing.JPanel implements Data "Exception while trying to retrieve a Content instance from the BlackboardArtifact '%s' (id=%d).", artifact.getDisplayName(), artifact.getArtifactID()), ex); } - + return false; } From 79189f3a448e0e50b10663c54bc28afbf2e6e75b Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Tue, 6 Nov 2018 17:49:11 -0500 Subject: [PATCH 067/129] 4354 update getters and setters for CorrelationAttributeInstance --- .../CorrelationAttributeInstance.java | 25 ++++++++----------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeInstance.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeInstance.java index 86a904d368..8fd6b4170e 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeInstance.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/CorrelationAttributeInstance.java @@ -119,14 +119,6 @@ public class CorrelationAttributeInstance implements Serializable { return correlationValue; } - /** - * @param correlationValue the correlationValue to set - */ - public void setCorrelationValue(String correlationValue) { - // Lower-case all values to normalize and improve correlation hits, going forward make sure this makes sense for all correlation types - this.correlationValue = correlationValue.toLowerCase(); - } - /** * @return the correlation Type */ @@ -134,13 +126,6 @@ public class CorrelationAttributeInstance implements Serializable { return correlationType; } - /** - * @param correlationType the correlation Type to set - */ - public void setCorrelationType(Type correlationType) { - this.correlationType = correlationType; - } - /** * Is this a database instance? * @@ -214,6 +199,16 @@ public class CorrelationAttributeInstance implements Serializable { this.knownStatus = knownStatus; } + /** + * Get the objectId of the file associated with the correlation attribute or + * NULL if the objectId is not available. + * + * @return the objectId of the file + */ + public Long getFileObjectId() { + return objectId; + } + // Type ID's for Default Correlation Types public static final int FILES_TYPE_ID = 0; public static final int DOMAIN_TYPE_ID = 1; From 91b4842e0b76292813d24ccf486302b8f46e2410 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Wed, 7 Nov 2018 10:02:37 -0500 Subject: [PATCH 068/129] Reverted the state idea, made the idea of the utility easier to grasp --- ...ableStream.java => SQLiteTableReader.java} | 181 ++++++------------ 1 file changed, 56 insertions(+), 125 deletions(-) rename Core/src/org/sleuthkit/autopsy/coreutils/{SQLiteTableStream.java => SQLiteTableReader.java} (72%) diff --git a/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableStream.java b/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReader.java similarity index 72% rename from Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableStream.java rename to Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReader.java index a455e7bfeb..8adef17a4a 100755 --- a/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableStream.java +++ b/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReader.java @@ -45,7 +45,7 @@ import org.sleuthkit.datamodel.TskCoreException; * * @author dsmyda */ -public class SQLiteTableStream implements AutoCloseable { +public class SQLiteTableReader implements AutoCloseable { /** * @@ -53,13 +53,14 @@ public class SQLiteTableStream implements AutoCloseable { public static class Builder { private final AbstractFile file; - private Consumer onMetaDataAction; - private Consumer> onStringAction; - private Consumer> onLongAction; - private Consumer> onIntegerAction; - private Consumer> onFloatAction; - private Consumer> onBlobAction; - private Consumer> forAllAction; + private Consumer onColumnNameAction; + + private Consumer onStringAction; + private Consumer onLongAction; + private Consumer onIntegerAction; + private Consumer onFloatAction; + private Consumer onBlobAction; + private Consumer forAllAction; /** * Creates a SQLiteTableReaderBuilder for this abstract file. @@ -78,8 +79,8 @@ public class SQLiteTableStream implements AutoCloseable { * * @return */ - public Builder onMetadata(Consumer action) { - this.onMetaDataAction = action; + public Builder onColumnNames(Consumer action) { + this.onColumnNameAction = action; return this; } @@ -91,7 +92,7 @@ public class SQLiteTableStream implements AutoCloseable { * * @return */ - public Builder onString(Consumer> action) { + public Builder onString(Consumer action) { this.onStringAction = action; return this; } @@ -104,7 +105,7 @@ public class SQLiteTableStream implements AutoCloseable { * * @return */ - public Builder onInteger(Consumer> action) { + public Builder onInteger(Consumer action) { this.onIntegerAction = action; return this; } @@ -117,7 +118,7 @@ public class SQLiteTableStream implements AutoCloseable { * * @return */ - public Builder onFloat(Consumer> action) { + public Builder onFloat(Consumer action) { this.onFloatAction = action; return this; } @@ -127,7 +128,7 @@ public class SQLiteTableStream implements AutoCloseable { * @param action * @return */ - public Builder onLong(Consumer> action) { + public Builder onLong(Consumer action) { this.onLongAction = action; return this; } @@ -137,7 +138,7 @@ public class SQLiteTableStream implements AutoCloseable { * @param action * @return */ - public Builder onBlob(Consumer> action) { + public Builder onBlob(Consumer action) { this.onBlobAction = action; return this; } @@ -149,7 +150,7 @@ public class SQLiteTableStream implements AutoCloseable { * * @return */ - public Builder forAll(Consumer> action) { + public Builder forAll(Consumer action) { this.forAllAction = action; return this; } @@ -160,17 +161,8 @@ public class SQLiteTableStream implements AutoCloseable { * * @return */ - public SQLiteTableStream build() { - return new SQLiteTableStream( - file, - onMetaDataAction, - onStringAction, - onIntegerAction, - onLongAction, - onFloatAction, - onBlobAction, - forAllAction - ); + public SQLiteTableReader build() { + return new SQLiteTableReader(this); } } @@ -180,17 +172,20 @@ public class SQLiteTableStream implements AutoCloseable { private PreparedStatement statement; private ResultSet queryResults; - private final Consumer onMetaDataAction; - private final Consumer> onStringAction; - private final Consumer> onLongAction; - private final Consumer> onIntegerAction; - private final Consumer> onFloatAction; - private final Consumer> onBlobAction; - private final Consumer> forAllAction; + private final Consumer onColumnNameAction; + private final Consumer onStringAction; + private final Consumer onLongAction; + private final Consumer onIntegerAction; + private final Consumer onFloatAction; + private final Consumer onBlobAction; + private final Consumer forAllAction; //Iteration state variables - private Integer currColumnCount; + private Integer currRowColumnIndex = 1; private boolean unfinishedRowState = false; + private Integer columnNameIndex = 1; + private Integer currentColumnCount; + private ResultSetMetaData currentMetadata; private boolean isFinished; private boolean hasOpened; @@ -202,27 +197,20 @@ public class SQLiteTableStream implements AutoCloseable { * Initialize a new table stream given the parameters passed in from the * StreamBuilder above. */ - private SQLiteTableStream(AbstractFile file, - Consumer metaDataAction, - Consumer> stringAction, - Consumer> integerAction, - Consumer> longAction, - Consumer> floatAction, - Consumer> blobAction, - Consumer> forAllAction) { + private SQLiteTableReader(Builder builder) { - this.onMetaDataAction = checkNonNull(metaDataAction); - this.onStringAction = checkNonNull(stringAction); - this.onIntegerAction = checkNonNull(integerAction); - this.onLongAction = checkNonNull(longAction); - this.onFloatAction = checkNonNull(floatAction); - this.onBlobAction = checkNonNull(blobAction); - this.forAllAction = checkNonNull(forAllAction); + this.onColumnNameAction = nonNullValue(builder.onColumnNameAction); + this.onStringAction = nonNullValue(builder.onStringAction); + this.onIntegerAction = nonNullValue(builder.onIntegerAction); + this.onLongAction = nonNullValue(builder.onLongAction); + this.onFloatAction = nonNullValue(builder.onFloatAction); + this.onBlobAction = nonNullValue(builder.onBlobAction); + this.forAllAction = nonNullValue(builder.forAllAction); - this.file = file; + this.file = builder.file; } - private Consumer checkNonNull(Consumer action) { + private Consumer nonNullValue(Consumer action) { if (Objects.nonNull(action)) { return action; } @@ -321,14 +309,8 @@ public class SQLiteTableStream implements AutoCloseable { * */ public void read(String tableName, BooleanSupplier condition) throws AutopsySQLiteException { - if(Objects.nonNull(prevTableName)) { - if(prevTableName.equals(tableName)) { - readHelper("SELECT * FROM \"" + tableName + "\"", condition); - } else { - prevTableName = tableName; - closeResultSet(); - readHelper("SELECT * FROM \"" + tableName + "\"", condition); - } + if(Objects.nonNull(prevTableName) && prevTableName.equals(tableName)) { + readHelper("SELECT * FROM \"" + tableName + "\"", condition); } else { prevTableName = tableName; closeResultSet(); @@ -349,21 +331,22 @@ public class SQLiteTableStream implements AutoCloseable { try { if(!hasOpened) { openResultSet(query); - ResultSetMetaData metaData = queryResults.getMetaData(); - this.onMetaDataAction.accept(metaData); + currentMetadata = queryResults.getMetaData(); + currentColumnCount = currentMetadata.getColumnCount(); } isFinished = false; - ResultSetMetaData metaData = queryResults.getMetaData(); - int columnCount = metaData.getColumnCount(); + for(; columnNameIndex <= currentColumnCount; columnNameIndex++) { + this.onColumnNameAction.accept(currentMetadata.getColumnName(columnNameIndex)); + } while (unfinishedRowState || queryResults.next()) { if (!unfinishedRowState) { - currColumnCount = 1; + currRowColumnIndex = 1; } - for (; currColumnCount <= columnCount; currColumnCount++) { + for (; currRowColumnIndex <= currentColumnCount; currRowColumnIndex++) { if (condition.getAsBoolean()) { unfinishedRowState = true; @@ -371,44 +354,20 @@ public class SQLiteTableStream implements AutoCloseable { } //getObject automatically instiantiates the correct java data type - Object item = queryResults.getObject(currColumnCount); + Object item = queryResults.getObject(currRowColumnIndex); if(item instanceof String) { - this.onStringAction.accept(new State<>( - (String) item, - metaData.getColumnName(currColumnCount), - currColumnCount) - ); + this.onStringAction.accept((String) item); } else if(item instanceof Integer) { - this.onIntegerAction.accept(new State<>( - (Integer) item, - metaData.getColumnName(currColumnCount), - currColumnCount) - ); + this.onIntegerAction.accept((Integer) item); } else if(item instanceof Double) { - this.onFloatAction.accept(new State<>( - (Double) item, - metaData.getColumnName(currColumnCount), - currColumnCount) - ); + this.onFloatAction.accept((Double) item); } else if(item instanceof Long) { - this.onLongAction.accept(new State<>( - (Long) item, - metaData.getColumnName(currColumnCount), - currColumnCount) - ); + this.onLongAction.accept((Long) item); } else if(item instanceof byte[]) { - this.onBlobAction.accept(new State<>( - (byte[]) item, - metaData.getColumnName(currColumnCount), - currColumnCount) - ); + this.onBlobAction.accept((byte[]) item); } - this.forAllAction.accept(new State<>( - (Object) item, - metaData.getColumnName(currColumnCount), - currColumnCount) - ); + this.forAllAction.accept((Object) item); } unfinishedRowState = false; @@ -584,32 +543,4 @@ public class SQLiteTableStream implements AutoCloseable { super.finalize(); close(); } - - /** - * - * @param - */ - public class State { - private final T value; - private final String columnName; - private final int columnIndex; - - private State(T value, String columnName, int columnIndex) { - this.value = value; - this.columnName = columnName; - this.columnIndex = columnIndex; - } - - public T getValue() { - return value; - } - - public String getColumnName() { - return columnName; - } - - public Integer getIndex() { - return columnIndex; - } - } } \ No newline at end of file From eef08cf50e53af298bf51a79d32d9433cf7d05aa Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Wed, 7 Nov 2018 10:28:19 -0500 Subject: [PATCH 069/129] 4354 insert object id into instance tables when correlation instance created --- .../datamodel/AbstractSqlEamDb.java | 13 +++-- .../datamodel/EamArtifactUtil.java | 58 +------------------ 2 files changed, 9 insertions(+), 62 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java index f57c27444f..7b7beb3174 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java @@ -808,9 +808,9 @@ abstract class AbstractSqlEamDb implements EamDb { String sql = "INSERT INTO " + tableName - + "(case_id, data_source_id, value, file_path, known_status, comment) " + + "(case_id, data_source_id, value, file_path, known_status, comment, object_id) " + "VALUES ((SELECT id FROM cases WHERE case_uid=? LIMIT 1), " - + "(SELECT id FROM data_sources WHERE device_id=? AND case_id=? LIMIT 1), ?, ?, ?, ?) " + + "(SELECT id FROM data_sources WHERE device_id=? AND case_id=? LIMIT 1), ?, ?, ?, ?, ?) " + getConflictClause(); try { @@ -824,12 +824,14 @@ abstract class AbstractSqlEamDb implements EamDb { preparedStatement.setString(4, eamArtifact.getCorrelationValue()); preparedStatement.setString(5, eamArtifact.getFilePath().toLowerCase()); preparedStatement.setByte(6, eamArtifact.getKnownStatus().getFileKnownValue()); + if ("".equals(eamArtifact.getComment())) { preparedStatement.setNull(7, Types.INTEGER); } else { preparedStatement.setString(7, eamArtifact.getComment()); } - + preparedStatement.setLong(8, eamArtifact.getFileObjectId()); + preparedStatement.executeUpdate(); } @@ -1233,9 +1235,9 @@ abstract class AbstractSqlEamDb implements EamDb { String sql = "INSERT INTO " + tableName - + " (case_id, data_source_id, value, file_path, known_status, comment) " + + " (case_id, data_source_id, value, file_path, known_status, comment, object_id) " + "VALUES ((SELECT id FROM cases WHERE case_uid=? LIMIT 1), " - + "(SELECT id FROM data_sources WHERE device_id=? AND case_id=? LIMIT 1), ?, ?, ?, ?) " + + "(SELECT id FROM data_sources WHERE device_id=? AND case_id=? LIMIT 1), ?, ?, ?, ?, ?) " + getConflictClause(); bulkPs = conn.prepareStatement(sql); @@ -1279,6 +1281,7 @@ abstract class AbstractSqlEamDb implements EamDb { } else { bulkPs.setString(7, eamArtifact.getComment()); } + bulkPs.setLong(8, eamArtifact.getFileObjectId()); bulkPs.addBatch(); } else { logger.log(Level.WARNING, ("Artifact value too long for central repository." diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactUtil.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactUtil.java index 66b4f565f3..b9e43f46ad 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactUtil.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactUtil.java @@ -225,62 +225,6 @@ public class EamArtifactUtil { } } - /** - * Retrieve CorrelationAttribute from the given Content. - * - * @param content The content object - * - * @return The new CorrelationAttribute, or null if retrieval failed. - */ - public static CorrelationAttributeInstance getInstanceFromContent2(Content content) { - - if (!(content instanceof AbstractFile)) { - return null; - } - - final AbstractFile file = (AbstractFile) content; - - if (!isSupportedAbstractFileType(file)) { - return null; - } - - CorrelationAttributeInstance.Type type; - CorrelationCase correlationCase; - CorrelationDataSource correlationDataSource; - String value; - String filePath; - - try { - type = EamDb.getInstance().getCorrelationTypeById(CorrelationAttributeInstance.FILES_TYPE_ID); - correlationCase = EamDb.getInstance().getCase(Case.getCurrentCaseThrows()); - if (null == correlationCase) { - //if the correlationCase is not in the Central repo then attributes generated in relation to it will not be - return null; - } - correlationDataSource = CorrelationDataSource.fromTSKDataSource(correlationCase, file.getDataSource()); - value = file.getMd5Hash(); - filePath = (file.getParentPath() + file.getName()).toLowerCase(); - } catch (TskCoreException | EamDbException ex) { - logger.log(Level.SEVERE, "Error retrieving correlation attribute.", ex); - return null; - } catch (NoCurrentCaseException ex) { - logger.log(Level.SEVERE, "Case is closed.", ex); - return null; - } - - CorrelationAttributeInstance correlationAttributeInstance; - try { - correlationAttributeInstance = EamDb.getInstance().getCorrelationAttributeInstance(type, correlationCase, correlationDataSource, value, filePath); - } catch (EamDbException | CorrelationAttributeNormalizationException ex) { - logger.log(Level.WARNING, String.format( - "Correlation attribute could not be retrieved for '%s' (id=%d): %s", - content.getName(), content.getId(), ex.getMessage())); - return null; - } - - return correlationAttributeInstance; - } - /** * Retrieve CorrelationAttribute from the given Content. * @@ -322,7 +266,7 @@ public class EamArtifactUtil { CorrelationAttributeInstance correlationAttributeInstance; try { - correlationAttributeInstance = EamDb.getInstance().getCorrelationAttributeInstance(type, correlationCase, correlationDataSource, content.getId()); + correlationAttributeInstance = EamDb.getInstance().getCorrelationAttributeInstance(type, correlationCase, correlationDataSource, file.getId()); } catch (EamDbException | CorrelationAttributeNormalizationException ex) { logger.log(Level.WARNING, String.format( "Correlation attribute could not be retrieved for '%s' (id=%d): %s", From 547e1001f0c92d440b00db65f742c07993e48f3b Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Wed, 7 Nov 2018 11:27:56 -0500 Subject: [PATCH 070/129] 4354 add comment to explain why object_id query for correlation attr might fail --- .../autopsy/centralrepository/datamodel/EamArtifactUtil.java | 1 + 1 file changed, 1 insertion(+) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactUtil.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactUtil.java index b9e43f46ad..7d098fc590 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactUtil.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamArtifactUtil.java @@ -273,6 +273,7 @@ public class EamArtifactUtil { content.getName(), content.getId(), ex.getMessage())); return null; } + //if there was no correlation attribute found for the item using object_id then check for attributes added with schema 1,1 which lack object_id if (correlationAttributeInstance == null) { String value = file.getMd5Hash(); String filePath = (file.getParentPath() + file.getName()).toLowerCase(); From d3f6678f8b3f67dba572d83ad30e22d38f89a5ef Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Wed, 7 Nov 2018 11:28:38 -0500 Subject: [PATCH 071/129] Bug fixes and file rename --- .../autopsy/coreutils/SQLiteTableReader.java | 49 ++++++++++--------- ...n.java => SQLiteTableReaderException.java} | 6 +-- 2 files changed, 28 insertions(+), 27 deletions(-) rename Core/src/org/sleuthkit/autopsy/coreutils/{AutopsySQLiteException.java => SQLiteTableReaderException.java} (64%) diff --git a/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReader.java b/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReader.java index 8adef17a4a..6a5540d222 100755 --- a/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReader.java +++ b/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReader.java @@ -181,9 +181,9 @@ public class SQLiteTableReader implements AutoCloseable { private final Consumer forAllAction; //Iteration state variables - private Integer currRowColumnIndex = 1; - private boolean unfinishedRowState = false; - private Integer columnNameIndex = 1; + private Integer currRowColumnIndex; + private boolean unfinishedRowState; + private Integer columnNameIndex; private Integer currentColumnCount; private ResultSetMetaData currentMetadata; @@ -224,10 +224,10 @@ public class SQLiteTableReader implements AutoCloseable { * Get table names from database * * @return - * @throws org.sleuthkit.autopsy.coreutils.AutopsySQLiteException + * @throws org.sleuthkit.autopsy.coreutils.SQLiteTableReaderException * */ - public List getTableNames() throws AutopsySQLiteException { + public List getTableNames() throws SQLiteTableReaderException { ensureOpen(); List tableNames = new ArrayList<>(); @@ -239,7 +239,7 @@ public class SQLiteTableReader implements AutoCloseable { tableNames.add(tableNameResult.getString("name")); //NON-NLS } } catch (SQLException ex) { - throw new AutopsySQLiteException(ex); + throw new SQLiteTableReaderException(ex); } return tableNames; @@ -249,9 +249,9 @@ public class SQLiteTableReader implements AutoCloseable { * * @param tableName * @return - * @throws org.sleuthkit.autopsy.coreutils.AutopsySQLiteException + * @throws org.sleuthkit.autopsy.coreutils.SQLiteTableReaderException */ - public int getRowCount(String tableName) throws AutopsySQLiteException { + public int getRowCount(String tableName) throws SQLiteTableReaderException { ensureOpen(); try (ResultSet countResult = conn.createStatement() @@ -259,11 +259,11 @@ public class SQLiteTableReader implements AutoCloseable { "\"" + tableName + "\"")) { return countResult.getInt("count"); } catch (SQLException ex) { - throw new AutopsySQLiteException(ex); + throw new SQLiteTableReaderException(ex); } } - public int getColumnCount(String tableName) throws AutopsySQLiteException { + public int getColumnCount(String tableName) throws SQLiteTableReaderException { ensureOpen(); try (ResultSet columnCount = conn.createStatement() @@ -271,16 +271,16 @@ public class SQLiteTableReader implements AutoCloseable { "\"" + tableName + "\"")) { return columnCount.getMetaData().getColumnCount(); } catch (SQLException ex) { - throw new AutopsySQLiteException(ex); + throw new SQLiteTableReaderException(ex); } } /** * * @param tableName - * @throws org.sleuthkit.autopsy.coreutils.AutopsySQLiteException + * @throws org.sleuthkit.autopsy.coreutils.SQLiteTableReaderException */ - public void read(String tableName) throws AutopsySQLiteException { + public void read(String tableName) throws SQLiteTableReaderException { readHelper("SELECT * FROM \"" + tableName +"\"", alwaysFalseCondition); } @@ -291,10 +291,10 @@ public class SQLiteTableReader implements AutoCloseable { * @param tableName * @param limit * @param offset - * @throws org.sleuthkit.autopsy.coreutils.AutopsySQLiteException + * @throws org.sleuthkit.autopsy.coreutils.SQLiteTableReaderException * */ - public void read(String tableName, int limit, int offset) throws AutopsySQLiteException { + public void read(String tableName, int limit, int offset) throws SQLiteTableReaderException { readHelper("SELECT * FROM \"" + tableName +"\" LIMIT " + limit + " OFFSET " + offset, alwaysFalseCondition); } @@ -305,10 +305,10 @@ public class SQLiteTableReader implements AutoCloseable { * * @param tableName * @param condition - * @throws org.sleuthkit.autopsy.coreutils.AutopsySQLiteException + * @throws org.sleuthkit.autopsy.coreutils.SQLiteTableReaderException * */ - public void read(String tableName, BooleanSupplier condition) throws AutopsySQLiteException { + public void read(String tableName, BooleanSupplier condition) throws SQLiteTableReaderException { if(Objects.nonNull(prevTableName) && prevTableName.equals(tableName)) { readHelper("SELECT * FROM \"" + tableName + "\"", condition); } else { @@ -327,12 +327,13 @@ public class SQLiteTableReader implements AutoCloseable { * * @throws org.sleuthkit.autopsy.core.AutopsySQLiteException */ - private void readHelper(String query, BooleanSupplier condition) throws AutopsySQLiteException { + private void readHelper(String query, BooleanSupplier condition) throws SQLiteTableReaderException { try { if(!hasOpened) { openResultSet(query); currentMetadata = queryResults.getMetaData(); currentColumnCount = currentMetadata.getColumnCount(); + columnNameIndex = 1; } isFinished = false; @@ -367,7 +368,7 @@ public class SQLiteTableReader implements AutoCloseable { this.onBlobAction.accept((byte[]) item); } - this.forAllAction.accept((Object) item); + this.forAllAction.accept(item); } unfinishedRowState = false; @@ -378,7 +379,7 @@ public class SQLiteTableReader implements AutoCloseable { } catch (SQLException ex) { closeResultSet(); isFinished = true; - throw new AutopsySQLiteException(ex); + throw new SQLiteTableReaderException(ex); } } @@ -386,7 +387,7 @@ public class SQLiteTableReader implements AutoCloseable { * * @throws org.sleuthkit.autopsy.core.AutopsySQLiteException */ - private void ensureOpen() throws AutopsySQLiteException { + private void ensureOpen() throws SQLiteTableReaderException { if (Objects.isNull(conn)) { try { Class.forName("org.sqlite.JDBC"); //NON-NLS @@ -395,7 +396,7 @@ public class SQLiteTableReader implements AutoCloseable { conn = DriverManager.getConnection("jdbc:sqlite:" + localDiskPath); } catch (NoCurrentCaseException | TskCoreException | IOException | ClassNotFoundException | SQLException ex) { - throw new AutopsySQLiteException(ex); + throw new SQLiteTableReaderException(ex); } } } @@ -479,7 +480,7 @@ public class SQLiteTableReader implements AutoCloseable { * @param query * @throws SQLException */ - private void openResultSet(String query) throws AutopsySQLiteException { + private void openResultSet(String query) throws SQLiteTableReaderException { ensureOpen(); try { @@ -488,7 +489,7 @@ public class SQLiteTableReader implements AutoCloseable { queryResults = statement.executeQuery(); hasOpened = true; } catch (SQLException ex) { - throw new AutopsySQLiteException(ex); + throw new SQLiteTableReaderException(ex); } } diff --git a/Core/src/org/sleuthkit/autopsy/coreutils/AutopsySQLiteException.java b/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReaderException.java similarity index 64% rename from Core/src/org/sleuthkit/autopsy/coreutils/AutopsySQLiteException.java rename to Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReaderException.java index 7bc9adb727..177bd7a500 100755 --- a/Core/src/org/sleuthkit/autopsy/coreutils/AutopsySQLiteException.java +++ b/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReaderException.java @@ -9,13 +9,13 @@ package org.sleuthkit.autopsy.coreutils; * * @author dsmyda */ -public class AutopsySQLiteException extends Exception { +public class SQLiteTableReaderException extends Exception { - public AutopsySQLiteException(String msg, Throwable ex) { + public SQLiteTableReaderException(String msg, Throwable ex) { super(msg, ex); } - public AutopsySQLiteException(Throwable ex) { + public SQLiteTableReaderException(Throwable ex) { super(ex); } } From aac459f4b5b9f2ea79abcb9f6ace082575af5bb7 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Wed, 7 Nov 2018 13:29:32 -0500 Subject: [PATCH 072/129] Made content viewer impl new streaming sqlite class --- .../autopsy/contentviewers/SQLiteViewer.java | 285 ++++++++---------- 1 file changed, 118 insertions(+), 167 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/contentviewers/SQLiteViewer.java b/Core/src/org/sleuthkit/autopsy/contentviewers/SQLiteViewer.java index 9c09ca01ca..541b057d9e 100755 --- a/Core/src/org/sleuthkit/autopsy/contentviewers/SQLiteViewer.java +++ b/Core/src/org/sleuthkit/autopsy/contentviewers/SQLiteViewer.java @@ -24,18 +24,11 @@ import java.awt.Cursor; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.ResultSet; -import java.sql.ResultSetMetaData; -import java.sql.SQLException; -import java.sql.Statement; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.LinkedHashMap; -import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Objects; @@ -48,11 +41,11 @@ import org.apache.commons.io.FilenameUtils; import org.openide.util.NbBundle; import org.openide.windows.WindowManager; import org.sleuthkit.autopsy.casemodule.Case; -import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; +import org.sleuthkit.autopsy.coreutils.SQLiteTableReaderException; import org.sleuthkit.autopsy.coreutils.Logger; -import org.sleuthkit.datamodel.AbstractFile; -import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; +import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.autopsy.coreutils.SQLiteTableReader; /** * A file content viewer for SQLite database files. @@ -66,11 +59,15 @@ class SQLiteViewer extends javax.swing.JPanel implements FileTypeViewer { private static final Logger logger = Logger.getLogger(FileViewer.class.getName()); private final SQLiteTableView selectedTableView = new SQLiteTableView(); private AbstractFile sqliteDbFile; - private File tmpDbFile; - private Connection connection; + + private SQLiteTableReader viewReader; + + private Map rowMap = new LinkedHashMap<>(); + private List> chunk = new ArrayList<>(); + private int numRows; // num of rows in the selected table private int currPage = 0; // curr page of rows being displayed - + /** * Constructs a file content viewer for SQLite database files. */ @@ -264,18 +261,18 @@ class SQLiteViewer extends javax.swing.JPanel implements FileTypeViewer { }//GEN-LAST:event_tablesDropdownListActionPerformed /** - * The action when the Export Csv button is pressed. The file chooser window will pop - * up to choose where the user wants to save the csv file. The default location is case export directory. + * The action when the Export Csv button is pressed. The file chooser window + * will pop up to choose where the user wants to save the csv file. The + * default location is case export directory. * * @param evt the action event */ - @NbBundle.Messages({"SQLiteViewer.csvExport.fileName.empty=Please input a file name for exporting.", - "SQLiteViewer.csvExport.title=Export to csv file", - "SQLiteViewer.csvExport.confirm.msg=Do you want to overwrite the existing file?"}) + "SQLiteViewer.csvExport.title=Export to csv file", + "SQLiteViewer.csvExport.confirm.msg=Do you want to overwrite the existing file?"}) private void exportCsvButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_exportCsvButtonActionPerformed Case openCase = Case.getCurrentCase(); - File caseDirectory = new File(openCase.getExportDirectory()); + File caseDirectory = new File(openCase.getExportDirectory()); JFileChooser fileChooser = new JFileChooser(); fileChooser.setDragEnabled(false); fileChooser.setCurrentDirectory(caseDirectory); @@ -292,14 +289,14 @@ class SQLiteViewer extends javax.swing.JPanel implements FileTypeViewer { File file = fileChooser.getSelectedFile(); if (file.exists() && FilenameUtils.getExtension(file.getName()).equalsIgnoreCase("csv")) { if (JOptionPane.YES_OPTION == JOptionPane.showConfirmDialog(this, - Bundle.SQLiteViewer_csvExport_confirm_msg(), - Bundle.SQLiteViewer_csvExport_title(), + Bundle.SQLiteViewer_csvExport_confirm_msg(), + Bundle.SQLiteViewer_csvExport_title(), JOptionPane.YES_NO_OPTION)) { } else { return; - } + } } - + exportTableToCsv(file); } }//GEN-LAST:event_exportCsvButtonActionPerformed @@ -328,6 +325,7 @@ class SQLiteViewer extends javax.swing.JPanel implements FileTypeViewer { public void setFile(AbstractFile file) { WindowManager.getDefault().getMainWindow().setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR)); sqliteDbFile = file; + initReader(); processSQLiteFile(); WindowManager.getDefault().getMainWindow().setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR)); } @@ -342,17 +340,10 @@ class SQLiteViewer extends javax.swing.JPanel implements FileTypeViewer { tablesDropdownList.setEnabled(true); tablesDropdownList.removeAllItems(); numEntriesField.setText(""); - - // close DB connection to file - if (null != connection) { - try { - connection.close(); - connection = null; - } catch (SQLException ex) { - logger.log(Level.SEVERE, "Failed to close DB connection to file.", ex); //NON-NLS - } - } - + viewReader.close(); + rowMap = new LinkedHashMap<>(); + chunk = new ArrayList<>(); + viewReader = null; sqliteDbFile = null; } @@ -368,17 +359,10 @@ class SQLiteViewer extends javax.swing.JPanel implements FileTypeViewer { "SQLiteViewer.errorMessage.failedToinitJDBCDriver=The JDBC driver for SQLite could not be loaded.", "# {0} - exception message", "SQLiteViewer.errorMessage.unexpectedError=An unexpected error occurred:\n{0).",}) private void processSQLiteFile() { - - tablesDropdownList.removeAllItems(); - try { - String localDiskPath = SqliteUtil.writeAbstractFileToLocalDisk(sqliteDbFile); - SqliteUtil.findAndCopySQLiteMetaFile(sqliteDbFile); - // Load the SQLite JDBC driver, if necessary. - Class.forName("org.sqlite.JDBC"); //NON-NLS - connection = DriverManager.getConnection("jdbc:sqlite:" + localDiskPath); //NON-NLS + tablesDropdownList.removeAllItems(); - Collection dbTablesMap = getTables(); + Collection dbTablesMap = viewReader.getTableNames(); if (dbTablesMap.isEmpty()) { tablesDropdownList.addItem(Bundle.SQLiteViewer_comboBox_noTableEntry()); tablesDropdownList.setEnabled(false); @@ -387,46 +371,20 @@ class SQLiteViewer extends javax.swing.JPanel implements FileTypeViewer { tablesDropdownList.addItem(tableName); }); } - } catch (ClassNotFoundException ex) { - logger.log(Level.SEVERE, String.format("Failed to initialize JDBC SQLite '%s' (objId=%d)", sqliteDbFile.getName(), sqliteDbFile.getId()), ex); //NON-NLS - MessageNotifyUtil.Message.error(Bundle.SQLiteViewer_errorMessage_failedToinitJDBCDriver()); - } catch (SQLException ex) { - logger.log(Level.SEVERE, String.format("Failed to get tables from DB file '%s' (objId=%d)", sqliteDbFile.getName(), sqliteDbFile.getId()), ex); //NON-NLS + } catch (SQLiteTableReaderException ex) { + logger.log(Level.WARNING, String.format("Unable to get table names " + + "from sqlite file [%s] with id=[%d].", sqliteDbFile.getName(), + sqliteDbFile.getId(), ex.getMessage())); MessageNotifyUtil.Message.error(Bundle.SQLiteViewer_errorMessage_failedToQueryDatabase()); - } catch (IOException | NoCurrentCaseException | TskCoreException ex) { - logger.log(Level.SEVERE, String.format("Failed to create temp copy of DB file '%s' (objId=%d)", sqliteDbFile.getName(), sqliteDbFile.getId()), ex); //NON-NLS - MessageNotifyUtil.Message.error(Bundle.SQLiteViewer_errorMessage_failedToExtractFile()); } } - /** - * Gets a collection of table names from the SQLite database file. - * - * @return A collection of table names - */ - private Collection getTables() throws SQLException { - Collection tableNames = new LinkedList<>(); - try (Statement statement = connection.createStatement(); - ResultSet resultSet = statement.executeQuery( - "SELECT name FROM sqlite_master " - + " WHERE type= 'table' ")){ - while (resultSet.next()) { - tableNames.add(resultSet.getString("name")); //NON-NLS - } - } - return tableNames; - } - @NbBundle.Messages({"# {0} - tableName", "SQLiteViewer.selectTable.errorText=Error getting row count for table: {0}" }) private void selectTable(String tableName) { - - try (Statement statement = connection.createStatement(); - ResultSet resultSet = statement.executeQuery( - "SELECT count (*) as count FROM " + "\"" + tableName + "\"")) { //NON-NLS{ - - numRows = resultSet.getInt("count"); + try { + numRows = viewReader.getRowCount(tableName); numEntriesField.setText(numRows + " entries"); currPage = 1; @@ -444,120 +402,113 @@ class SQLiteViewer extends javax.swing.JPanel implements FileTypeViewer { nextPageButton.setEnabled(false); selectedTableView.setupTable(Collections.emptyList()); } - - } catch (SQLException ex) { - logger.log(Level.SEVERE, String.format("Failed to load table %s from DB file '%s' (objId=%d)", tableName, sqliteDbFile.getName(), sqliteDbFile.getId()), ex); //NON-NLS + } catch (SQLiteTableReaderException ex) { + logger.log(Level.WARNING, String.format("Failed to load table %s from DB file '%s' (objId=%d)", tableName, sqliteDbFile.getName(), sqliteDbFile.getId()), ex); //NON-NLS MessageNotifyUtil.Message.error(Bundle.SQLiteViewer_selectTable_errorText(tableName)); } } + private String prevTableName; @NbBundle.Messages({"# {0} - tableName", "SQLiteViewer.readTable.errorText=Error getting rows for table: {0}"}) private void readTable(String tableName, int startRow, int numRowsToRead) { - - try ( - Statement statement = connection.createStatement(); - ResultSet resultSet = statement.executeQuery( - "SELECT * FROM " + "\"" + tableName + "\"" - + " LIMIT " + Integer.toString(numRowsToRead) - + " OFFSET " + Integer.toString(startRow - 1))) { - - List> rows = resultSetToArrayList(resultSet); - if (Objects.nonNull(rows)) { - selectedTableView.setupTable(rows); - } else { - selectedTableView.setupTable(Collections.emptyList()); + try { + if(!tableName.equals(prevTableName)) { + prevTableName = tableName; + header = new ArrayList<>(); + rowIndex = 0; } - } catch (SQLException ex) { - logger.log(Level.SEVERE, String.format("Failed to read table %s from DB file '%s' (objId=%d)", tableName, sqliteDbFile.getName(), sqliteDbFile.getId()), ex); //NON-NLS + viewReader.read(tableName, numRowsToRead, startRow - 1); + selectedTableView.setupTable(chunk); + chunk = new ArrayList<>(); + } catch (SQLiteTableReaderException ex) { + logger.log(Level.WARNING, String.format("Failed to read table %s from DB file '%s' " //NON-NLS + + "(objId=%d) starting at row [%d] and limit [%d]", //NON-NLS + tableName, sqliteDbFile.getName(), sqliteDbFile.getId(), startRow - 1, numRowsToRead), ex.getMessage()); MessageNotifyUtil.Message.error(Bundle.SQLiteViewer_readTable_errorText(tableName)); } } - @NbBundle.Messages("SQLiteViewer.BlobNotShown.message=BLOB Data not shown") - private List> resultSetToArrayList(ResultSet resultSet) throws SQLException { - ResultSetMetaData metaData = resultSet.getMetaData(); - int columns = metaData.getColumnCount(); - ArrayList> rowlist = new ArrayList<>(); - while (resultSet.next()) { - Map row = new LinkedHashMap<>(columns); - for (int i = 1; i <= columns; ++i) { - if (resultSet.getObject(i) == null) { - row.put(metaData.getColumnName(i), ""); - } else { - if (metaData.getColumnTypeName(i).compareToIgnoreCase("blob") == 0) { - row.put(metaData.getColumnName(i), Bundle.SQLiteViewer_BlobNotShown_message()); - } else { - row.put(metaData.getColumnName(i), resultSet.getObject(i)); - } - } - } - rowlist.add(row); - } + List header; + private int rowIndex; + private void initReader() { + viewReader = new SQLiteTableReader.Builder(sqliteDbFile) + .onColumnNames((columnName) -> { + header.add(columnName); + }) + .forAll((Object o) -> { + rowIndex++; + String objectStr = (o instanceof byte[]) ? "BLOB Data not shown" : Objects.toString(o, ""); - return rowlist; + rowMap.put(header.get(rowIndex - 1), objectStr); + //If this result is at the end of a row, then add it to the + //chunk! + if (rowIndex == header.size()) { + chunk.add(rowMap); + rowMap = new LinkedHashMap<>(); + } + rowIndex %= header.size(); + }).build(); } + + private int csvRowIndex; + private int columnCount; + private boolean afterHeader; @NbBundle.Messages({"SQLiteViewer.exportTableToCsv.write.errText=Failed to export table content to csv file.", - "SQLiteViewer.exportTableToCsv.FileName=File name: ", - "SQLiteViewer.exportTableToCsv.TableName=Table name: " + "SQLiteViewer.exportTableToCsv.FileName=File name: ", + "SQLiteViewer.exportTableToCsv.TableName=Table name: " }) private void exportTableToCsv(File file) { + csvRowIndex = 0; + columnCount = 0; + afterHeader = true; + File csvFile = new File(file.toString() + ".csv"); String tableName = (String) this.tablesDropdownList.getSelectedItem(); - try ( - Statement statement = connection.createStatement(); - ResultSet resultSet = statement.executeQuery("SELECT * FROM " + "\"" + tableName + "\"")) { - List> currentTableRows = resultSetToArrayList(resultSet); - - if (Objects.isNull(currentTableRows) || currentTableRows.isEmpty()) { - logger.log(Level.INFO, String.format("The table %s is empty. (objId=%d)", tableName, sqliteDbFile.getId())); //NON-NLS - } else { - File csvFile; - String fileName = file.getName(); - if (FilenameUtils.getExtension(fileName).equalsIgnoreCase("csv")) { - csvFile = file; - } else { - csvFile = new File(file.toString() + ".csv"); - } - - try (FileOutputStream out = new FileOutputStream(csvFile, false)) { - - out.write((Bundle.SQLiteViewer_exportTableToCsv_FileName() + csvFile.getName() + "\n").getBytes()); - out.write((Bundle.SQLiteViewer_exportTableToCsv_TableName() + tableName + "\n").getBytes()); - // Set up the column names - Map row = currentTableRows.get(0); - StringBuffer header = new StringBuffer(); - for (Map.Entry col : row.entrySet()) { - String colName = col.getKey(); - if (header.length() > 0) { - header.append(',').append(colName); - } else { - header.append(colName); - } - } - out.write(header.append('\n').toString().getBytes()); - - for (Map maps : currentTableRows) { - StringBuffer valueLine = new StringBuffer(); - maps.values().forEach((value) -> { - if (valueLine.length() > 0) { - valueLine.append(',').append(value.toString()); - } else { - valueLine.append(value.toString()); + try (FileOutputStream out = new FileOutputStream(csvFile, false)) { + try (SQLiteTableReader sqliteStream = new SQLiteTableReader.Builder(sqliteDbFile) + .onColumnNames((columnName) -> { + columnCount++; + try { + if(columnCount == 1) { + columnName = "\"" + columnName + "\""; + } else { + columnName = ",\"" + columnName + "\""; } - }); - out.write(valueLine.append('\n').toString().getBytes()); - } - } + out.write(columnName.getBytes()); + } catch (IOException ex) { + + } + }).forAll((Object o) -> { + csvRowIndex++; + String objectStr = (o instanceof byte[]) ? "BLOB Data not shown" : Objects.toString(o, ""); + objectStr = "\"" + objectStr + "\""; + + if (csvRowIndex > 1) { + objectStr = "," + objectStr; + } if(csvRowIndex == columnCount) { + objectStr += "\n"; + } + + if(afterHeader) { + objectStr = "\n" + objectStr; + afterHeader = false; + } + + try { + out.write(objectStr.getBytes()); + } catch (IOException ex) { + + } + csvRowIndex = csvRowIndex % columnCount; + }).build()) { + sqliteStream.read(tableName); } - } catch (SQLException ex) { - logger.log(Level.SEVERE, String.format("Failed to read table %s from DB file '%s' (objId=%d)", tableName, sqliteDbFile.getName(), sqliteDbFile.getId()), ex); //NON-NLS - MessageNotifyUtil.Message.error(Bundle.SQLiteViewer_readTable_errorText(tableName)); - } catch (IOException ex) { - logger.log(Level.SEVERE, String.format("Failed to export table %s to file '%s'", tableName, file.getName()), ex); //NON-NLS + } catch (IOException | SQLiteTableReaderException ex) { + logger.log(Level.WARNING, String.format("Failed to export table [%s]" + + " to CSV in sqlite file '%s' (objId=%d)", tableName, sqliteDbFile.getName(), + sqliteDbFile.getId()), ex.getMessage()); //NON-NLS MessageNotifyUtil.Message.error(Bundle.SQLiteViewer_exportTableToCsv_write_errText()); } } - - } From fbed670a4bd9a9120f5e77cd9dc7995537d7c1b5 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Wed, 7 Nov 2018 13:30:51 -0500 Subject: [PATCH 073/129] Fixed text extractor using streaming impl --- .../keywordsearch/SqliteTextExtractor.java | 385 ++++++------------ 1 file changed, 128 insertions(+), 257 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SqliteTextExtractor.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SqliteTextExtractor.java index c8bbe289e4..bbf55553dc 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SqliteTextExtractor.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SqliteTextExtractor.java @@ -1,41 +1,17 @@ -/* - * Autopsy Forensic Browser - * - * Copyright 2018-2018 Basis Technology Corp. - * Contact: carrier sleuthkit org - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ package org.sleuthkit.autopsy.keywordsearch; import com.google.common.io.CharSource; import java.io.IOException; import java.io.Reader; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.ResultSet; -import java.sql.ResultSetMetaData; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.Collection; import java.util.Iterator; -import java.util.LinkedList; +import java.util.Objects; import java.util.logging.Level; -import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; +import org.openide.util.Exceptions; +import org.sleuthkit.autopsy.coreutils.SQLiteTableReaderException; import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.autopsy.coreutils.SQLiteTableReader; import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.AbstractFile; -import org.sleuthkit.datamodel.TskCoreException; /** * Dedicated SqliteTextExtractor to solve the problems associated with Tika's @@ -91,267 +67,162 @@ class SqliteTextExtractor extends ContentTextExtractor { */ @Override public Reader getReader(Content source) throws TextExtractorException { - try { - //Firewall for any content that is not an AbstractFile - if (!AbstractFile.class.isInstance(source)) { + //Firewall for any content that is not an AbstractFile + if (!AbstractFile.class.isInstance(source)) { + try { return CharSource.wrap(EMPTY_CHARACTER_SEQUENCE).openStream(); + } catch (IOException ex) { + throw new TextExtractorException("", ex); } - return new SQLiteTableReader((AbstractFile) source); - } catch (NoCurrentCaseException | IOException | TskCoreException - | ClassNotFoundException | SQLException ex) { - throw new TextExtractorException( - String.format("Encountered an issue while trying to initialize " //NON-NLS - + "a sqlite table steamer for abstract file with id: [%s], name: " //NON-NLS - + "[%s].", source.getId(), source.getName()), ex); //NON-NLS } + + return new SQLiteStreamReader((AbstractFile) source); } - /** - * Lazily loads tables from the database during reading to conserve memory. - */ - private class SQLiteTableReader extends Reader { + public class SQLiteStreamReader extends Reader { - private final Iterator tableIterator; - private final Connection connection; - private Reader currentTableReader; - private final AbstractFile source; + private final SQLiteTableReader reader; + private Iterator tableNames; + private String currentTableName; - /** - * Creates a reader that streams each table into memory and wraps a - * reader around it. Designed to save memory for large databases. - * - * @param file Sqlite database file - * - * @throws NoCurrentCaseException Current case has closed - * @throws IOException Exception copying abstract file over - * to local temp directory - * @throws TskCoreException Exception using file manager to find - * meta files - * @throws ClassNotFoundException Could not find sqlite JDBC class - * @throws SQLException Could not establish jdbc connection - */ - public SQLiteTableReader(AbstractFile file) throws NoCurrentCaseException, - IOException, TskCoreException, ClassNotFoundException, SQLException { - source = file; + private char[] buf; + private UnfinishedState unfinishedRead; + private int rowIndex; + private int columnCount; + private int totalColumns; - String localDiskPath = SqliteUtil.writeAbstractFileToLocalDisk(file); - SqliteUtil.findAndCopySQLiteMetaFile(file); - Class.forName("org.sqlite.JDBC"); //NON-NLS - connection = DriverManager.getConnection("jdbc:sqlite:" + localDiskPath); //NON-NLS - tableIterator = getTables().iterator(); - } + private int bufIndex; - /** - * Gets the table names from the SQLite database file. - * - * @return Collection of table names from the database schema - */ - private Collection getTables() throws SQLException { - Collection tableNames = new LinkedList<>(); - try (Statement statement = connection.createStatement(); - ResultSet resultSet = statement.executeQuery( - "SELECT name FROM sqlite_master " - + " WHERE type= 'table' ")) { - while (resultSet.next()) { - tableNames.add(resultSet.getString("name")); //NON-NLS - } - } - return tableNames; - } - - /** - * Reads from a database table and loads the contents into a table - * builder so that its properly formatted during indexing. - * - * @param tableName Database table to be read - */ - private String getTableAsString(String tableName) { - TableBuilder table = new TableBuilder(); - table.addTableName(tableName); - String quotedTableName = "\"" + tableName + "\""; - - try (Statement statement = connection.createStatement(); - ResultSet resultSet = statement.executeQuery( - "SELECT * FROM " + quotedTableName)) { //NON-NLS - ResultSetMetaData metaData = resultSet.getMetaData(); - int columnCount = resultSet.getMetaData().getColumnCount(); - Collection row = new LinkedList<>(); - - //Add column names once from metadata - for (int i = 1; i <= columnCount; i++) { - row.add(metaData.getColumnName(i)); - } - - table.addHeader(row); - while (resultSet.next()) { - row = new LinkedList<>(); - for (int i = 1; i <= columnCount; i++) { - Object result = resultSet.getObject(i); - String type = metaData.getColumnTypeName(i); - if (isValuableResult(result, type)) { - row.add(resultSet.getObject(i).toString()); + public SQLiteStreamReader(AbstractFile file) { + reader = new SQLiteTableReader.Builder(file) + .onColumnNames((columnName) -> { + if(columnCount == 0) { + fillBuffer("\n"+currentTableName + "\n\n\t"); } - } - table.addRow(row); + columnCount++; + + fillBuffer(columnName + ((columnCount == totalColumns) ? "\n" :" ")); + }) + .forAll((Object o) -> { + rowIndex++; + //Ignore blobs + String objectStr = (o instanceof byte[]) ? "" : Objects.toString(o, ""); + + if(rowIndex > 1 && rowIndex < totalColumns) { + objectStr += " "; + } if(rowIndex == 1){ + objectStr = "\t" + objectStr + " "; + } if(rowIndex == totalColumns) { + objectStr += "\n"; + } + + fillBuffer(objectStr); + rowIndex = rowIndex % totalColumns; + }).build(); + } + + private void fillBuffer(String val) { + for (int i = 0; i < val.length(); i++) { + if (bufIndex != buf.length) { + buf[bufIndex++] = val.charAt(i); + } else { + unfinishedRead = new UnfinishedState(val, i); + break; } - table.addCell("\n"); - } catch (SQLException ex) { - logger.log(Level.WARNING, String.format( - "Error attempting to read file table: [%s]" //NON-NLS - + " for file: [%s] (id=%d).", tableName, //NON-NLS - source.getName(), source.getId()), ex); } - - return table.toString(); } - /** - * Determines if the result from the result set is worth adding to the - * row. Ignores nulls and blobs for the time being. - * - * @param result Object result retrieved from resultSet - * @param type Type of objet retrieved from resultSet - * - * @return boolean where true means valuable, false implies it can be - * skipped. - */ - private boolean isValuableResult(Object result, String type) { - //Ignore nulls and blobs - return result != null && type.compareToIgnoreCase("blob") != 0; - } - - /** - * Loads a database file into the character buffer. The underlying - * implementation here only loads one table at a time to conserve - * memory. - * - * @param cbuf Buffer to copy database content characters into - * @param off offset to begin loading in buffer - * @param len length of the buffer - * - * @return The number of characters read from the reader - * - * @throws IOException If there is an error with the CharSource wrapping - */ @Override public int read(char[] cbuf, int off, int len) throws IOException { - if (currentTableReader == null) { - String tableResults = getNextTable(); - if (tableResults == null) { + buf = cbuf; + + bufIndex = off; + + if (Objects.isNull(tableNames)) { + try { + tableNames = reader.getTableNames().iterator(); + } catch (SQLiteTableReaderException ex) { + //Can't get table names so can't read the file! return -1; } - currentTableReader = CharSource.wrap(tableResults).openStream(); } - int charactersRead = currentTableReader.read(cbuf, off, len); - while (charactersRead == -1) { - String tableResults = getNextTable(); - if (tableResults == null) { - return -1; + if (Objects.nonNull(unfinishedRead) && !unfinishedRead.isFinished()) { + bufIndex += unfinishedRead.read(cbuf, off, len); + } + + //while buffer is not full! + while (bufIndex != len) { + if (Objects.isNull(currentTableName) || reader.isFinished()) { + if (tableNames.hasNext()) { + currentTableName = tableNames.next(); + rowIndex = 0; + columnCount = 0; + try { + totalColumns = reader.getColumnCount(currentTableName); + reader.read(currentTableName, () -> { + return bufIndex == len; + }); + } catch (SQLiteTableReaderException ex) { + Exceptions.printStackTrace(ex); + } + } else { + if (bufIndex == off) { + return -1; + } + return bufIndex; + } + } else { + try { + reader.read(currentTableName, () -> { + return bufIndex == len; + }); + } catch (SQLiteTableReaderException ex) { + Exceptions.printStackTrace(ex); + } } - currentTableReader = CharSource.wrap(tableResults).openStream(); - charactersRead = currentTableReader.read(cbuf, off, len); } - return charactersRead; + return bufIndex; } - /** - * Grab the next table name from the collection of all table names, once - * we no longer have a table to process, return null which will be - * understood to mean the end of parsing. - * - * @return Current table contents or null meaning there are not more - * tables to process - */ - private String getNextTable() { - if (tableIterator.hasNext()) { - return getTableAsString(tableIterator.next()); - } else { - return null; - } - } - - /** - * Close the underlying connection to the database. - * - * @throws IOException Not applicable, we can just catch the - * SQLException - */ @Override public void close() throws IOException { - try { - connection.close(); - } catch (SQLException ex) { - //Non-essential exception, user has no need for the connection - //object at this stage so closing details are not important - logger.log(Level.WARNING, "Could not close JDBC connection", ex); + reader.close(); + } + + /** + * Wrapper for an unfinished read during the previous chunker call. So, + * for example, the buffer passed to read() fills and we are left with + * only a partially read entity. One of these objects will encapsulate + * its state so that it can pick up where we left off on the next call + * to read(). + */ + private class UnfinishedState { + + private final String entity; + private Integer pointer; + + public UnfinishedState(String entity, Integer pointer) { + this.entity = entity; + this.pointer = pointer; } - } - } + public boolean isFinished() { + return entity.length() == pointer; + } - /** - * Formats input so that it reads as a table in the console or in a text - * viewer - */ - private class TableBuilder { + public int read(char[] buf, int off, int len) { + for (int i = off; i < len; i++) { + if (isFinished()) { + return i - off; + } - private final Integer DEFAULT_CAPACITY = 32000; - private final StringBuilder table = new StringBuilder(DEFAULT_CAPACITY); + buf[i] = entity.charAt(pointer++); + } - private static final String TAB = "\t"; - private static final String NEW_LINE = "\n"; - private static final String SPACE = " "; - - /** - * Add the section to the top left corner of the table. This is where - * the name of the table should go - * - * @param tableName Table name - */ - public void addTableName(String tableName) { - table.append(tableName) - .append(NEW_LINE) - .append(NEW_LINE); - } - - /** - * Adds a formatted header row to the underlying StringBuilder - * - * @param vals - */ - public void addHeader(Collection vals) { - addRow(vals); - } - - /** - * Adds a formatted row to the underlying StringBuilder - * - * @param vals - */ - public void addRow(Collection vals) { - table.append(TAB); - vals.forEach((val) -> { - table.append(val); - table.append(SPACE); - }); - table.append(NEW_LINE); - } - - public void addCell(String cell) { - table.append(cell); - } - - /** - * Returns a string version of the table, with all of the escape - * sequences necessary to print nicely in the console output. - * - * @return Formated table contents - */ - @Override - public String toString() { - return table.toString(); + return len - off; + } } } } From f78ac991970d4257575d1df3671185b30dcd4821 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Wed, 7 Nov 2018 15:04:04 -0500 Subject: [PATCH 074/129] 4305 remove changes for flag items global setting --- .../datamodel/EamDbUtil.java | 59 +++++-------------- 1 file changed, 14 insertions(+), 45 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDbUtil.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDbUtil.java index db6687222a..e30ed174eb 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDbUtil.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDbUtil.java @@ -39,7 +39,6 @@ public class EamDbUtil { private final static Logger LOGGER = Logger.getLogger(EamDbUtil.class.getName()); private static final String CENTRAL_REPO_NAME = "CentralRepository"; private static final String CENTRAL_REPO_USE_KEY = "db.useCentralRepo"; - private static final String CENTRAL_REPO_FLAG_ITEMS_KEY = "db.flagNotableItems"; private static final String DEFAULT_ORG_NAME = "Not Specified"; /** @@ -193,9 +192,9 @@ public class EamDbUtil { } /** - * Upgrade the current central reposity to the newest version. If the - * upgrade fails, the central repository will be disabled and the current - * settings will be cleared. + * Upgrade the current central reposity to the newest version. If the upgrade + * fails, the central repository will be disabled and the current settings + * will be cleared. * * @return true if the upgrade succeeds, false otherwise. */ @@ -203,7 +202,7 @@ public class EamDbUtil { if (!EamDb.isEnabled()) { return true; } - + CoordinationService.Lock lock = null; try { EamDb db = EamDb.getInstance(); @@ -219,23 +218,23 @@ public class EamDbUtil { LOGGER.log(Level.SEVERE, "Error updating central repository", ex); // Disable the central repo and clear the current settings. - try { + try{ if (null != EamDb.getInstance()) { EamDb.getInstance().shutdownConnections(); } - } catch (EamDbException ex2) { + } catch (EamDbException ex2){ LOGGER.log(Level.SEVERE, "Error shutting down central repo connection pool", ex); - } + } setUseCentralRepo(false); EamDbPlatformEnum.setSelectedPlatform(EamDbPlatformEnum.DISABLED.name()); EamDbPlatformEnum.saveSelectedPlatform(); - + return false; } finally { - if (lock != null) { - try { + if(lock != null){ + try{ lock.release(); - } catch (CoordinationServiceException ex) { + } catch (CoordinationServiceException ex){ LOGGER.log(Level.SEVERE, "Error releasing database lock", ex); } } @@ -256,7 +255,6 @@ public class EamDbUtil { * Check whether the given org is the default organization. * * @param org - * * @return true if it is the default org, false otherwise */ public static boolean isDefaultOrg(EamOrganization org) { @@ -267,7 +265,6 @@ public class EamDbUtil { * Add the default organization to the database * * @param conn - * * @return true if successful, false otherwise */ static boolean insertDefaultOrganization(Connection conn) { @@ -298,7 +295,7 @@ public class EamDbUtil { * If the Central Repos use has been enabled. * * @return true if the Central Repo may be configured, false if it should - * not be able to be + * not be able to be */ public static boolean useCentralRepo() { return Boolean.parseBoolean(ModuleSettings.getConfigSetting(CENTRAL_REPO_NAME, CENTRAL_REPO_USE_KEY)); @@ -309,40 +306,12 @@ public class EamDbUtil { * configured. * * @param centralRepoCheckBoxIsSelected - true if the central repo can be - * used + * used */ public static void setUseCentralRepo(boolean centralRepoCheckBoxIsSelected) { ModuleSettings.setConfigSetting(CENTRAL_REPO_NAME, CENTRAL_REPO_USE_KEY, Boolean.toString(centralRepoCheckBoxIsSelected)); } - /** - * If items previously tagged as notable should be flagged after adding them - * to the Central Repo. - * - * @return true if the previously tagged as notable items should be flagged, - * false if previously tagged as notable items should not be flagged - */ - public static boolean flagNotableItems() { - String flagNotableItems = ModuleSettings.getConfigSetting(CENTRAL_REPO_NAME, CENTRAL_REPO_FLAG_ITEMS_KEY); - if (flagNotableItems == null){ //getConfigSetting can return null if the setting did not exist - setFlagNotableItems(true); - return true; - } - return Boolean.parseBoolean(flagNotableItems); - } - - /** - * Saves the setting for whether items previously tagged as notable should - * be flagged after adding them to the Central Repo. - * - * @param flagNotableItems - true if the previously tagged as notable items - * should be flagged, false if previously tagged as - * notable items should not be flagged - */ - public static void setFlagNotableItems(boolean flagNotableItems) { - ModuleSettings.setConfigSetting(CENTRAL_REPO_NAME, CENTRAL_REPO_FLAG_ITEMS_KEY, Boolean.toString(flagNotableItems)); - } - /** * Use the current settings and the validation query to test the connection * to the database. @@ -396,7 +365,7 @@ public class EamDbUtil { * Close the prepared statement. * * @param preparedStatement The prepared statement to be closed. - * + * * @deprecated Use closeStatement() instead. * * @throws EamDbException From a694e7870185300294348f5847a3cce9c426bd67 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Wed, 7 Nov 2018 17:50:53 -0500 Subject: [PATCH 075/129] 4305 allow version to be updated if schema changes already present --- .../centralrepository/datamodel/AbstractSqlEamDb.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java index e239f0b987..e1e87024f0 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java @@ -2960,11 +2960,10 @@ abstract class AbstractSqlEamDb implements EamDb { resultSet.getString("poc_phone")); } - CorrelationCase eamCase = new CorrelationCase(resultSet.getInt("case_id"), resultSet.getString("case_uid"), eamOrg, resultSet.getString("case_name"), - resultSet.getString("creation_date"), resultSet.getString("case_number"), resultSet.getString("examiner_name"), + CorrelationCase eamCase = new CorrelationCase(resultSet.getInt("case_id"), resultSet.getString("case_uid"), eamOrg, resultSet.getString("case_name"), + resultSet.getString("creation_date"), resultSet.getString("case_number"), resultSet.getString("examiner_name"), resultSet.getString("examiner_email"), resultSet.getString("examiner_phone"), resultSet.getString("notes")); - return eamCase; } @@ -3119,7 +3118,7 @@ abstract class AbstractSqlEamDb implements EamDb { logger.log(Level.INFO, "Central Repository is of newer version than software creates"); return; } - + // Update from 1.0 to 1.1 if (dbSchemaVersion.compareTo(new CaseDbSchemaVersionNumber(1, 1)) < 0) { statement.execute("ALTER TABLE reference_sets ADD COLUMN known_status INTEGER;"); //NON-NLS @@ -3140,9 +3139,11 @@ abstract class AbstractSqlEamDb implements EamDb { final String addDataSourceIdIndexTemplate; final String addValueIndexTemplate; final String addKnownStatusIndexTemplate; + final String addAttributeSql; //get the data base specific code for creating a new _instance table switch (selectedPlatform) { case POSTGRESQL: + addAttributeSql = "INSERT INTO correlation_types(id, display_name, db_table_name, supported, enabled) VALUES (?, ?, ?, ?, ?) " + getConflictClause(); addSsidTableTemplate = PostgresEamDbSettings.getCreateArtifactInstancesTableTemplate(); addCaseIdIndexTemplate = PostgresEamDbSettings.getAddCaseIdIndexTemplate(); addDataSourceIdIndexTemplate = PostgresEamDbSettings.getAddDataSourceIdIndexTemplate(); @@ -3150,6 +3151,7 @@ abstract class AbstractSqlEamDb implements EamDb { addKnownStatusIndexTemplate = PostgresEamDbSettings.getAddKnownStatusIndexTemplate(); break; case SQLITE: + addAttributeSql = "INSERT OR IGNORE INTO correlation_types(id, display_name, db_table_name, supported, enabled) VALUES (?, ?, ?, ?, ?)"; addSsidTableTemplate = SqliteEamDbSettings.getCreateArtifactInstancesTableTemplate(); addCaseIdIndexTemplate = SqliteEamDbSettings.getAddCaseIdIndexTemplate(); addDataSourceIdIndexTemplate = SqliteEamDbSettings.getAddDataSourceIdIndexTemplate(); @@ -3162,7 +3164,6 @@ abstract class AbstractSqlEamDb implements EamDb { final String wirelessNetworsDbTableName = "wireless_networks"; final String wirelessNetworksTableInstanceName = wirelessNetworsDbTableName + "_instances"; //add the wireless_networks attribute to the correlation_types table - final String addAttributeSql = "INSERT INTO correlation_types(id, display_name, db_table_name, supported, enabled) VALUES (?, ?, ?, ?, ?)"; preparedStatement = conn.prepareStatement(addAttributeSql); preparedStatement.setInt(1, CorrelationAttributeInstance.SSID_TYPE_ID); preparedStatement.setString(2, Bundle.CorrelationType_SSID_displayName()); From 422e6a7250376b775a2cec79ad2bef27e2d9574f Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Thu, 8 Nov 2018 10:04:03 -0500 Subject: [PATCH 076/129] Commented and cleaned up the code for the text extractor --- .../autopsy/coreutils/SQLiteTableReader.java | 4 +- .../keywordsearch/SqliteTextExtractor.java | 187 +++++++++++++----- .../autopsy/keywordsearch/SqliteUtil.java | 130 ------------ 3 files changed, 137 insertions(+), 184 deletions(-) delete mode 100755 KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SqliteUtil.java diff --git a/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReader.java b/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReader.java index 6a5540d222..7374c2d7f9 100755 --- a/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReader.java +++ b/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReader.java @@ -191,7 +191,7 @@ public class SQLiteTableReader implements AutoCloseable { private boolean hasOpened; private String prevTableName; - private final BooleanSupplier alwaysFalseCondition = () -> {return false;}; + private final BooleanSupplier alwaysFalseCondition = () -> false; /** * Initialize a new table stream given the parameters passed in from the @@ -217,7 +217,7 @@ public class SQLiteTableReader implements AutoCloseable { //No-op lambda, keep from NPE or having to check during iteration //if action == null. - return (NO_OP) -> {}; + return NO_OP -> {}; } /** diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SqliteTextExtractor.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SqliteTextExtractor.java index bbf55553dc..419805590b 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SqliteTextExtractor.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SqliteTextExtractor.java @@ -5,8 +5,8 @@ import java.io.IOException; import java.io.Reader; import java.util.Iterator; import java.util.Objects; +import java.util.function.Consumer; import java.util.logging.Level; -import org.openide.util.Exceptions; import org.sleuthkit.autopsy.coreutils.SQLiteTableReaderException; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.SQLiteTableReader; @@ -25,7 +25,6 @@ class SqliteTextExtractor extends ContentTextExtractor { private static final String SQLITE_MIMETYPE = "application/x-sqlite3"; private static final Logger logger = Logger.getLogger(SqliteTextExtractor.class.getName()); - private static final CharSequence EMPTY_CHARACTER_SEQUENCE = ""; @Override boolean isContentTypeSpecific() { @@ -56,7 +55,7 @@ class SqliteTextExtractor extends ContentTextExtractor { } /** - * Returns an input stream that will read from a sqlite database. + * Returns a stream that will read from a sqlite database. * * @param source Content file * @@ -70,7 +69,7 @@ class SqliteTextExtractor extends ContentTextExtractor { //Firewall for any content that is not an AbstractFile if (!AbstractFile.class.isInstance(source)) { try { - return CharSource.wrap(EMPTY_CHARACTER_SEQUENCE).openStream(); + return CharSource.wrap("").openStream(); } catch (IOException ex) { throw new TextExtractorException("", ex); } @@ -79,65 +78,140 @@ class SqliteTextExtractor extends ContentTextExtractor { return new SQLiteStreamReader((AbstractFile) source); } + /** + * Produces a continuous stream of characters from a database file. To + * achieve this, all table names are queues up and a SQLiteTableReader is + * used to do the actual queries and table iteration. + */ public class SQLiteStreamReader extends Reader { private final SQLiteTableReader reader; + private final AbstractFile file; + private Iterator tableNames; private String currentTableName; private char[] buf; - private UnfinishedState unfinishedRead; - private int rowIndex; - private int columnCount; + private ExcessBytes leftOvers; private int totalColumns; private int bufIndex; + /** + * Creates a new reader for the sqlite file. This table reader class + * will iterate through a table row by row and pass the values to + * different functions based on data type. Here we define what to do on + * the column names and we define what to do for all data types. + * + * @param file Sqlite file + */ public SQLiteStreamReader(AbstractFile file) { + this.file = file; reader = new SQLiteTableReader.Builder(file) - .onColumnNames((columnName) -> { - if(columnCount == 0) { - fillBuffer("\n"+currentTableName + "\n\n\t"); - } - columnCount++; - - fillBuffer(columnName + ((columnCount == totalColumns) ? "\n" :" ")); - }) - .forAll((Object o) -> { - rowIndex++; - //Ignore blobs - String objectStr = (o instanceof byte[]) ? "" : Objects.toString(o, ""); - - if(rowIndex > 1 && rowIndex < totalColumns) { - objectStr += " "; - } if(rowIndex == 1){ - objectStr = "\t" + objectStr + " "; - } if(rowIndex == totalColumns) { - objectStr += "\n"; - } - - fillBuffer(objectStr); - rowIndex = rowIndex % totalColumns; - }).build(); + .onColumnNames(getColumnNameStrategy()) + .forAll(getForAllStrategy()).build(); } + /** + * On every item in the database we want to do the following series of + * steps: 1) Get it's string representation (ignore blobs with empty + * string). 2) Format it based on its positioning in the row. 3) Write + * it to buffer + * + * rowIndex is purely for keeping track of where the object is in the + * table, hence the bounds checking with the mod function. + * + * @return Our consumer class defined to do the steps above. + */ + private Consumer getForAllStrategy() { + return new Consumer() { + private int rowIndex = 0; + + @Override + public void accept(Object t) { + rowIndex++; + //Ignore blobs + String objectStr = (t instanceof byte[]) ? "" : Objects.toString(t, ""); + + if (rowIndex > 1 && rowIndex < totalColumns) { + objectStr += " "; + } + if (rowIndex == 1) { + objectStr = "\t" + objectStr + " "; + } + if (rowIndex == totalColumns) { + objectStr += "\n"; + } + + fillBuffer(objectStr); + rowIndex = rowIndex % totalColumns; + } + }; + } + + /** + * On every column name in the header do the following series of steps: + * 1) Write the tableName before the header. 2) Format the column name + * based on row positioning 3) Reset the count if we are at the end, + * that way if we want to read multiple tables we can do so without + * having to build new consumers. + * + * columnIndex is purely for keeping track of where the column name is + * in the table, hence the bounds checking with the mod function. + * + * @return Our consumer class defined to do the steps above. + */ + private Consumer getColumnNameStrategy() { + return new Consumer() { + private int columnIndex = 0; + + @Override + public void accept(String columnName) { + if (columnIndex == 0) { + fillBuffer("\n" + currentTableName + "\n\n\t"); + } + columnIndex++; + + fillBuffer(columnName + ((columnIndex == totalColumns) ? "\n" : " ")); + + //Reset the columnCount to 0 for next table read + columnIndex = columnIndex % totalColumns; + } + }; + } + + /** + * This functions writes the string representation of a database value + * into the read buffer. If the buffer becomes full, we save the extra + * characters and hold on to them until the next call to read(). + * + * @param val Formatted database value string + */ private void fillBuffer(String val) { for (int i = 0; i < val.length(); i++) { if (bufIndex != buf.length) { buf[bufIndex++] = val.charAt(i); } else { - unfinishedRead = new UnfinishedState(val, i); + leftOvers = new ExcessBytes(val, i); break; } } } + /** + * Reads database values into the buffer. This function is responsible for + * getting the next table in the queue, initiating calls to the SQLiteTableReader, + * and filling in any excess bytes that are lingering from the previous call. + * + * @throws IOException + */ @Override public int read(char[] cbuf, int off, int len) throws IOException { buf = cbuf; bufIndex = off; + //Lazily wait to get table names until first call to read. if (Objects.isNull(tableNames)) { try { tableNames = reader.getTableNames().iterator(); @@ -147,24 +221,25 @@ class SqliteTextExtractor extends ContentTextExtractor { } } - if (Objects.nonNull(unfinishedRead) && !unfinishedRead.isFinished()) { - bufIndex += unfinishedRead.read(cbuf, off, len); + //If there are excess bytes from last read, then copy thoses in. + if (Objects.nonNull(leftOvers) && !leftOvers.isFinished()) { + bufIndex += leftOvers.read(cbuf, off, len); } - //while buffer is not full! + //Keep grabbing table names from the queue and reading them until + //our buffer is full. while (bufIndex != len) { if (Objects.isNull(currentTableName) || reader.isFinished()) { if (tableNames.hasNext()) { currentTableName = tableNames.next(); - rowIndex = 0; - columnCount = 0; try { totalColumns = reader.getColumnCount(currentTableName); - reader.read(currentTableName, () -> { - return bufIndex == len; - }); + reader.read(currentTableName, () -> bufIndex == len); } catch (SQLiteTableReaderException ex) { - Exceptions.printStackTrace(ex); + logger.log(Level.WARNING, String.format( + "Error attempting to read file table: [%s]" //NON-NLS + + " for file: [%s] (id=%d).", currentTableName, //NON-NLS + file.getName(), file.getId()), ex.getMessage()); } } else { if (bufIndex == off) { @@ -174,11 +249,12 @@ class SqliteTextExtractor extends ContentTextExtractor { } } else { try { - reader.read(currentTableName, () -> { - return bufIndex == len; - }); + reader.read(currentTableName, () -> bufIndex == len); } catch (SQLiteTableReaderException ex) { - Exceptions.printStackTrace(ex); + logger.log(Level.WARNING, String.format( + "Error attempting to read file table: [%s]" //NON-NLS + + " for file: [%s] (id=%d).", currentTableName, //NON-NLS + file.getName(), file.getId()), ex.getMessage()); } } } @@ -192,18 +268,15 @@ class SqliteTextExtractor extends ContentTextExtractor { } /** - * Wrapper for an unfinished read during the previous chunker call. So, - * for example, the buffer passed to read() fills and we are left with - * only a partially read entity. One of these objects will encapsulate - * its state so that it can pick up where we left off on the next call - * to read(). + * Wrapper that holds the excess bytes that were left over from the previous + * call to read(). */ - private class UnfinishedState { + private class ExcessBytes { private final String entity; private Integer pointer; - public UnfinishedState(String entity, Integer pointer) { + public ExcessBytes(String entity, Integer pointer) { this.entity = entity; this.pointer = pointer; } @@ -212,6 +285,16 @@ class SqliteTextExtractor extends ContentTextExtractor { return entity.length() == pointer; } + /** + * Copies the excess bytes this instance is holding onto into the + * buffer. + * + * @param buf buffer to write into + * @param off index in buffer to start the write + * @param len length of the write + * + * @return number of characters read into the buffer + */ public int read(char[] buf, int off, int len) { for (int i = off; i < len; i++) { if (isFinished()) { diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SqliteUtil.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SqliteUtil.java deleted file mode 100755 index 08eefe7232..0000000000 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SqliteUtil.java +++ /dev/null @@ -1,130 +0,0 @@ -/* - * Autopsy Forensic Browser - * - * Copyright 2018-2018 Basis Technology Corp. - * Contact: carrier sleuthkit org - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.sleuthkit.autopsy.keywordsearch; - -import java.io.File; -import java.io.IOException; -import java.util.List; -import org.sleuthkit.autopsy.casemodule.Case; -import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; -import org.sleuthkit.autopsy.casemodule.services.FileManager; -import org.sleuthkit.autopsy.casemodule.services.Services; -import org.sleuthkit.autopsy.datamodel.ContentUtils; -import org.sleuthkit.datamodel.AbstractFile; -import org.sleuthkit.datamodel.SleuthkitCase; -import org.sleuthkit.datamodel.TskCoreException; - -/** - * Sqlite utility class. Find and copy metafiles, write sqlite abstract files to - * temp directory, and generate unique temp directory paths. - */ -final class SqliteUtil { - - private SqliteUtil() { - - } - - /** - * Overloaded implementation of - * {@link #findAndCopySQLiteMetaFile(AbstractFile, String) findAndCopySQLiteMetaFile} - * , automatically tries to copy -wal and -shm files without needing to know - * their existence. - * - * @param sqliteFile file which has -wal and -shm meta files - * - * @throws NoCurrentCaseException Case has been closed. - * @throws TskCoreException fileManager cannot find AbstractFile - * files. - * @throws IOException Issue during writing to file. - */ - public static void findAndCopySQLiteMetaFile(AbstractFile sqliteFile) - throws NoCurrentCaseException, TskCoreException, IOException { - - findAndCopySQLiteMetaFile(sqliteFile, sqliteFile.getName() + "-wal"); - findAndCopySQLiteMetaFile(sqliteFile, sqliteFile.getName() + "-shm"); - } - - /** - * Searches for a meta file associated with the give SQLite database. If - * found, it copies this file into the temp directory of the current case. - * - * @param sqliteFile file being processed - * @param metaFileName name of meta file to look for - * - * @throws NoCurrentCaseException Case has been closed. - * @throws TskCoreException fileManager cannot find AbstractFile - * files. - * @throws IOException Issue during writing to file. - */ - public static void findAndCopySQLiteMetaFile(AbstractFile sqliteFile, - String metaFileName) throws NoCurrentCaseException, TskCoreException, IOException { - - Case openCase = Case.getCurrentCaseThrows(); - SleuthkitCase sleuthkitCase = openCase.getSleuthkitCase(); - Services services = new Services(sleuthkitCase); - FileManager fileManager = services.getFileManager(); - - List metaFiles = fileManager.findFiles( - sqliteFile.getDataSource(), metaFileName, - sqliteFile.getParent().getName()); - - if (metaFiles != null) { - for (AbstractFile metaFile : metaFiles) { - writeAbstractFileToLocalDisk(metaFile); - } - } - } - - /** - * Copies the file contents into a unique path in the current case temp - * directory. - * - * @param file AbstractFile from the data source - * - * @return The path of the file on disk - * - * @throws IOException Exception writing file contents - * @throws NoCurrentCaseException Current case closed during file copying - */ - public static String writeAbstractFileToLocalDisk(AbstractFile file) - throws IOException, NoCurrentCaseException { - - String localDiskPath = getUniqueTempDirectoryPath(file); - File localDatabaseFile = new File(localDiskPath); - if (!localDatabaseFile.exists()) { - ContentUtils.writeToFile(file, localDatabaseFile); - } - return localDiskPath; - } - - /** - * Generates a unique local disk path that resides in the temp directory of - * the current case. - * - * @param file The database abstract file - * - * @return Unique local disk path living in the temp directory of the case - * - * @throws org.sleuthkit.autopsy.casemodule.NoCurrentCaseException - */ - public static String getUniqueTempDirectoryPath(AbstractFile file) throws NoCurrentCaseException { - return Case.getCurrentCaseThrows().getTempDirectory() - + File.separator + file.getId() + file.getName(); - } -} From b13736b67cf0f47cfdd725e4cb606ccf4afc98ee Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Thu, 8 Nov 2018 10:07:57 -0500 Subject: [PATCH 077/129] Added license back in --- .../keywordsearch/SqliteTextExtractor.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SqliteTextExtractor.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SqliteTextExtractor.java index 419805590b..790db9c005 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SqliteTextExtractor.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SqliteTextExtractor.java @@ -1,3 +1,21 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2018-2018 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.sleuthkit.autopsy.keywordsearch; import com.google.common.io.CharSource; From b907d6db0eaf09b7fbbd8de02ee8855495e67682 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Thu, 8 Nov 2018 10:25:32 -0500 Subject: [PATCH 078/129] Change the build type back to DEVELOPMENT --- nbproject/project.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nbproject/project.properties b/nbproject/project.properties index 8f3f577177..96658e548c 100644 --- a/nbproject/project.properties +++ b/nbproject/project.properties @@ -6,8 +6,8 @@ app.name=${branding.token} ### if left unset, version will default to today's date app.version=4.9.1 ### build.type must be one of: DEVELOPMENT, RELEASE -build.type=RELEASE -#build.type=DEVELOPMENT +#build.type=RELEASE +build.type=DEVELOPMENT project.org.netbeans.progress=org-netbeans-api-progress project.org.sleuthkit.autopsy.experimental=Experimental From ac4565f42fbfbfd9756758012e0ddc8c9c28dfb7 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Thu, 8 Nov 2018 11:27:42 -0500 Subject: [PATCH 079/129] Refactored some code to make it more readable and added comments --- .../autopsy/contentviewers/SQLiteViewer.java | 230 ++++++++++++------ 1 file changed, 160 insertions(+), 70 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/contentviewers/SQLiteViewer.java b/Core/src/org/sleuthkit/autopsy/contentviewers/SQLiteViewer.java index 541b057d9e..6de21f9ec0 100755 --- a/Core/src/org/sleuthkit/autopsy/contentviewers/SQLiteViewer.java +++ b/Core/src/org/sleuthkit/autopsy/contentviewers/SQLiteViewer.java @@ -32,6 +32,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.function.Consumer; import java.util.logging.Level; import javax.swing.JComboBox; import javax.swing.JFileChooser; @@ -62,12 +63,14 @@ class SQLiteViewer extends javax.swing.JPanel implements FileTypeViewer { private SQLiteTableReader viewReader; - private Map rowMap = new LinkedHashMap<>(); - private List> chunk = new ArrayList<>(); + private Map row = new LinkedHashMap<>(); + private List> pageOfTableRows = new ArrayList<>(); + private List currentTableHeader = new ArrayList<>(); + private String prevTableName; private int numRows; // num of rows in the selected table private int currPage = 0; // curr page of rows being displayed - + /** * Constructs a file content viewer for SQLite database files. */ @@ -340,9 +343,11 @@ class SQLiteViewer extends javax.swing.JPanel implements FileTypeViewer { tablesDropdownList.setEnabled(true); tablesDropdownList.removeAllItems(); numEntriesField.setText(""); + viewReader.close(); - rowMap = new LinkedHashMap<>(); - chunk = new ArrayList<>(); + row = new LinkedHashMap<>(); + pageOfTableRows = new ArrayList<>(); + currentTableHeader = new ArrayList<>(); viewReader = null; sqliteDbFile = null; } @@ -408,19 +413,20 @@ class SQLiteViewer extends javax.swing.JPanel implements FileTypeViewer { } } - private String prevTableName; @NbBundle.Messages({"# {0} - tableName", "SQLiteViewer.readTable.errorText=Error getting rows for table: {0}"}) private void readTable(String tableName, int startRow, int numRowsToRead) { try { - if(!tableName.equals(prevTableName)) { + //If the table name has changed, then clear our table header. SQLiteTableReader + //will also detect the table name has changed and begin reading it as if it + //were a brand new table. + if (!tableName.equals(prevTableName)) { prevTableName = tableName; - header = new ArrayList<>(); - rowIndex = 0; + currentTableHeader = new ArrayList<>(); } viewReader.read(tableName, numRowsToRead, startRow - 1); - selectedTableView.setupTable(chunk); - chunk = new ArrayList<>(); + selectedTableView.setupTable(pageOfTableRows); + pageOfTableRows = new ArrayList<>(); } catch (SQLiteTableReaderException ex) { logger.log(Level.WARNING, String.format("Failed to read table %s from DB file '%s' " //NON-NLS + "(objId=%d) starting at row [%d] and limit [%d]", //NON-NLS @@ -429,86 +435,170 @@ class SQLiteViewer extends javax.swing.JPanel implements FileTypeViewer { } } - List header; - private int rowIndex; + /** + * Creates a new SQLiteTableReader. This class will iterate through the + * table row by row and pass each value to the correct function based on its + * data type. For our use, we want to define an action when encountering + * column names and an action for all other data types. + */ private void initReader() { viewReader = new SQLiteTableReader.Builder(sqliteDbFile) .onColumnNames((columnName) -> { - header.add(columnName); + currentTableHeader.add(columnName); }) - .forAll((Object o) -> { - rowIndex++; - String objectStr = (o instanceof byte[]) ? "BLOB Data not shown" : Objects.toString(o, ""); - - rowMap.put(header.get(rowIndex - 1), objectStr); - //If this result is at the end of a row, then add it to the - //chunk! - if (rowIndex == header.size()) { - chunk.add(rowMap); - rowMap = new LinkedHashMap<>(); - } - rowIndex %= header.size(); - }).build(); + .forAll(getForAllStrategy()).build(); } - private int csvRowIndex; - private int columnCount; - private boolean afterHeader; - + /** + * For every database value we encounter on our read of the table do the + * following: 1) Get the string representation of the value 2) Collect the + * values until we have a full database row. 3) If we have the full row, + * write it to the UI. + * + * rowIndex is purely for indicating if we have read the full row. + * + * @return Consumer that will perform the actions above. When the + * SQLiteTableReader is reading, values will be passed to this + * consumer. + */ + private Consumer getForAllStrategy() { + return new Consumer() { + private int rowIndex = 0; + + @Override + public void accept(Object t) { + rowIndex++; + String objectStr = (t instanceof byte[]) ? "BLOB Data not shown" + : Objects.toString(t, ""); + + row.put(currentTableHeader.get(rowIndex - 1), objectStr); + + if (rowIndex == currentTableHeader.size()) { + pageOfTableRows.add(row); + row = new LinkedHashMap<>(); + } + rowIndex %= currentTableHeader.size(); + } + + }; + } + + private int totalColumnCount; + @NbBundle.Messages({"SQLiteViewer.exportTableToCsv.write.errText=Failed to export table content to csv file.", "SQLiteViewer.exportTableToCsv.FileName=File name: ", "SQLiteViewer.exportTableToCsv.TableName=Table name: " }) private void exportTableToCsv(File file) { - csvRowIndex = 0; - columnCount = 0; - afterHeader = true; File csvFile = new File(file.toString() + ".csv"); String tableName = (String) this.tablesDropdownList.getSelectedItem(); try (FileOutputStream out = new FileOutputStream(csvFile, false)) { try (SQLiteTableReader sqliteStream = new SQLiteTableReader.Builder(sqliteDbFile) - .onColumnNames((columnName) -> { - columnCount++; - try { - if(columnCount == 1) { - columnName = "\"" + columnName + "\""; - } else { - columnName = ",\"" + columnName + "\""; - } - out.write(columnName.getBytes()); - } catch (IOException ex) { - - } - }).forAll((Object o) -> { - csvRowIndex++; - String objectStr = (o instanceof byte[]) ? "BLOB Data not shown" : Objects.toString(o, ""); - objectStr = "\"" + objectStr + "\""; - - if (csvRowIndex > 1) { - objectStr = "," + objectStr; - } if(csvRowIndex == columnCount) { - objectStr += "\n"; - } - - if(afterHeader) { - objectStr = "\n" + objectStr; - afterHeader = false; - } - - try { - out.write(objectStr.getBytes()); - } catch (IOException ex) { - - } - csvRowIndex = csvRowIndex % columnCount; - }).build()) { + .onColumnNames(getColumnNameCSVStrategy(out)) + .forAll(getForAllCSVStrategy(out)).build()) { + totalColumnCount = sqliteStream.getColumnCount(tableName); sqliteStream.read(tableName); } - } catch (IOException | SQLiteTableReaderException ex) { + } catch (IOException | SQLiteTableReaderException | RuntimeException ex) { logger.log(Level.WARNING, String.format("Failed to export table [%s]" + " to CSV in sqlite file '%s' (objId=%d)", tableName, sqliteDbFile.getName(), sqliteDbFile.getId()), ex.getMessage()); //NON-NLS MessageNotifyUtil.Message.error(Bundle.SQLiteViewer_exportTableToCsv_write_errText()); } } + + /** + * For every column name we encounter on our read of the table do the + * following: 1) Format the name so that it is comma seperated 2) Write the + * value to the output stream. + * + * columnIndex is purely for keeping track of where the column name is in + * the table so the value can be correctly formatted. + * + * @param out Output stream that this database table is being written to. + * + * @return Consumer that will perform the actions above. When the + * SQLiteTableReader is reading, values will be passed to this + * consumer. + */ + private Consumer getColumnNameCSVStrategy(FileOutputStream out) { + return new Consumer() { + private int columnIndex = 0; + + @Override + public void accept(String columnName) { + columnIndex++; + + if (columnIndex == 1) { + columnName = "\"" + columnName + "\""; + } else { + columnName = ",\"" + columnName + "\""; + } + + if (columnIndex == totalColumnCount) { + columnName += "\n"; + } + + try { + out.write(columnName.getBytes()); + } catch (IOException ex) { + /* + * If we can no longer write to the output stream, toss a + * runtime exception to get out of iteration. We explicitly + * catch this in exportTableToCsv() above. + */ + throw new RuntimeException(ex); + } + } + }; + } + + /** + * For every database value we encounter on our read of the table do the + * following: 1) Get the string representation of the value 2) Format it so + * that it adheres to the CSV format. 3) Write it to the output file. + * + * rowIndex is purely for keeping track of positioning of the database value + * in the row, so that it can be properly formatted. + * + * @param out Output file + * + * @return Consumer that will perform the actions above. When the + * SQLiteTableReader is reading, values will be passed to this + * consumer. + */ + private Consumer getForAllCSVStrategy(FileOutputStream out) { + return new Consumer() { + private int rowIndex = 0; + + @Override + public void accept(Object tableValue) { + rowIndex++; + //Substitute string representation of blob with placeholder text. + //Automatically wrap the value in quotes in case it contains commas. + String objectStr = (tableValue instanceof byte[]) + ? "BLOB Data not shown" : Objects.toString(tableValue, ""); + objectStr = "\"" + objectStr + "\""; + + if (rowIndex > 1) { + objectStr = "," + objectStr; + } + if (rowIndex == totalColumnCount) { + objectStr += "\n"; + } + + try { + out.write(objectStr.getBytes()); + } catch (IOException ex) { + /* + * If we can no longer write to the output stream, toss a + * runtime exception to get out of iteration. We explicitly + * catch this in exportTableToCsv() above. + */ + throw new RuntimeException(ex); + } + rowIndex = rowIndex % totalColumnCount; + } + }; + } } From 03e851a341dd6762e5b533890e51f99ea21d41da Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Thu, 8 Nov 2018 11:31:58 -0500 Subject: [PATCH 080/129] Deleted SqliteUtil and fixed a log message --- .../autopsy/contentviewers/SQLiteViewer.java | 7 +- .../autopsy/contentviewers/SqliteUtil.java | 130 ------------------ 2 files changed, 5 insertions(+), 132 deletions(-) delete mode 100755 Core/src/org/sleuthkit/autopsy/contentviewers/SqliteUtil.java diff --git a/Core/src/org/sleuthkit/autopsy/contentviewers/SQLiteViewer.java b/Core/src/org/sleuthkit/autopsy/contentviewers/SQLiteViewer.java index 6de21f9ec0..e96a538580 100755 --- a/Core/src/org/sleuthkit/autopsy/contentviewers/SQLiteViewer.java +++ b/Core/src/org/sleuthkit/autopsy/contentviewers/SQLiteViewer.java @@ -408,7 +408,9 @@ class SQLiteViewer extends javax.swing.JPanel implements FileTypeViewer { selectedTableView.setupTable(Collections.emptyList()); } } catch (SQLiteTableReaderException ex) { - logger.log(Level.WARNING, String.format("Failed to load table %s from DB file '%s' (objId=%d)", tableName, sqliteDbFile.getName(), sqliteDbFile.getId()), ex); //NON-NLS + logger.log(Level.WARNING, String.format("Failed to load table %s " //NON-NLS + + "from DB file '%s' (objId=%d)", tableName, sqliteDbFile.getName(), //NON-NLS + sqliteDbFile.getId()), ex.getMessage()); MessageNotifyUtil.Message.error(Bundle.SQLiteViewer_selectTable_errorText(tableName)); } } @@ -430,7 +432,8 @@ class SQLiteViewer extends javax.swing.JPanel implements FileTypeViewer { } catch (SQLiteTableReaderException ex) { logger.log(Level.WARNING, String.format("Failed to read table %s from DB file '%s' " //NON-NLS + "(objId=%d) starting at row [%d] and limit [%d]", //NON-NLS - tableName, sqliteDbFile.getName(), sqliteDbFile.getId(), startRow - 1, numRowsToRead), ex.getMessage()); + tableName, sqliteDbFile.getName(), sqliteDbFile.getId(), + startRow - 1, numRowsToRead), ex.getMessage()); MessageNotifyUtil.Message.error(Bundle.SQLiteViewer_readTable_errorText(tableName)); } } diff --git a/Core/src/org/sleuthkit/autopsy/contentviewers/SqliteUtil.java b/Core/src/org/sleuthkit/autopsy/contentviewers/SqliteUtil.java deleted file mode 100755 index 4fc220cf0d..0000000000 --- a/Core/src/org/sleuthkit/autopsy/contentviewers/SqliteUtil.java +++ /dev/null @@ -1,130 +0,0 @@ -/* - * Autopsy Forensic Browser - * - * Copyright 2018-2018 Basis Technology Corp. - * Contact: carrier sleuthkit org - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.sleuthkit.autopsy.contentviewers; - -import java.io.File; -import java.io.IOException; -import java.util.List; -import org.sleuthkit.autopsy.casemodule.Case; -import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; -import org.sleuthkit.autopsy.casemodule.services.FileManager; -import org.sleuthkit.autopsy.casemodule.services.Services; -import org.sleuthkit.autopsy.datamodel.ContentUtils; -import org.sleuthkit.datamodel.AbstractFile; -import org.sleuthkit.datamodel.SleuthkitCase; -import org.sleuthkit.datamodel.TskCoreException; - -/** - * Sqlite utility class. Find and copy metafiles, write sqlite abstract files to - * temp directory, and generate unique temp directory paths. - */ -final class SqliteUtil { - - private SqliteUtil() { - - } - - /** - * Overloaded implementation of - * {@link #findAndCopySQLiteMetaFile(AbstractFile, String) findAndCopySQLiteMetaFile} - * , automatically tries to copy -wal and -shm files without needing to know - * their existence. - * - * @param sqliteFile file which has -wal and -shm meta files - * - * @throws NoCurrentCaseException Case has been closed. - * @throws TskCoreException fileManager cannot find AbstractFile - * files. - * @throws IOException Issue during writing to file. - */ - public static void findAndCopySQLiteMetaFile(AbstractFile sqliteFile) - throws NoCurrentCaseException, TskCoreException, IOException { - - findAndCopySQLiteMetaFile(sqliteFile, sqliteFile.getName() + "-wal"); - findAndCopySQLiteMetaFile(sqliteFile, sqliteFile.getName() + "-shm"); - } - - /** - * Searches for a meta file associated with the give SQLite database. If - * found, it copies this file into the temp directory of the current case. - * - * @param sqliteFile file being processed - * @param metaFileName name of meta file to look for - * - * @throws NoCurrentCaseException Case has been closed. - * @throws TskCoreException fileManager cannot find AbstractFile - * files. - * @throws IOException Issue during writing to file. - */ - public static void findAndCopySQLiteMetaFile(AbstractFile sqliteFile, - String metaFileName) throws NoCurrentCaseException, TskCoreException, IOException { - - Case openCase = Case.getCurrentCaseThrows(); - SleuthkitCase sleuthkitCase = openCase.getSleuthkitCase(); - Services services = new Services(sleuthkitCase); - FileManager fileManager = services.getFileManager(); - - List metaFiles = fileManager.findFiles( - sqliteFile.getDataSource(), metaFileName, - sqliteFile.getParent().getName()); - - if (metaFiles != null) { - for (AbstractFile metaFile : metaFiles) { - writeAbstractFileToLocalDisk(metaFile); - } - } - } - - /** - * Copies the file contents into a unique path in the current case temp - * directory. - * - * @param file AbstractFile from the data source - * - * @return The path of the file on disk - * - * @throws IOException Exception writing file contents - * @throws NoCurrentCaseException Current case closed during file copying - */ - public static String writeAbstractFileToLocalDisk(AbstractFile file) - throws IOException, NoCurrentCaseException { - - String localDiskPath = getUniqueTempDirectoryPath(file); - File localDatabaseFile = new File(localDiskPath); - if (!localDatabaseFile.exists()) { - ContentUtils.writeToFile(file, localDatabaseFile); - } - return localDiskPath; - } - - /** - * Generates a unique local disk path that resides in the temp directory of - * the current case. - * - * @param file The database abstract file - * - * @return Unique local disk path living in the temp directory of the case - * - * @throws org.sleuthkit.autopsy.casemodule.NoCurrentCaseException - */ - public static String getUniqueTempDirectoryPath(AbstractFile file) throws NoCurrentCaseException { - return Case.getCurrentCaseThrows().getTempDirectory() - + File.separator + file.getId() + file.getName(); - } -} From b4cf1a0470bdd8c3fc84e56d8d63af294844366e Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Thu, 8 Nov 2018 11:36:07 -0500 Subject: [PATCH 081/129] Added a few more comments --- .../org/sleuthkit/autopsy/contentviewers/SQLiteViewer.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/contentviewers/SQLiteViewer.java b/Core/src/org/sleuthkit/autopsy/contentviewers/SQLiteViewer.java index e96a538580..e41fbd1eef 100755 --- a/Core/src/org/sleuthkit/autopsy/contentviewers/SQLiteViewer.java +++ b/Core/src/org/sleuthkit/autopsy/contentviewers/SQLiteViewer.java @@ -432,7 +432,7 @@ class SQLiteViewer extends javax.swing.JPanel implements FileTypeViewer { } catch (SQLiteTableReaderException ex) { logger.log(Level.WARNING, String.format("Failed to read table %s from DB file '%s' " //NON-NLS + "(objId=%d) starting at row [%d] and limit [%d]", //NON-NLS - tableName, sqliteDbFile.getName(), sqliteDbFile.getId(), + tableName, sqliteDbFile.getName(), sqliteDbFile.getId(), startRow - 1, numRowsToRead), ex.getMessage()); MessageNotifyUtil.Message.error(Bundle.SQLiteViewer_readTable_errorText(tableName)); } @@ -476,6 +476,8 @@ class SQLiteViewer extends javax.swing.JPanel implements FileTypeViewer { row.put(currentTableHeader.get(rowIndex - 1), objectStr); + //If we have built up a full database row, then add it to our page + //of rows to be displayed in the UI. if (rowIndex == currentTableHeader.size()) { pageOfTableRows.add(row); row = new LinkedHashMap<>(); @@ -532,12 +534,12 @@ class SQLiteViewer extends javax.swing.JPanel implements FileTypeViewer { public void accept(String columnName) { columnIndex++; + //Format the value to adhere to the format of a CSV file if (columnIndex == 1) { columnName = "\"" + columnName + "\""; } else { columnName = ",\"" + columnName + "\""; } - if (columnIndex == totalColumnCount) { columnName += "\n"; } From 5a2de8a02cbf6fbfdc35c402af6fa74add069d2d Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Thu, 8 Nov 2018 12:27:51 -0500 Subject: [PATCH 082/129] Bug fix on empty table display --- Core/src/org/sleuthkit/autopsy/contentviewers/SQLiteViewer.java | 1 + 1 file changed, 1 insertion(+) diff --git a/Core/src/org/sleuthkit/autopsy/contentviewers/SQLiteViewer.java b/Core/src/org/sleuthkit/autopsy/contentviewers/SQLiteViewer.java index 3706b92b92..28a719a855 100755 --- a/Core/src/org/sleuthkit/autopsy/contentviewers/SQLiteViewer.java +++ b/Core/src/org/sleuthkit/autopsy/contentviewers/SQLiteViewer.java @@ -406,6 +406,7 @@ class SQLiteViewer extends javax.swing.JPanel implements FileTypeViewer { exportCsvButton.setEnabled(false); nextPageButton.setEnabled(false); + currentTableHeader = new ArrayList<>(); viewReader.read(tableName); Map columnRow = new LinkedHashMap<>(); for(int i = 0; i< currentTableHeader.size(); i++){ From 819cf99f9ea66ae6e3206e675d41a7f699fa2cbf Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Thu, 8 Nov 2018 13:41:12 -0500 Subject: [PATCH 083/129] 4354 adjust comments regarding objectID --- .../datamodel/AbstractSqlEamDb.java | 5 +- .../centralrepository/datamodel/EamDb.java | 62 ++++++++++++------- 2 files changed, 43 insertions(+), 24 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java index 08f6763b17..ec6ee83a31 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java @@ -1448,13 +1448,12 @@ abstract class AbstractSqlEamDb implements EamDb { /** * Find a correlation attribute in the Central Repository database given the - * instance type, case, data source, value, and file path. + * instance type, case, data source, object id. * * @param type The type of instance. * @param correlationCase The case tied to the instance. * @param correlationDataSource The data source tied to the instance. - * @param value The value tied to the instance. - * @param filePath The file path tied to the instance. + * @param objectID The object id of the file tied to the instance. * * @return The correlation attribute if it exists; otherwise null. * diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDb.java index dc1420ff5e..9c09b0d0c7 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDb.java @@ -34,8 +34,7 @@ public interface EamDb { public static final int SCHEMA_VERSION = 2; public static final CaseDbSchemaVersionNumber CURRENT_DB_SCHEMA_VERSION = new CaseDbSchemaVersionNumber(1, 2); - - + /** * Get the instance * @@ -183,13 +182,14 @@ public interface EamDb { * @return The retrieved case */ CorrelationCase getCaseById(int caseId) throws EamDbException; + /** * Retrieves cases that are in DB. * * @return List of cases */ List getCases() throws EamDbException; - + /** * Creates new Data Source in the database * @@ -208,18 +208,17 @@ public interface EamDb { */ CorrelationDataSource getDataSource(CorrelationCase correlationCase, String dataSourceDeviceId) throws EamDbException; - /** * Retrieves Data Source details based on data source ID * - * @param correlationCase the current CorrelationCase used for ensuring - * uniqueness of DataSource - * @param dataSourceId the data source ID number + * @param correlationCase the current CorrelationCase used for ensuring + * uniqueness of DataSource + * @param dataSourceId the data source ID number * * @return The data source */ CorrelationDataSource getDataSourceById(CorrelationCase correlationCase, int dataSourceId) throws EamDbException; - + /** * Retrieves data sources that are in DB * @@ -245,7 +244,7 @@ public interface EamDb { * @return List of artifact instances for a given type/value */ List getArtifactInstancesByTypeValue(CorrelationAttributeInstance.Type aType, String value) throws EamDbException, CorrelationAttributeNormalizationException; - + /** * Retrieves eamArtifact instances from the database that are associated * with the aType and filePath @@ -314,8 +313,8 @@ public interface EamDb { /** * Adds an eamArtifact to an internal list to be later added to DB. Artifact - can have 1 or more Artifact Instances. Insert will be triggered by a - threshold or a call to commitAttributeInstancesBulk(). + * can have 1 or more Artifact Instances. Insert will be triggered by a + * threshold or a call to commitAttributeInstancesBulk(). * * @param eamArtifact The artifact to add */ @@ -323,7 +322,7 @@ public interface EamDb { /** * Executes a bulk insert of the eamArtifacts added from the - addAttributeInstanceBulk() method + * addAttributeInstanceBulk() method */ void commitAttributeInstancesBulk() throws EamDbException; @@ -355,16 +354,32 @@ public interface EamDb { * * @return The correlation attribute if it exists; otherwise null. * - * @deprecated - included to support Central Reposities version 1,1 and older + * @deprecated - included to support instances added using Central Repository version 1,1 and + * older * @throws EamDbException */ - @Deprecated + @Deprecated CorrelationAttributeInstance getCorrelationAttributeInstance(CorrelationAttributeInstance.Type type, CorrelationCase correlationCase, CorrelationDataSource correlationDataSource, String value, String filePath) throws EamDbException, CorrelationAttributeNormalizationException; - + /** + * Find a correlation attribute in the Central Repository database given the + * instance type, case, data source, object id. + * + * @param type The type of instance. + * @param correlationCase The case tied to the instance. + * @param correlationDataSource The data source tied to the instance. + * @param objectID The object id of the file tied to the instance. + * + * @return The correlation attribute if it exists; otherwise null. + * + * @deprecated - included to support Central Reposities version 1,1 and + * older + * @throws EamDbException + */ CorrelationAttributeInstance getCorrelationAttributeInstance(CorrelationAttributeInstance.Type type, CorrelationCase correlationCase, CorrelationDataSource correlationDataSource, long objectID) throws EamDbException, CorrelationAttributeNormalizationException; + /** * Sets an eamArtifact instance to the given known status. If eamArtifact * exists, it is updated. If eamArtifact does not exist nothing happens @@ -388,12 +403,15 @@ public interface EamDb { /** * Gets list of matching eamArtifact instances that have knownStatus = * "Bad". - * + * * @param aType EamArtifact.Type to search for + * * @return List with 0 or more matching eamArtifact instances. + * * @throws EamDbException */ List getArtifactInstancesKnownBad(CorrelationAttributeInstance.Type aType) throws EamDbException; + /** * Count matching eamArtifacts instances that have knownStatus = "Bad". * @@ -495,7 +513,7 @@ public interface EamDb { * * @param eamOrg The organization to add * - * @return The organization with the org ID set. + * @return The organization with the org ID set. * * @throws EamDbException */ @@ -705,18 +723,20 @@ public interface EamDb { /** * Process the Artifact instance in the EamDb * - * @param type EamArtifact.Type to search for + * @param type EamArtifact.Type to search for * @param instanceTableCallback callback to process the instance + * * @throws EamDbException */ void processInstanceTable(CorrelationAttributeInstance.Type type, InstanceTableCallback instanceTableCallback) throws EamDbException; - + /** * Process the Artifact instance in the EamDb * - * @param type EamArtifact.Type to search for + * @param type EamArtifact.Type to search for * @param instanceTableCallback callback to process the instance - * @param whereClause query string to execute + * @param whereClause query string to execute + * * @throws EamDbException */ void processInstanceTableWhere(CorrelationAttributeInstance.Type type, String whereClause, InstanceTableCallback instanceTableCallback) throws EamDbException; From 7199f0e955cb81aa632aa602bbe361a54aeb81a9 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Thu, 8 Nov 2018 14:11:08 -0500 Subject: [PATCH 084/129] Commented and refactored the solution, need to test nothing is broken --- .../autopsy/coreutils/SQLiteTableReader.java | 406 ++++++++++-------- .../coreutils/SQLiteTableReaderException.java | 33 +- 2 files changed, 246 insertions(+), 193 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReader.java b/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReader.java index 6a5540d222..5279e059ee 100755 --- a/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReader.java +++ b/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReader.java @@ -16,7 +16,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.sleuthkit.autopsy.coreutils; import java.io.File; @@ -42,19 +41,41 @@ import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.TskCoreException; /** - * - * @author dsmyda + * Reads row by row through SQLite tables and performs user-defined actions on the row values. + * Table values are processed by data type. Users configure these actions for certain data types + * in the Builder. Example usage: + * + * SQLiteTableReader reader = new SQLiteTableReader.Builder(file) + * .onInteger((i) -> { + * System.out.println(i); + * }).build(); + * reader.read(tableName); + * + * or + * + * SQLiteTableReader reader = new SQLiteTableReader.Builder(file) + * .onInteger(new Consumer() { + * @Override + * public void accept(Integer i) { + * System.out.println(i); + * } + * }).build(); + * reader.reader(tableName); + * + * Invocation of read(String tableName) causes that table name to be processed row by row. + * When an Integer is encountered, its value will be passed to the Consumer that + * was defined above. */ public class SQLiteTableReader implements AutoCloseable { /** - * + * Builder patten for configuring SQLiteTableReader instances. */ public static class Builder { private final AbstractFile file; private Consumer onColumnNameAction; - + private Consumer onStringAction; private Consumer onLongAction; private Consumer onIntegerAction; @@ -63,7 +84,7 @@ public class SQLiteTableReader implements AutoCloseable { private Consumer forAllAction; /** - * Creates a SQLiteTableReaderBuilder for this abstract file. + * Creates a Builder for this abstract file. * * @param file */ @@ -72,12 +93,12 @@ public class SQLiteTableReader implements AutoCloseable { } /** - * Specify a function to handle MetaData parsing. The MetaData object - * will be parsed before any contents are read from the table. + * Specify a function to do on column names. Column names will be read + * from left to right. * - * @param action + * @param action Consumer of column name strings * - * @return + * @return Builder reference */ public Builder onColumnNames(Consumer action) { this.onColumnNameAction = action; @@ -85,12 +106,12 @@ public class SQLiteTableReader implements AutoCloseable { } /** - * Specify a function to do on receiving a database entry that is type - * String. + * Specify a function to do when encountering a database value that is + * of java type String. * - * @param action + * @param action Consumer of strings * - * @return + * @return Builder reference */ public Builder onString(Consumer action) { this.onStringAction = action; @@ -98,12 +119,12 @@ public class SQLiteTableReader implements AutoCloseable { } /** - * Specify a function to do on receiving a database entry that is type - * Integer. + * Specify a function to do when encountering a database value that is + * of java type Integer. * - * @param action + * @param action Consumer of integer * - * @return + * @return Builder reference */ public Builder onInteger(Consumer action) { this.onIntegerAction = action; @@ -111,32 +132,38 @@ public class SQLiteTableReader implements AutoCloseable { } /** - * Specify a function to do on receiving a database entry that is type - * Real. + * Specify a function to do when encountering a database value that is + * of java type Double. * - * @param action + * @param action Consumer of doubles * - * @return + * @return Builder reference */ public Builder onFloat(Consumer action) { this.onFloatAction = action; return this; } - + /** - * - * @param action - * @return + * Specify a function to do when encountering a database value that is + * of java type Long. + * + * @param action Consumer of longs + * + * @return Builder reference */ public Builder onLong(Consumer action) { this.onLongAction = action; return this; } - + /** - * - * @param action - * @return + * Specify a function to do when encountering a database value that is + * of java type byte[] aka blob. + * + * @param action Consumer of blobs + * + * @return Builder reference */ public Builder onBlob(Consumer action) { this.onBlobAction = action; @@ -144,11 +171,13 @@ public class SQLiteTableReader implements AutoCloseable { } /** - * Specify a function to do for any database entry, regardless of type. + * Specify a function to do when encountering any database value, + * regardless of type. This function only captures database values, not + * column names. * - * @param action + * @param action Consumer of objects * - * @return + * @return Builder reference */ public Builder forAll(Consumer action) { this.forAllAction = action; @@ -156,10 +185,10 @@ public class SQLiteTableReader implements AutoCloseable { } /** - * Pass all params to the SQLTableStream so that it can iterate through - * the table + * Creates a SQLiteTableReader instance given this Builder + * configuration. * - * @return + * @return SQLiteTableReader instance */ public SQLiteTableReader build() { return new SQLiteTableReader(this); @@ -182,20 +211,15 @@ public class SQLiteTableReader implements AutoCloseable { //Iteration state variables private Integer currRowColumnIndex; - private boolean unfinishedRowState; private Integer columnNameIndex; private Integer currentColumnCount; private ResultSetMetaData currentMetadata; - private boolean isFinished; - private boolean hasOpened; + private boolean liveResultSet; private String prevTableName; - - private final BooleanSupplier alwaysFalseCondition = () -> {return false;}; /** - * Initialize a new table stream given the parameters passed in from the - * StreamBuilder above. + * Assigns references to each action based on the Builder configuration. */ private SQLiteTableReader(Builder builder) { @@ -209,66 +233,80 @@ public class SQLiteTableReader implements AutoCloseable { this.file = builder.file; } - - private Consumer nonNullValue(Consumer action) { - if (Objects.nonNull(action)) { - return action; - } - //No-op lambda, keep from NPE or having to check during iteration - //if action == null. - return (NO_OP) -> {}; + /** + * Ensures the action is null safe. If action is left null, then during + * iteration null checks would be necessary. To mitigate against that, no-op + * lambdas are substituted for null values. + * + * @param Generic type of consumer + * @param action Consumer for generic type, supplied by Builder. + * + * @return If action is null, then a no-op lambda, if not then the action + * itself. + */ + private Consumer nonNullValue(Consumer action) { + return (Objects.nonNull(action)) ? action : NO_OP -> { + }; } /** - * Get table names from database + * Fetches all table names from the database. * - * @return - * @throws org.sleuthkit.autopsy.coreutils.SQLiteTableReaderException + * @return List of all table names found while querying the sqlite_master + * table * + * @throws SQLiteTableReaderException */ public List getTableNames() throws SQLiteTableReaderException { ensureOpen(); - - List tableNames = new ArrayList<>(); - try (ResultSet tableNameResult = conn.createStatement() .executeQuery("SELECT name FROM sqlite_master " - + " WHERE type= 'table' ")) { + + " WHERE type= 'table' ")) { + List tableNames = new ArrayList<>(); while (tableNameResult.next()) { tableNames.add(tableNameResult.getString("name")); //NON-NLS } + return tableNames; } catch (SQLException ex) { throw new SQLiteTableReaderException(ex); } - - return tableNames; } - + /** - * - * @param tableName - * @return - * @throws org.sleuthkit.autopsy.coreutils.SQLiteTableReaderException + * Fetches the row count. + * + * @param tableName Source table to count + * + * @return Count as an integer + * + * @throws SQLiteTableReaderException */ public int getRowCount(String tableName) throws SQLiteTableReaderException { ensureOpen(); - try (ResultSet countResult = conn.createStatement() - .executeQuery("SELECT count (*) as count FROM " + - "\"" + tableName + "\"")) { + .executeQuery("SELECT count (*) as count FROM " + + "\"" + tableName + "\"")) { return countResult.getInt("count"); } catch (SQLException ex) { throw new SQLiteTableReaderException(ex); } } - + + /** + * Fetches the column count of the table. + * + * @param tableName Source table to count + * + * @return Count as an integer + * + * @throws SQLiteTableReaderException + */ public int getColumnCount(String tableName) throws SQLiteTableReaderException { ensureOpen(); - try (ResultSet columnCount = conn.createStatement() - .executeQuery("SELECT * FROM " + - "\"" + tableName + "\"")) { + .executeQuery("SELECT * FROM " + + "\"" + tableName + "\"")) { return columnCount.getMetaData().getColumnCount(); } catch (SQLException ex) { throw new SQLiteTableReaderException(ex); @@ -276,151 +314,135 @@ public class SQLiteTableReader implements AutoCloseable { } /** - * - * @param tableName - * @throws org.sleuthkit.autopsy.coreutils.SQLiteTableReaderException + * Reads column names and values from the table. Only actions that were + * configured in the Builder will be invoked during iteration. Iteration + * will stop when the table read has completed or an exception was + * encountered. + * + * @param tableName Source table to read + * + * @throws SQLiteTableReaderException */ public void read(String tableName) throws SQLiteTableReaderException { - readHelper("SELECT * FROM \"" + tableName +"\"", alwaysFalseCondition); + readHelper("SELECT * FROM \"" + tableName + "\"", () -> false); } /** - * Read x number of rows (limit), starting from row number y (offset) in - * table z (tableName). + * Reads column names and values from the table. Only actions that were + * configured in the Builder will be invoked during iteration. Column names + * are only read during the first call to this function. Iteration will stop + * when the table read has completed or an exception was encountered. * - * @param tableName - * @param limit - * @param offset - * @throws org.sleuthkit.autopsy.coreutils.SQLiteTableReaderException + * @param tableName Source table to perform a read + * @param limit Number of rows to read from the table + * @param offset Starting row to read from in the table + * + * @throws SQLiteTableReaderException * */ public void read(String tableName, int limit, int offset) throws SQLiteTableReaderException { - readHelper("SELECT * FROM \"" + tableName +"\" LIMIT " + limit - + " OFFSET " + offset, alwaysFalseCondition); + readHelper("SELECT * FROM \"" + tableName + "\" LIMIT " + limit + + " OFFSET " + offset, () -> false); } /** - * Iterate through the table stopping if we are done, an exception is - * thrown, or the condition is false! + * Reads column names and values from the table. Iteration will stop when + * the condition is true. * - * @param tableName - * @param condition - * @throws org.sleuthkit.autopsy.coreutils.SQLiteTableReaderException + * @param tableName Source table to perform a read + * @param condition Condition to stop iteration when true + * + * @throws SQLiteTableReaderException * */ public void read(String tableName, BooleanSupplier condition) throws SQLiteTableReaderException { - if(Objects.nonNull(prevTableName) && prevTableName.equals(tableName)) { + if (Objects.nonNull(prevTableName) && prevTableName.equals(tableName)) { readHelper("SELECT * FROM \"" + tableName + "\"", condition); } else { prevTableName = tableName; - closeResultSet(); + closeTableResources(); readHelper("SELECT * FROM \"" + tableName + "\"", condition); } } /** - * Iterate through the entire table calling the correct function given the - * datatype. Only stop when there is nothing left to read or a SQLException - * is thrown. + * Performs the result set iteration and is responsible for maintaining state + * of the read over multiple invocations. * - * @param tableName - * - * @throws org.sleuthkit.autopsy.core.AutopsySQLiteException + * @throws SQLiteTableReaderException */ private void readHelper(String query, BooleanSupplier condition) throws SQLiteTableReaderException { try { - if(!hasOpened) { - openResultSet(query); - currentMetadata = queryResults.getMetaData(); - currentColumnCount = currentMetadata.getColumnCount(); + if (!liveResultSet) { + openTableResources(query); columnNameIndex = 1; } - - isFinished = false; - - for(; columnNameIndex <= currentColumnCount; columnNameIndex++) { - this.onColumnNameAction.accept(currentMetadata.getColumnName(columnNameIndex)); - } - - while (unfinishedRowState || queryResults.next()) { - if (!unfinishedRowState) { - currRowColumnIndex = 1; + + //Process column names before reading the database table values + for (; columnNameIndex <= currentColumnCount; columnNameIndex++) { + if (condition.getAsBoolean()) { + return; } - - for (; currRowColumnIndex <= currentColumnCount; currRowColumnIndex++) { - + this.onColumnNameAction.accept(currentMetadata + .getColumnName(columnNameIndex)); + } + + //currRowColumnIndex > 0 means we are still reading the current result set row + while (currRowColumnIndex > 0 || queryResults.next()) { + while(currRowColumnIndex < currentColumnCount) { if (condition.getAsBoolean()) { - unfinishedRowState = true; return; } - //getObject automatically instiantiates the correct java data type - Object item = queryResults.getObject(currRowColumnIndex); - if(item instanceof String) { + Object item = queryResults.getObject(++currRowColumnIndex); + if (item instanceof String) { this.onStringAction.accept((String) item); - } else if(item instanceof Integer) { + } else if (item instanceof Integer) { this.onIntegerAction.accept((Integer) item); - } else if(item instanceof Double) { + } else if (item instanceof Double) { this.onFloatAction.accept((Double) item); - } else if(item instanceof Long) { + } else if (item instanceof Long) { this.onLongAction.accept((Long) item); - } else if(item instanceof byte[]) { + } else if (item instanceof byte[]) { this.onBlobAction.accept((byte[]) item); } - + this.forAllAction.accept(item); } - - unfinishedRowState = false; + //Wrap column index back around if we've reached the end of the row + currRowColumnIndex = (currRowColumnIndex % currentColumnCount); } - - isFinished = true; - closeResultSet(); + closeTableResources(); } catch (SQLException ex) { - closeResultSet(); - isFinished = true; + closeTableResources(); throw new SQLiteTableReaderException(ex); } } /** - * - * @throws org.sleuthkit.autopsy.core.AutopsySQLiteException + * Ensures that the underlying database connection is open. This entails + * copying the abstract file contents to temp directory, copying over any + * WAL or SHM files and getting the connection from the DriverManager. + * + * @throws SQLiteTableReaderException */ private void ensureOpen() throws SQLiteTableReaderException { if (Objects.isNull(conn)) { try { Class.forName("org.sqlite.JDBC"); //NON-NLS - String localDiskPath = writeAbstractFileToLocalDisk(file, file.getId()); - findAndCopySQLiteMetaFile(file); + String localDiskPath = copyFileToTempDirectory(file, file.getId()); + + //Find and copy both WAL and SHM meta files + findAndCopySQLiteMetaFile(file, file.getName() + "-wal"); + findAndCopySQLiteMetaFile(file, file.getName() + "-shm"); conn = DriverManager.getConnection("jdbc:sqlite:" + localDiskPath); - } catch (NoCurrentCaseException | TskCoreException | IOException | - ClassNotFoundException | SQLException ex) { + } catch (NoCurrentCaseException | TskCoreException | IOException + | ClassNotFoundException | SQLException ex) { throw new SQLiteTableReaderException(ex); } } } - - /** - * Overloaded implementation of - * {@link #findAndCopySQLiteMetaFile(AbstractFile, String) findAndCopySQLiteMetaFile} - * , automatically tries to copy -wal and -shm files without needing to know - * their existence. - * - * @param sqliteFile file which has -wal and -shm meta files - * - * @throws NoCurrentCaseException Case has been closed. - * @throws TskCoreException fileManager cannot find AbstractFile - * files. - * @throws IOException Issue during writing to file. - */ - private void findAndCopySQLiteMetaFile(AbstractFile sqliteFile) - throws NoCurrentCaseException, TskCoreException, IOException { - findAndCopySQLiteMetaFile(sqliteFile, sqliteFile.getName() + "-wal"); - findAndCopySQLiteMetaFile(sqliteFile, sqliteFile.getName() + "-shm"); - } - /** * Searches for a meta file associated with the give SQLite database. If * found, it copies this file into the temp directory of the current case. @@ -447,11 +469,11 @@ public class SQLiteTableReader implements AutoCloseable { if (metaFiles != null) { for (AbstractFile metaFile : metaFiles) { - writeAbstractFileToLocalDisk(metaFile, sqliteFile.getId()); + copyFileToTempDirectory(metaFile, sqliteFile.getId()); } } } - + /** * Copies the file contents into a unique path in the current case temp * directory. @@ -463,7 +485,7 @@ public class SQLiteTableReader implements AutoCloseable { * @throws IOException Exception writing file contents * @throws NoCurrentCaseException Current case closed during file copying */ - private String writeAbstractFileToLocalDisk(AbstractFile file, long id) + private String copyFileToTempDirectory(AbstractFile file, long id) throws IOException, NoCurrentCaseException { String localDiskPath = Case.getCurrentCaseThrows().getTempDirectory() @@ -474,74 +496,82 @@ public class SQLiteTableReader implements AutoCloseable { } return localDiskPath; } - + /** - * - * @param query - * @throws SQLException + * Executes the query and assigns resource references to instance variables. + * + * @param query Input query to execute + * + * @throws SQLiteTableReaderException */ - private void openResultSet(String query) throws SQLiteTableReaderException { - ensureOpen(); - - try { + private void openTableResources(String query) throws SQLiteTableReaderException { + try { + ensureOpen(); statement = conn.prepareStatement(query); - queryResults = statement.executeQuery(); - hasOpened = true; + currentMetadata = queryResults.getMetaData(); + currentColumnCount = currentMetadata.getColumnCount(); + liveResultSet = true; } catch (SQLException ex) { throw new SQLiteTableReaderException(ex); } } /** - * + * Ensures both the statement and the result set for a table are closed. */ - private void closeResultSet() { + private void closeTableResources() { try { - if(Objects.nonNull(statement)) { + if (Objects.nonNull(statement)) { statement.close(); - } - if(Objects.nonNull(queryResults)) { + } + if (Objects.nonNull(queryResults)) { queryResults.close(); } - hasOpened = false; + liveResultSet = false; } catch (SQLException ex) { //Do nothing, can't close.. tried our best. } } /** - * Closes all connections with the database. + * Closes all resources attached to the database file. + * + * @throws SQLiteTableReaderException */ @Override - public void close() { + public void close() throws SQLiteTableReaderException { try { - closeResultSet(); - if(Objects.nonNull(conn)) { + closeTableResources(); + if (Objects.nonNull(conn)) { conn.close(); } } catch (SQLException ex) { - //Do nothing, can't close.. tried our best. + throw new SQLiteTableReaderException(ex); } } /** - * Checks if there is still work to do on the result set. + * Provides status of the current read operation. * - * @return boolean + * @return */ public boolean isFinished() { - return isFinished; + return !liveResultSet; } /** - * Last ditch effort to close the connections. - * - * @throws Throwable + * Last ditch effort to close the connections during garbage collection. + * + * @throws Throwable */ @Override public void finalize() throws Throwable { super.finalize(); - close(); + try { + close(); + } catch (SQLiteTableReaderException ex) { + //Do nothing, we tried out best to close the connection. + } } -} \ No newline at end of file +} diff --git a/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReaderException.java b/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReaderException.java index 177bd7a500..70ef801673 100755 --- a/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReaderException.java +++ b/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReaderException.java @@ -1,20 +1,43 @@ /* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. + * Autopsy Forensic Browser + * + * Copyright 2018-2018 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.sleuthkit.autopsy.coreutils; /** - * - * @author dsmyda + * Provides a system exception for the SQLiteTableReader class. */ public class SQLiteTableReaderException extends Exception { + /** + * Accepts both a message and a parent exception. + * + * @param msg Message detailing the cause + * @param ex Parent exception + */ public SQLiteTableReaderException(String msg, Throwable ex) { super(msg, ex); } + /** + * Accepts only a parent exception. + * + * @param ex Parent exception + */ public SQLiteTableReaderException(Throwable ex) { super(ex); } From 711ed7a3882ea68e2398413e70c67963c21c775d Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Thu, 8 Nov 2018 14:29:39 -0500 Subject: [PATCH 085/129] Close now throws exception, so had to wrap it here --- .../org/sleuthkit/autopsy/contentviewers/SQLiteViewer.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/contentviewers/SQLiteViewer.java b/Core/src/org/sleuthkit/autopsy/contentviewers/SQLiteViewer.java index 28a719a855..bce1eb9c7e 100755 --- a/Core/src/org/sleuthkit/autopsy/contentviewers/SQLiteViewer.java +++ b/Core/src/org/sleuthkit/autopsy/contentviewers/SQLiteViewer.java @@ -344,7 +344,11 @@ class SQLiteViewer extends javax.swing.JPanel implements FileTypeViewer { tablesDropdownList.removeAllItems(); numEntriesField.setText(""); - viewReader.close(); + try { + viewReader.close(); + } catch (SQLiteTableReaderException ex) { + //Could not successfully close the reader, nothing we can do to recover. + } row = new LinkedHashMap<>(); pageOfTableRows = new ArrayList<>(); currentTableHeader = new ArrayList<>(); From f0998c7a02cd1fd5b571ef4c92e14994dd633e91 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Thu, 8 Nov 2018 14:45:57 -0500 Subject: [PATCH 086/129] Fixed a bug with clicking the same table name --- Core/src/org/sleuthkit/autopsy/contentviewers/SQLiteViewer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/contentviewers/SQLiteViewer.java b/Core/src/org/sleuthkit/autopsy/contentviewers/SQLiteViewer.java index bce1eb9c7e..f2db7c2532 100755 --- a/Core/src/org/sleuthkit/autopsy/contentviewers/SQLiteViewer.java +++ b/Core/src/org/sleuthkit/autopsy/contentviewers/SQLiteViewer.java @@ -435,8 +435,8 @@ class SQLiteViewer extends javax.swing.JPanel implements FileTypeViewer { //were a brand new table. if (!tableName.equals(prevTableName)) { prevTableName = tableName; - currentTableHeader = new ArrayList<>(); } + currentTableHeader = new ArrayList<>(); viewReader.read(tableName, numRowsToRead, startRow - 1); selectedTableView.setupTable(pageOfTableRows); pageOfTableRows = new ArrayList<>(); From 7cc9fc6b789cd51023c5d06341037455de81290c Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Thu, 8 Nov 2018 14:47:42 -0500 Subject: [PATCH 087/129] Minor changes --- .../autopsy/coreutils/SQLiteTableReader.java | 23 +++++++++---------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReader.java b/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReader.java index 5279e059ee..1caf06d5fe 100755 --- a/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReader.java +++ b/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReader.java @@ -210,9 +210,9 @@ public class SQLiteTableReader implements AutoCloseable { private final Consumer forAllAction; //Iteration state variables - private Integer currRowColumnIndex; - private Integer columnNameIndex; - private Integer currentColumnCount; + private int currRowColumnIndex; + private int columnNameIndex; + private int totalColumnCount; private ResultSetMetaData currentMetadata; private boolean liveResultSet; @@ -230,7 +230,7 @@ public class SQLiteTableReader implements AutoCloseable { this.onFloatAction = nonNullValue(builder.onFloatAction); this.onBlobAction = nonNullValue(builder.onBlobAction); this.forAllAction = nonNullValue(builder.forAllAction); - + this.file = builder.file; } @@ -329,8 +329,7 @@ public class SQLiteTableReader implements AutoCloseable { /** * Reads column names and values from the table. Only actions that were - * configured in the Builder will be invoked during iteration. Column names - * are only read during the first call to this function. Iteration will stop + * configured in the Builder will be invoked during iteration. Iteration will stop * when the table read has completed or an exception was encountered. * * @param tableName Source table to perform a read @@ -375,21 +374,21 @@ public class SQLiteTableReader implements AutoCloseable { try { if (!liveResultSet) { openTableResources(query); - columnNameIndex = 1; + columnNameIndex = 0; } //Process column names before reading the database table values - for (; columnNameIndex <= currentColumnCount; columnNameIndex++) { + while(columnNameIndex < totalColumnCount) { if (condition.getAsBoolean()) { return; } this.onColumnNameAction.accept(currentMetadata - .getColumnName(columnNameIndex)); + .getColumnName(++columnNameIndex)); } //currRowColumnIndex > 0 means we are still reading the current result set row while (currRowColumnIndex > 0 || queryResults.next()) { - while(currRowColumnIndex < currentColumnCount) { + while(currRowColumnIndex < totalColumnCount) { if (condition.getAsBoolean()) { return; } @@ -410,7 +409,7 @@ public class SQLiteTableReader implements AutoCloseable { this.forAllAction.accept(item); } //Wrap column index back around if we've reached the end of the row - currRowColumnIndex = (currRowColumnIndex % currentColumnCount); + currRowColumnIndex = currRowColumnIndex % totalColumnCount; } closeTableResources(); } catch (SQLException ex) { @@ -510,7 +509,7 @@ public class SQLiteTableReader implements AutoCloseable { statement = conn.prepareStatement(query); queryResults = statement.executeQuery(); currentMetadata = queryResults.getMetaData(); - currentColumnCount = currentMetadata.getColumnCount(); + totalColumnCount = currentMetadata.getColumnCount(); liveResultSet = true; } catch (SQLException ex) { throw new SQLiteTableReaderException(ex); From 550874ead65f0c819c6e3c0f25d1aa04cc4e7105 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Thu, 8 Nov 2018 15:13:38 -0500 Subject: [PATCH 088/129] added try catch around close function, tested to make sure not SQLiteTableReader changes that got merged in broke anything --- .../autopsy/keywordsearch/SqliteTextExtractor.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SqliteTextExtractor.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SqliteTextExtractor.java index 790db9c005..455a22e367 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SqliteTextExtractor.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SqliteTextExtractor.java @@ -25,6 +25,7 @@ import java.util.Iterator; import java.util.Objects; import java.util.function.Consumer; import java.util.logging.Level; +import org.openide.util.Exceptions; import org.sleuthkit.autopsy.coreutils.SQLiteTableReaderException; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.SQLiteTableReader; @@ -282,7 +283,12 @@ class SqliteTextExtractor extends ContentTextExtractor { @Override public void close() throws IOException { - reader.close(); + try { + reader.close(); + } catch (SQLiteTableReaderException ex) { + //Done reading, but couldn't close the resources. Nothing we can + //do as a recovery. + } } /** From 1fd7ba7f7c5fdc72f3f34a370ad6006cbb5768e0 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Thu, 8 Nov 2018 15:22:27 -0500 Subject: [PATCH 089/129] Added param comment to method header --- Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReader.java | 1 + 1 file changed, 1 insertion(+) diff --git a/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReader.java b/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReader.java index 1caf06d5fe..e59a280ebe 100755 --- a/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReader.java +++ b/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReader.java @@ -478,6 +478,7 @@ public class SQLiteTableReader implements AutoCloseable { * directory. * * @param file AbstractFile from the data source + * @param id The input files id value * * @return The path of the file on disk * From 0ebb6797789d39ff2e84c78711f755f89a9b7752 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Thu, 8 Nov 2018 15:29:19 -0500 Subject: [PATCH 090/129] 4354 make upgrade code succeed when object_id column already exists --- .../datamodel/AbstractSqlEamDb.java | 23 ++++- .../datamodel/PostgresEamDb.java | 67 +++++++++----- .../datamodel/SqliteEamDb.java | 89 +++++++++++++------ 3 files changed, 126 insertions(+), 53 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java index ec6ee83a31..47b27a1e6a 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java @@ -824,14 +824,14 @@ abstract class AbstractSqlEamDb implements EamDb { preparedStatement.setString(4, eamArtifact.getCorrelationValue()); preparedStatement.setString(5, eamArtifact.getFilePath().toLowerCase()); preparedStatement.setByte(6, eamArtifact.getKnownStatus().getFileKnownValue()); - + if ("".equals(eamArtifact.getComment())) { preparedStatement.setNull(7, Types.INTEGER); } else { preparedStatement.setString(7, eamArtifact.getComment()); } preparedStatement.setLong(8, eamArtifact.getFileObjectId()); - + preparedStatement.executeUpdate(); } @@ -1453,7 +1453,8 @@ abstract class AbstractSqlEamDb implements EamDb { * @param type The type of instance. * @param correlationCase The case tied to the instance. * @param correlationDataSource The data source tied to the instance. - * @param objectID The object id of the file tied to the instance. + * @param objectID The object id of the file tied to the + * instance. * * @return The correlation attribute if it exists; otherwise null. * @@ -3139,6 +3140,18 @@ abstract class AbstractSqlEamDb implements EamDb { ); } + /** + * Determine if a specific column already exists in a specific table + * + * @param tableName the table to check for the specified column + * @param columnName the name of the column to check for + * + * @return true if the column exists, false if the column does not exist + * + * @throws EamDbException + */ + abstract boolean doesColumnExist(Connection conn, String tableName, String columnName) throws SQLException; + /** * Upgrade the schema of the database (if needed) * @@ -3239,7 +3252,9 @@ abstract class AbstractSqlEamDb implements EamDb { //add object_id column to existing _instances tables for (CorrelationAttributeInstance.Type type : CorrelationAttributeInstance.getDefaultCorrelationTypes()) { instance_type_dbname = EamDbUtil.correlationTypeToInstanceTableName(type); - statement.execute(String.format(addObjectIdColumnTemplate, instance_type_dbname)); //NON-NLS + if (!doesColumnExist(conn, instance_type_dbname, "object_id")) { + statement.execute(String.format(addObjectIdColumnTemplate, instance_type_dbname)); //NON-NLS + } statement.execute(String.format(addObjectIdIndexTemplate, instance_type_dbname, instance_type_dbname)); } //update central repository to be able to store new correlation attributes diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDb.java index 97abd1dec9..0873fef015 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDb.java @@ -19,6 +19,7 @@ package org.sleuthkit.autopsy.centralrepository.datamodel; import java.sql.Connection; +import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import java.util.concurrent.TimeUnit; @@ -29,8 +30,7 @@ import org.sleuthkit.autopsy.core.UserPreferences; import org.sleuthkit.autopsy.coreutils.Logger; /** - * Central Repository database implementation using Postgres as a - * backend + * Central Repository database implementation using Postgres as a backend */ final class PostgresEamDb extends AbstractSqlEamDb { @@ -47,10 +47,11 @@ final class PostgresEamDb extends AbstractSqlEamDb { /** * Get the singleton instance of PostgresEamDb - * + * * @return the singleton instance of PostgresEamDb - * - * @throws EamDbException if one or more default correlation type(s) have an invalid db table name. + * + * @throws EamDbException if one or more default correlation type(s) have an + * invalid db table name. */ public synchronized static PostgresEamDb getInstance() throws EamDbException { if (instance == null) { @@ -61,9 +62,10 @@ final class PostgresEamDb extends AbstractSqlEamDb { } /** - * - * @throws EamDbException if the AbstractSqlEamDb class has one or more default - * correlation type(s) having an invalid db table name. + * + * @throws EamDbException if the AbstractSqlEamDb class has one or more + * default correlation type(s) having an invalid db + * table name. */ private PostgresEamDb() throws EamDbException { dbSettings = new PostgresEamDbSettings(); @@ -73,8 +75,8 @@ final class PostgresEamDb extends AbstractSqlEamDb { @Override public void shutdownConnections() throws EamDbException { try { - synchronized(this) { - if(connectionPool != null){ + synchronized (this) { + if (connectionPool != null) { connectionPool.close(); connectionPool = null; // force it to be re-created on next connect() } @@ -148,7 +150,7 @@ final class PostgresEamDb extends AbstractSqlEamDb { connectionURL.append(dbSettings.getPort()); connectionURL.append("/"); connectionURL.append(dbSettings.getDbName()); - + connectionPool.setUrl(connectionURL.toString()); connectionPool.setUsername(dbSettings.getUserName()); connectionPool.setPassword(dbSettings.getPassword()); @@ -189,31 +191,34 @@ final class PostgresEamDb extends AbstractSqlEamDb { protected String getConflictClause() { return CONFLICT_CLAUSE; } - + /** - * Gets an exclusive lock (if applicable). - * Will return the lock if successful, null if unsuccessful because locking - * isn't supported, and throw an exception if we should have been able to get the - * lock but failed (meaning the database is in use). + * Gets an exclusive lock (if applicable). Will return the lock if + * successful, null if unsuccessful because locking isn't supported, and + * throw an exception if we should have been able to get the lock but failed + * (meaning the database is in use). + * * @return the lock, or null if locking is not supported - * @throws EamDbException if the coordination service is running but we fail to get the lock + * + * @throws EamDbException if the coordination service is running but we fail + * to get the lock */ @Override - public CoordinationService.Lock getExclusiveMultiUserDbLock() throws EamDbException{ + public CoordinationService.Lock getExclusiveMultiUserDbLock() throws EamDbException { try { // First check if multi user mode is enabled - if not there's no point trying to get a lock - if( ! UserPreferences.getIsMultiUserModeEnabled()){ + if (!UserPreferences.getIsMultiUserModeEnabled()) { return null; } - + String databaseNodeName = dbSettings.getHost() + "_" + dbSettings.getDbName(); CoordinationService.Lock lock = CoordinationService.getInstance().tryGetExclusiveLock(CoordinationService.CategoryNode.CENTRAL_REPO, databaseNodeName, 5, TimeUnit.MINUTES); - if(lock != null){ + if (lock != null) { return lock; } throw new EamDbException("Error acquiring database lock"); - } catch (InterruptedException ex){ + } catch (InterruptedException ex) { throw new EamDbException("Error acquiring database lock"); } catch (CoordinationService.CoordinationServiceException ex) { // This likely just means the coordination service isn't running, which is ok @@ -221,4 +226,22 @@ final class PostgresEamDb extends AbstractSqlEamDb { } } + @Override + boolean doesColumnExist(Connection conn, String tableName, String columnName) throws SQLException { + final String objectIdColumnExistsTemplate = "SELECT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name='%s' AND column_name='%s')"; + ResultSet resultSet; + Statement statement = conn.createStatement(); + boolean columnExists = false; + resultSet = statement.executeQuery(String.format(objectIdColumnExistsTemplate, tableName, columnName)); + try { + if (resultSet.next()) { + columnExists = resultSet.getBoolean(1); + } + } finally { + resultSet.close(); + statement.close(); + } + return columnExists; + } + } diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDb.java index a75f4648ff..ddfdb0df3f 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDb.java @@ -19,6 +19,7 @@ package org.sleuthkit.autopsy.centralrepository.datamodel; import java.sql.Connection; +import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import java.util.Arrays; @@ -57,7 +58,7 @@ final class SqliteEamDb extends AbstractSqlEamDb { * @return the singleton instance of SqliteEamDb * * @throws EamDbException if one or more default correlation type(s) have an - * invalid db table name. + * invalid db table name. */ public synchronized static SqliteEamDb getInstance() throws EamDbException { if (instance == null) { @@ -70,7 +71,8 @@ final class SqliteEamDb extends AbstractSqlEamDb { /** * * @throws EamDbException if the AbstractSqlEamDb class has one or more - * default correlation type(s) having an invalid db table name. + * default correlation type(s) having an invalid db + * table name. */ private SqliteEamDb() throws EamDbException { dbSettings = new SqliteEamDbSettings(); @@ -205,7 +207,7 @@ final class SqliteEamDb extends AbstractSqlEamDb { /** * Add a new name/value pair in the db_info table. * - * @param name Key to set + * @param name Key to set * @param value Value to set * * @throws EamDbException @@ -242,7 +244,7 @@ final class SqliteEamDb extends AbstractSqlEamDb { /** * Update the value for a name in the name/value db_info table. * - * @param name Name to find + * @param name Name to find * @param value Value to assign to name. * * @throws EamDbException @@ -372,8 +374,8 @@ final class SqliteEamDb extends AbstractSqlEamDb { /** * Retrieves Data Source details based on data source device ID * - * @param correlationCase the current CorrelationCase used for ensuring - * uniqueness of DataSource + * @param correlationCase the current CorrelationCase used for ensuring + * uniqueness of DataSource * @param dataSourceDeviceId the data source device ID number * * @return The data source @@ -387,13 +389,13 @@ final class SqliteEamDb extends AbstractSqlEamDb { releaseSharedLock(); } } - + /** * Retrieves Data Source details based on data source ID * - * @param correlationCase the current CorrelationCase used for ensuring - * uniqueness of DataSource - * @param dataSourceId the data source ID number + * @param correlationCase the current CorrelationCase used for ensuring + * uniqueness of DataSource + * @param dataSourceId the data source ID number * * @return The data source */ @@ -461,7 +463,7 @@ final class SqliteEamDb extends AbstractSqlEamDb { * Retrieves eamArtifact instances from the database that are associated * with the aType and filePath * - * @param aType EamArtifact.Type to search for + * @param aType EamArtifact.Type to search for * @param filePath File path to search for * * @return List of 0 or more EamArtifactInstances @@ -486,7 +488,8 @@ final class SqliteEamDb extends AbstractSqlEamDb { * @param value The value to search for * * @return Number of artifact instances having ArtifactType and - * ArtifactValue. + * ArtifactValue. + * * @throws EamDbException */ @Override @@ -518,6 +521,7 @@ final class SqliteEamDb extends AbstractSqlEamDb { * @param value The value to search for * * @return Number of unique tuples + * * @throws EamDbException */ @Override @@ -545,11 +549,11 @@ final class SqliteEamDb extends AbstractSqlEamDb { * associated with the caseDisplayName and dataSource of the given * eamArtifact instance. * - * @param caseUUID Case ID to search for + * @param caseUUID Case ID to search for * @param dataSourceID Data source ID to search for * * @return Number of artifact instances having caseDisplayName and - * dataSource + * dataSource */ @Override public Long getCountArtifactInstancesByCaseDataSource(String caseUUID, String dataSourceID) throws EamDbException { @@ -563,7 +567,7 @@ final class SqliteEamDb extends AbstractSqlEamDb { /** * Executes a bulk insert of the eamArtifacts added from the - addAttributeInstanceBulk() method + * addAttributeInstanceBulk() method */ @Override public void commitAttributeInstancesBulk() throws EamDbException { @@ -596,7 +600,7 @@ final class SqliteEamDb extends AbstractSqlEamDb { * * @param eamArtifact Artifact containing exactly one (1) ArtifactInstance. * @param knownStatus The status to change the artifact to. Should never be - * KNOWN + * KNOWN */ @Override public void setAttributeInstanceKnownStatus(CorrelationAttributeInstance eamArtifact, TskData.FileKnown knownStatus) throws EamDbException { @@ -633,7 +637,9 @@ final class SqliteEamDb extends AbstractSqlEamDb { * "Bad". * * @param aType EamArtifact.Type to search for + * * @return List with 0 or more matching eamArtifact instances. + * * @throws EamDbException */ @Override @@ -672,7 +678,7 @@ final class SqliteEamDb extends AbstractSqlEamDb { * @param value Value to search for * * @return List of cases containing this artifact with instances marked as - * bad + * bad * * @throws EamDbException */ @@ -690,6 +696,7 @@ final class SqliteEamDb extends AbstractSqlEamDb { * Remove a reference set and all values contained in it. * * @param referenceSetID + * * @throws EamDbException */ @Override @@ -708,6 +715,7 @@ final class SqliteEamDb extends AbstractSqlEamDb { * @param value * @param referenceSetID * @param correlationTypeID + * * @return true if the hash is found in the reference set */ @Override @@ -723,8 +731,9 @@ final class SqliteEamDb extends AbstractSqlEamDb { /** * Process the Artifact instance in the EamDb * - * @param type EamArtifact.Type to search for + * @param type EamArtifact.Type to search for * @param instanceTableCallback callback to process the instance + * * @throws EamDbException */ @Override @@ -736,12 +745,13 @@ final class SqliteEamDb extends AbstractSqlEamDb { releaseSharedLock(); } } - + /** * Process the Artifact instance in the EamDb * - * @param type EamArtifact.Type to search for + * @param type EamArtifact.Type to search for * @param instanceTableCallback callback to process the instance + * * @throws EamDbException */ @Override @@ -752,7 +762,7 @@ final class SqliteEamDb extends AbstractSqlEamDb { } finally { releaseSharedLock(); } - } + } /** * Check whether a reference set with the given name/version is in the @@ -761,7 +771,9 @@ final class SqliteEamDb extends AbstractSqlEamDb { * * @param referenceSetName * @param version + * * @return true if a matching set is found + * * @throws EamDbException */ @Override @@ -928,7 +940,8 @@ final class SqliteEamDb extends AbstractSqlEamDb { * Add a new reference instance * * @param eamGlobalFileInstance The reference instance to add - * @param correlationType Correlation Type that this Reference Instance is + * @param correlationType Correlation Type that this Reference + * Instance is * * @throws EamDbException */ @@ -960,7 +973,7 @@ final class SqliteEamDb extends AbstractSqlEamDb { /** * Get all reference entries having a given correlation type and value * - * @param aType Type to use for matching + * @param aType Type to use for matching * @param aValue Value to use for matching * * @return List of all global file instances with a type and value @@ -1001,7 +1014,7 @@ final class SqliteEamDb extends AbstractSqlEamDb { * artifacts. * * @return List of EamArtifact.Type's. If none are defined in the database, - * the default list will be returned. + * the default list will be returned. * * @throws EamDbException */ @@ -1020,7 +1033,7 @@ final class SqliteEamDb extends AbstractSqlEamDb { * artifacts. * * @return List of enabled EamArtifact.Type's. If none are defined in the - * database, the default list will be returned. + * database, the default list will be returned. * * @throws EamDbException */ @@ -1039,7 +1052,7 @@ final class SqliteEamDb extends AbstractSqlEamDb { * correlate artifacts. * * @return List of supported EamArtifact.Type's. If none are defined in the - * database, the default list will be returned. + * database, the default list will be returned. * * @throws EamDbException */ @@ -1111,8 +1124,9 @@ final class SqliteEamDb extends AbstractSqlEamDb { * (meaning the database is in use). * * @return the lock, or null if locking is not supported + * * @throws EamDbException if the coordination service is running but we fail - * to get the lock + * to get the lock */ @Override public CoordinationService.Lock getExclusiveMultiUserDbLock() throws EamDbException { @@ -1156,4 +1170,25 @@ final class SqliteEamDb extends AbstractSqlEamDb { rwLock.readLock().unlock(); } + @Override + boolean doesColumnExist(Connection conn, String tableName, String columnName) throws SQLException { + final String tableInfoQueryTemplate = "PRAGMA table_info(%s)"; + ResultSet resultSet; + Statement statement = conn.createStatement(); + boolean columnExists = false; + resultSet = statement.executeQuery(String.format(tableInfoQueryTemplate, tableName)); + try { + while (resultSet.next()) { + // the second value ( 2 ) is the column name + if (resultSet.getString(2).equals(columnName)) { + columnExists = true; + break; + } + } + } finally { + resultSet.close(); + statement.close(); + } + return columnExists; + } } From 67a2dbb294793c1148f28a8b882e2a51757bc153 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Thu, 8 Nov 2018 15:40:27 -0500 Subject: [PATCH 091/129] 4354 add NON-NLS comment to database opperations --- .../centralrepository/datamodel/AbstractSqlEamDb.java | 8 ++++---- .../centralrepository/datamodel/PostgresEamDb.java | 2 +- .../autopsy/centralrepository/datamodel/SqliteEamDb.java | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java index 47b27a1e6a..c350e2f441 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java @@ -3217,7 +3217,7 @@ abstract class AbstractSqlEamDb implements EamDb { if (dbSchemaVersion.compareTo(new CaseDbSchemaVersionNumber(1, 2)) < 0) { EamDbPlatformEnum selectedPlatform = EamDbPlatformEnum.getSelectedPlatform(); - final String addObjectIdColumnTemplate = "ALTER TABLE %s ADD COLUMN object_id INTEGER;"; + final String addObjectIdColumnTemplate = "ALTER TABLE %s ADD COLUMN object_id INTEGER;"; //NON-NLS final String addSsidTableTemplate; final String addCaseIdIndexTemplate; final String addDataSourceIdIndexTemplate; @@ -3228,7 +3228,7 @@ abstract class AbstractSqlEamDb implements EamDb { //get the data base specific code for creating a new _instance table switch (selectedPlatform) { case POSTGRESQL: - addAttributeSql = "INSERT INTO correlation_types(id, display_name, db_table_name, supported, enabled) VALUES (?, ?, ?, ?, ?) " + getConflictClause(); + addAttributeSql = "INSERT INTO correlation_types(id, display_name, db_table_name, supported, enabled) VALUES (?, ?, ?, ?, ?) " + getConflictClause(); //NON-NLS addSsidTableTemplate = PostgresEamDbSettings.getCreateArtifactInstancesTableTemplate(); addCaseIdIndexTemplate = PostgresEamDbSettings.getAddCaseIdIndexTemplate(); addDataSourceIdIndexTemplate = PostgresEamDbSettings.getAddDataSourceIdIndexTemplate(); @@ -3237,7 +3237,7 @@ abstract class AbstractSqlEamDb implements EamDb { addObjectIdIndexTemplate = PostgresEamDbSettings.getAddObjectIdIndexTemplate(); break; case SQLITE: - addAttributeSql = "INSERT OR IGNORE INTO correlation_types(id, display_name, db_table_name, supported, enabled) VALUES (?, ?, ?, ?, ?)"; + addAttributeSql = "INSERT OR IGNORE INTO correlation_types(id, display_name, db_table_name, supported, enabled) VALUES (?, ?, ?, ?, ?)"; //NON-NLS addSsidTableTemplate = SqliteEamDbSettings.getCreateArtifactInstancesTableTemplate(); addCaseIdIndexTemplate = SqliteEamDbSettings.getAddCaseIdIndexTemplate(); addDataSourceIdIndexTemplate = SqliteEamDbSettings.getAddDataSourceIdIndexTemplate(); @@ -3249,7 +3249,7 @@ abstract class AbstractSqlEamDb implements EamDb { throw new EamDbException("Currently selected database platform \"" + selectedPlatform.name() + "\" can not be upgraded."); } String instance_type_dbname; - //add object_id column to existing _instances tables + //add object_id column to existing _instances table for (CorrelationAttributeInstance.Type type : CorrelationAttributeInstance.getDefaultCorrelationTypes()) { instance_type_dbname = EamDbUtil.correlationTypeToInstanceTableName(type); if (!doesColumnExist(conn, instance_type_dbname, "object_id")) { diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDb.java index 0873fef015..532cc46ad0 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDb.java @@ -228,7 +228,7 @@ final class PostgresEamDb extends AbstractSqlEamDb { @Override boolean doesColumnExist(Connection conn, String tableName, String columnName) throws SQLException { - final String objectIdColumnExistsTemplate = "SELECT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name='%s' AND column_name='%s')"; + final String objectIdColumnExistsTemplate = "SELECT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name='%s' AND column_name='%s')"; //NON-NLS ResultSet resultSet; Statement statement = conn.createStatement(); boolean columnExists = false; diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDb.java index ddfdb0df3f..485bc58295 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDb.java @@ -1172,7 +1172,7 @@ final class SqliteEamDb extends AbstractSqlEamDb { @Override boolean doesColumnExist(Connection conn, String tableName, String columnName) throws SQLException { - final String tableInfoQueryTemplate = "PRAGMA table_info(%s)"; + final String tableInfoQueryTemplate = "PRAGMA table_info(%s)"; //NON-NLS ResultSet resultSet; Statement statement = conn.createStatement(); boolean columnExists = false; From 3f2a748d9d48868c9774ad3ba328e72136710487 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Thu, 8 Nov 2018 15:43:32 -0500 Subject: [PATCH 092/129] Fixing unncessary class indirection --- .../SevenZipExtractor.java | 53 ++----------------- 1 file changed, 4 insertions(+), 49 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java index 0deea2baf0..d3cc8f159f 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java +++ b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java @@ -785,51 +785,6 @@ class SevenZipExtractor { .mapToInt(Integer::intValue) .toArray(); } - - /** - * Stream used to unpack the archive item to local file. This will be used when - * the 7ZIP bindings call getStream() on the StandardIArchiveExtractCallback. - */ - private final static class MutableEncodedFileOutputStream extends EncodedFileOutputStream - implements AutoCloseable { - - private static final int HEADER_LENGTH = 32; - private static final String HEADER = "TSK_CONTAINER_XOR1_xxxxxxxxxxxxx"; - - MutableEncodedFileOutputStream(String localAbsPath) throws IOException { - super(new FileOutputStream(localAbsPath), TskData.EncodingType.XOR1); - } - - /** - * Update the OutputStream to point to a new File. This method mutates the state so - * that there is no overhead of creating a new object and buffer. Additionally, the - * 7zip binding has a memory leak, so this prevents multiple streams from being created - * and never GC'd since the 7zip lib never frees a reference to them. - * - * @param localAbsPath Path to local file - * @throws IOException - */ - public void setNewOutputStream(String localAbsPath) throws IOException { - this.out.close(); - this.out = new FileOutputStream(localAbsPath); - writeHeader(); - } - - private byte[] getEncodedHeader() { - byte [] encHeader = new byte[HEADER_LENGTH]; - byte [] plainHeader = HEADER.getBytes(); - - for(int i = 0; i < HEADER_LENGTH; i++){ - encHeader[i] = ((byte)(plainHeader[i] ^ 0xca)); - } - return encHeader; - } - - private void writeHeader() throws IOException { - // We get the encoded header here so it will be in plaintext after encoding - write(getEncodedHeader(), 0, HEADER_LENGTH); - } - } /** * UnpackStream used by the SevenZipBindings to do archive extraction. A memory @@ -841,18 +796,18 @@ class SevenZipExtractor { */ private final static class UnpackStream implements ISequentialOutStream { - private final MutableEncodedFileOutputStream output; + private EncodedFileOutputStream output; private String localAbsPath; private int bytesWritten; UnpackStream(String localAbsPath) throws IOException { - this.output = new MutableEncodedFileOutputStream(localAbsPath); + this.output = new EncodedFileOutputStream(localAbsPath); this.localAbsPath = localAbsPath; this.bytesWritten = 0; } public void setNewOutputStream(String localAbsPath) throws IOException { - this.output.setNewOutputStream(localAbsPath); + this.output = new EncodedFileOutputStream(localAbsPath); this.localAbsPath = localAbsPath; this.bytesWritten = 0; } @@ -876,7 +831,7 @@ class SevenZipExtractor { } public void close() throws IOException { - try(MutableEncodedFileOutputStream out = output) { + try(EncodedFileOutputStream out = output) { out.flush(); } } From 3040976ab81645ab670f0e49110a4c75d9fdf2c2 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Thu, 8 Nov 2018 15:55:11 -0500 Subject: [PATCH 093/129] Made only unpackstream wrap an encodedfileoutputstream --- .../modules/embeddedfileextractor/SevenZipExtractor.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java index dba3e82e81..c9cdf59d74 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java +++ b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java @@ -809,13 +809,14 @@ class SevenZipExtractor { private int bytesWritten; UnpackStream(String localAbsPath) throws IOException { - this.output = new EncodedFileOutputStream(localAbsPath); + this.output = new EncodedFileOutputStream(new FileOutputStream(localAbsPath), TskData.EncodingType.XOR1); this.localAbsPath = localAbsPath; this.bytesWritten = 0; } public void setNewOutputStream(String localAbsPath) throws IOException { - this.output = new EncodedFileOutputStream(localAbsPath); + this.output.close(); + this.output = new EncodedFileOutputStream(new FileOutputStream(localAbsPath), TskData.EncodingType.XOR1); this.localAbsPath = localAbsPath; this.bytesWritten = 0; } From e77faa4d2d011822c66a1951c760a775b1b9cc37 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Thu, 8 Nov 2018 15:57:16 -0500 Subject: [PATCH 094/129] Fixed a log message --- .../modules/embeddedfileextractor/SevenZipExtractor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java index c9cdf59d74..45963d7cdd 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java +++ b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java @@ -955,7 +955,7 @@ class SevenZipExtractor { } } catch (IOException ex) { logger.log(Level.WARNING, String.format("Error opening or setting new stream " //NON-NLS - + "for archive file at %s", localAbsPath), ex); //NON-NLS + + "for archive file at %s", localAbsPath), ex.getMessage()); //NON-NLS return null; } From b1708f777eed309fb70e030f1487b0c9df145855 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Fri, 9 Nov 2018 09:48:52 -0500 Subject: [PATCH 095/129] Bug fix on iteration --- .../org/sleuthkit/autopsy/coreutils/SQLiteTableReader.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReader.java b/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReader.java index e59a280ebe..94306fa258 100755 --- a/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReader.java +++ b/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReader.java @@ -213,6 +213,7 @@ public class SQLiteTableReader implements AutoCloseable { private int currRowColumnIndex; private int columnNameIndex; private int totalColumnCount; + private boolean unfinishedRow; private ResultSetMetaData currentMetadata; private boolean liveResultSet; @@ -386,10 +387,10 @@ public class SQLiteTableReader implements AutoCloseable { .getColumnName(++columnNameIndex)); } - //currRowColumnIndex > 0 means we are still reading the current result set row - while (currRowColumnIndex > 0 || queryResults.next()) { + while (unfinishedRow || queryResults.next()) { while(currRowColumnIndex < totalColumnCount) { if (condition.getAsBoolean()) { + unfinishedRow = true; return; } @@ -408,6 +409,7 @@ public class SQLiteTableReader implements AutoCloseable { this.forAllAction.accept(item); } + unfinishedRow = false; //Wrap column index back around if we've reached the end of the row currRowColumnIndex = currRowColumnIndex % totalColumnCount; } From 8feb1314f1d834c4fac1a524832812c3e680dc71 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Fri, 9 Nov 2018 16:09:19 -0500 Subject: [PATCH 096/129] 4354 fix issues found in PR review and ordering issue with upgrade --- .../datamodel/AbstractSqlEamDb.java | 23 +++++++++---------- .../centralrepository/datamodel/EamDb.java | 10 ++++---- .../datamodel/PostgresEamDb.java | 11 +++++---- .../datamodel/SqliteEamDb.java | 11 +++++---- 4 files changed, 27 insertions(+), 28 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java index c350e2f441..bddfbea7b1 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java @@ -1564,7 +1564,7 @@ abstract class AbstractSqlEamDb implements EamDb { int instanceId = resultSet.getInt(1); int knownStatus = resultSet.getInt(2); String comment = resultSet.getString(3); - //null objectId used because we only fall back to using this method when objec + //null objectId used because we only fall back to using this method when objectID was not available correlationAttributeInstance = new CorrelationAttributeInstance(type, value, instanceId, correlationCase, correlationDataSource, filePath, comment, TskData.FileKnown.valueOf((byte) knownStatus), null); } @@ -3248,15 +3248,7 @@ abstract class AbstractSqlEamDb implements EamDb { default: throw new EamDbException("Currently selected database platform \"" + selectedPlatform.name() + "\" can not be upgraded."); } - String instance_type_dbname; - //add object_id column to existing _instances table - for (CorrelationAttributeInstance.Type type : CorrelationAttributeInstance.getDefaultCorrelationTypes()) { - instance_type_dbname = EamDbUtil.correlationTypeToInstanceTableName(type); - if (!doesColumnExist(conn, instance_type_dbname, "object_id")) { - statement.execute(String.format(addObjectIdColumnTemplate, instance_type_dbname)); //NON-NLS - } - statement.execute(String.format(addObjectIdIndexTemplate, instance_type_dbname, instance_type_dbname)); - } + //update central repository to be able to store new correlation attributes final String wirelessNetworsDbTableName = "wireless_networks"; final String wirelessNetworksTableInstanceName = wirelessNetworsDbTableName + "_instances"; @@ -3268,7 +3260,6 @@ abstract class AbstractSqlEamDb implements EamDb { preparedStatement.setInt(4, 1); preparedStatement.setInt(5, 1); preparedStatement.execute(); - //create a new wireless_networks_instances table and add indexes for its columns statement.execute(String.format(addSsidTableTemplate, wirelessNetworksTableInstanceName, wirelessNetworksTableInstanceName)); statement.execute(String.format(addCaseIdIndexTemplate, wirelessNetworksTableInstanceName, wirelessNetworksTableInstanceName)); @@ -3276,7 +3267,15 @@ abstract class AbstractSqlEamDb implements EamDb { statement.execute(String.format(addValueIndexTemplate, wirelessNetworksTableInstanceName, wirelessNetworksTableInstanceName)); statement.execute(String.format(addKnownStatusIndexTemplate, wirelessNetworksTableInstanceName, wirelessNetworksTableInstanceName)); statement.execute(String.format(addObjectIdIndexTemplate, wirelessNetworksTableInstanceName, wirelessNetworksTableInstanceName)); - + //add object_id column to _instances table which do not already have it + String instance_type_dbname; + for (CorrelationAttributeInstance.Type type : CorrelationAttributeInstance.getDefaultCorrelationTypes()) { + instance_type_dbname = EamDbUtil.correlationTypeToInstanceTableName(type); + if (!doesColumnExist(conn, instance_type_dbname, "object_id")) { + statement.execute(String.format(addObjectIdColumnTemplate, instance_type_dbname)); //NON-NLS + } + statement.execute(String.format(addObjectIdIndexTemplate, instance_type_dbname, instance_type_dbname)); + } } if (!updateSchemaVersion(conn)) { throw new EamDbException("Error updating schema version"); diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDb.java index 9c09b0d0c7..95584d0ebe 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/EamDb.java @@ -345,6 +345,9 @@ public interface EamDb { /** * Find a correlation attribute in the Central Repository database given the * instance type, case, data source, value, and file path. + * + * Method exists to support instances added using Central Repository version 1,1 and + * older * * @param type The type of instance. * @param correlationCase The case tied to the instance. @@ -353,12 +356,9 @@ public interface EamDb { * @param filePath The file path tied to the instance. * * @return The correlation attribute if it exists; otherwise null. - * - * @deprecated - included to support instances added using Central Repository version 1,1 and - * older + * * @throws EamDbException */ - @Deprecated CorrelationAttributeInstance getCorrelationAttributeInstance(CorrelationAttributeInstance.Type type, CorrelationCase correlationCase, CorrelationDataSource correlationDataSource, String value, String filePath) throws EamDbException, CorrelationAttributeNormalizationException; @@ -373,8 +373,6 @@ public interface EamDb { * * @return The correlation attribute if it exists; otherwise null. * - * @deprecated - included to support Central Reposities version 1,1 and - * older * @throws EamDbException */ CorrelationAttributeInstance getCorrelationAttributeInstance(CorrelationAttributeInstance.Type type, CorrelationCase correlationCase, diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDb.java index 532cc46ad0..769b49bfd3 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDb.java @@ -229,17 +229,18 @@ final class PostgresEamDb extends AbstractSqlEamDb { @Override boolean doesColumnExist(Connection conn, String tableName, String columnName) throws SQLException { final String objectIdColumnExistsTemplate = "SELECT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name='%s' AND column_name='%s')"; //NON-NLS - ResultSet resultSet; - Statement statement = conn.createStatement(); + ResultSet resultSet = null; + Statement statement = null; boolean columnExists = false; - resultSet = statement.executeQuery(String.format(objectIdColumnExistsTemplate, tableName, columnName)); try { + statement = conn.createStatement(); + resultSet = statement.executeQuery(String.format(objectIdColumnExistsTemplate, tableName, columnName)); if (resultSet.next()) { columnExists = resultSet.getBoolean(1); } } finally { - resultSet.close(); - statement.close(); + EamDbUtil.closeResultSet(resultSet); + EamDbUtil.closeStatement(statement); } return columnExists; } diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDb.java index 485bc58295..6468801a57 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDb.java @@ -1173,11 +1173,12 @@ final class SqliteEamDb extends AbstractSqlEamDb { @Override boolean doesColumnExist(Connection conn, String tableName, String columnName) throws SQLException { final String tableInfoQueryTemplate = "PRAGMA table_info(%s)"; //NON-NLS - ResultSet resultSet; - Statement statement = conn.createStatement(); + ResultSet resultSet = null; + Statement statement = null; boolean columnExists = false; - resultSet = statement.executeQuery(String.format(tableInfoQueryTemplate, tableName)); try { + statement = conn.createStatement(); + resultSet = statement.executeQuery(String.format(tableInfoQueryTemplate, tableName)); while (resultSet.next()) { // the second value ( 2 ) is the column name if (resultSet.getString(2).equals(columnName)) { @@ -1186,8 +1187,8 @@ final class SqliteEamDb extends AbstractSqlEamDb { } } } finally { - resultSet.close(); - statement.close(); + EamDbUtil.closeResultSet(resultSet); + EamDbUtil.closeStatement(statement); } return columnExists; } From 061a5debb1edbabdcc217c0d60e1ede7f09a0b15 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Fri, 9 Nov 2018 17:46:55 -0500 Subject: [PATCH 097/129] Add missing break to case statement for HTML Report generation --- Core/src/org/sleuthkit/autopsy/report/ReportHTML.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/report/ReportHTML.java b/Core/src/org/sleuthkit/autopsy/report/ReportHTML.java index 0d9210df27..13a5078658 100644 --- a/Core/src/org/sleuthkit/autopsy/report/ReportHTML.java +++ b/Core/src/org/sleuthkit/autopsy/report/ReportHTML.java @@ -272,7 +272,8 @@ class ReportHTML implements TableReportModule { in = getClass().getResourceAsStream("/org/sleuthkit/autopsy/report/images/accounts.png"); //NON-NLS break; case TSK_WIFI_NETWORK: - in = getClass().getResourceAsStream("/org/sleuthkit/autopsy/report/images/network-wifi.png"); //NON-NLS + in = getClass().getResourceAsStream("/org/sleuthkit/autopsy/report/images/network-wifi.png"); //NON-NLS + break; default: logger.log(Level.WARNING, "useDataTypeIcon: unhandled artifact type = {0}", dataType); //NON-NLS in = getClass().getResourceAsStream("/org/sleuthkit/autopsy/report/images/star.png"); //NON-NLS From 5c7fdf4b712db6144abcc1217fea23628f8a5285 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Mon, 12 Nov 2018 09:03:05 -0500 Subject: [PATCH 098/129] Did some codacy suggestions --- .../autopsy/coreutils/SQLiteTableReader.java | 163 ++++++++---------- .../coreutils/SQLiteTableReaderException.java | 12 +- 2 files changed, 78 insertions(+), 97 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReader.java b/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReader.java index 94306fa258..44a9696eae 100755 --- a/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReader.java +++ b/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReader.java @@ -41,30 +41,30 @@ import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.TskCoreException; /** - * Reads row by row through SQLite tables and performs user-defined actions on the row values. - * Table values are processed by data type. Users configure these actions for certain data types - * in the Builder. Example usage: + * Reads row by row through SQLite tables and performs user-defined actions on + * the row values. Table values are processed by data type. Users configure + * these actions for certain data types in the Builder. Example usage: + * + * SQLiteTableReader reader = new SQLiteTableReader.Builder(file) + * .onInteger((i) + * -> { System.out.println(i); }) + * .build(); * - * SQLiteTableReader reader = new SQLiteTableReader.Builder(file) - * .onInteger((i) -> { - * System.out.println(i); - * }).build(); - * reader.read(tableName); + * reader.read(tableName); + * + * or + * + * SQLiteTableReader reader = new SQLiteTableReader.Builder(file) + * .onInteger(new Consumer() { + * @Override public void accept(Integer i) { + * System.out.println(i); + * } + * }).build(); * - * or - * - * SQLiteTableReader reader = new SQLiteTableReader.Builder(file) - * .onInteger(new Consumer() { - * @Override - * public void accept(Integer i) { - * System.out.println(i); - * } - * }).build(); - * reader.reader(tableName); - * - * Invocation of read(String tableName) causes that table name to be processed row by row. - * When an Integer is encountered, its value will be passed to the Consumer that - * was defined above. + * reader.reader(tableName); + * + * Invocation of read(String tableName) reads row by row. When an Integer is + * encountered, its value will be passed to the Consumer that was defined above. */ public class SQLiteTableReader implements AutoCloseable { @@ -74,14 +74,18 @@ public class SQLiteTableReader implements AutoCloseable { public static class Builder { private final AbstractFile file; + private Consumer onColumnNameAction; - private Consumer onStringAction; private Consumer onLongAction; private Consumer onIntegerAction; private Consumer onFloatAction; private Consumer onBlobAction; private Consumer forAllAction; + + static Consumer doNothing() { + return NOOP -> {}; + } /** * Creates a Builder for this abstract file. @@ -90,6 +94,14 @@ public class SQLiteTableReader implements AutoCloseable { */ public Builder(AbstractFile file) { this.file = file; + + this.onColumnNameAction = Builder.doNothing(); + this.onStringAction = Builder.doNothing(); + this.onLongAction = Builder.doNothing(); + this.onIntegerAction = Builder.doNothing(); + this.onFloatAction = Builder.doNothing(); + this.onBlobAction = Builder.doNothing(); + this.forAllAction = Builder.doNothing(); } /** @@ -196,61 +208,32 @@ public class SQLiteTableReader implements AutoCloseable { } private final AbstractFile file; + private final Builder builder; + + private final String SELECT_ALL_QUERY = "SELECT * FROM \"%s\""; private Connection conn; private PreparedStatement statement; private ResultSet queryResults; + private ResultSetMetaData currentMetadata; - private final Consumer onColumnNameAction; - private final Consumer onStringAction; - private final Consumer onLongAction; - private final Consumer onIntegerAction; - private final Consumer onFloatAction; - private final Consumer onBlobAction; - private final Consumer forAllAction; - - //Iteration state variables + //Iteration state private int currRowColumnIndex; private int columnNameIndex; private int totalColumnCount; private boolean unfinishedRow; - private ResultSetMetaData currentMetadata; - private boolean liveResultSet; private String prevTableName; /** - * Assigns references to each action based on the Builder configuration. + * Holds reference to the builder instance so that we can use its actions + * during iteration. */ private SQLiteTableReader(Builder builder) { - - this.onColumnNameAction = nonNullValue(builder.onColumnNameAction); - this.onStringAction = nonNullValue(builder.onStringAction); - this.onIntegerAction = nonNullValue(builder.onIntegerAction); - this.onLongAction = nonNullValue(builder.onLongAction); - this.onFloatAction = nonNullValue(builder.onFloatAction); - this.onBlobAction = nonNullValue(builder.onBlobAction); - this.forAllAction = nonNullValue(builder.forAllAction); - + this.builder = builder; this.file = builder.file; } - /** - * Ensures the action is null safe. If action is left null, then during - * iteration null checks would be necessary. To mitigate against that, no-op - * lambdas are substituted for null values. - * - * @param Generic type of consumer - * @param action Consumer for generic type, supplied by Builder. - * - * @return If action is null, then a no-op lambda, if not then the action - * itself. - */ - private Consumer nonNullValue(Consumer action) { - return (Objects.nonNull(action)) ? action : NO_OP -> { - }; - } - /** * Fetches all table names from the database. * @@ -306,8 +289,7 @@ public class SQLiteTableReader implements AutoCloseable { public int getColumnCount(String tableName) throws SQLiteTableReaderException { ensureOpen(); try (ResultSet columnCount = conn.createStatement() - .executeQuery("SELECT * FROM " - + "\"" + tableName + "\"")) { + .executeQuery(String.format(SELECT_ALL_QUERY, tableName))) { return columnCount.getMetaData().getColumnCount(); } catch (SQLException ex) { throw new SQLiteTableReaderException(ex); @@ -325,23 +307,24 @@ public class SQLiteTableReader implements AutoCloseable { * @throws SQLiteTableReaderException */ public void read(String tableName) throws SQLiteTableReaderException { - readHelper("SELECT * FROM \"" + tableName + "\"", () -> false); + readHelper(String.format(SELECT_ALL_QUERY, tableName), () -> false); } /** * Reads column names and values from the table. Only actions that were - * configured in the Builder will be invoked during iteration. Iteration will stop - * when the table read has completed or an exception was encountered. + * configured in the Builder will be invoked during iteration. Iteration + * will stop when the table read has completed or an exception was + * encountered. * * @param tableName Source table to perform a read - * @param limit Number of rows to read from the table - * @param offset Starting row to read from in the table + * @param limit Number of rows to read from the table + * @param offset Starting row to read from in the table * * @throws SQLiteTableReaderException * */ public void read(String tableName, int limit, int offset) throws SQLiteTableReaderException { - readHelper("SELECT * FROM \"" + tableName + "\" LIMIT " + limit + readHelper(String.format(SELECT_ALL_QUERY, tableName)+ " LIMIT " + limit + " OFFSET " + offset, () -> false); } @@ -356,18 +339,16 @@ public class SQLiteTableReader implements AutoCloseable { * */ public void read(String tableName, BooleanSupplier condition) throws SQLiteTableReaderException { - if (Objects.nonNull(prevTableName) && prevTableName.equals(tableName)) { - readHelper("SELECT * FROM \"" + tableName + "\"", condition); - } else { + if (Objects.isNull(prevTableName) || !prevTableName.equals(tableName)) { prevTableName = tableName; - closeTableResources(); - readHelper("SELECT * FROM \"" + tableName + "\"", condition); + closeTableResources(); } + readHelper(String.format(SELECT_ALL_QUERY, tableName), condition); } /** - * Performs the result set iteration and is responsible for maintaining state - * of the read over multiple invocations. + * Performs the result set iteration and is responsible for maintaining + * state of the read over multiple invocations. * * @throws SQLiteTableReaderException */ @@ -379,35 +360,35 @@ public class SQLiteTableReader implements AutoCloseable { } //Process column names before reading the database table values - while(columnNameIndex < totalColumnCount) { + while (columnNameIndex < totalColumnCount) { if (condition.getAsBoolean()) { - return; + return; } - this.onColumnNameAction.accept(currentMetadata + builder.onColumnNameAction.accept(currentMetadata .getColumnName(++columnNameIndex)); } while (unfinishedRow || queryResults.next()) { - while(currRowColumnIndex < totalColumnCount) { + while (currRowColumnIndex < totalColumnCount) { if (condition.getAsBoolean()) { unfinishedRow = true; return; } - + Object item = queryResults.getObject(++currRowColumnIndex); if (item instanceof String) { - this.onStringAction.accept((String) item); + builder.onStringAction.accept((String) item); } else if (item instanceof Integer) { - this.onIntegerAction.accept((Integer) item); + builder.onIntegerAction.accept((Integer) item); } else if (item instanceof Double) { - this.onFloatAction.accept((Double) item); + builder.onFloatAction.accept((Double) item); } else if (item instanceof Long) { - this.onLongAction.accept((Long) item); + builder.onLongAction.accept((Long) item); } else if (item instanceof byte[]) { - this.onBlobAction.accept((byte[]) item); + builder.onBlobAction.accept((byte[]) item); } - this.forAllAction.accept(item); + builder.forAllAction.accept(item); } unfinishedRow = false; //Wrap column index back around if we've reached the end of the row @@ -480,18 +461,18 @@ public class SQLiteTableReader implements AutoCloseable { * directory. * * @param file AbstractFile from the data source - * @param id The input files id value + * @param id The input files id value * * @return The path of the file on disk * * @throws IOException Exception writing file contents * @throws NoCurrentCaseException Current case closed during file copying */ - private String copyFileToTempDirectory(AbstractFile file, long id) + private String copyFileToTempDirectory(AbstractFile file, long fileId) throws IOException, NoCurrentCaseException { String localDiskPath = Case.getCurrentCaseThrows().getTempDirectory() - + File.separator + id + file.getName(); + + File.separator + fileId + file.getName(); File localDatabaseFile = new File(localDiskPath); if (!localDatabaseFile.exists()) { ContentUtils.writeToFile(file, localDatabaseFile); @@ -568,12 +549,12 @@ public class SQLiteTableReader implements AutoCloseable { * @throws Throwable */ @Override - public void finalize() throws Throwable { - super.finalize(); + protected void finalize() throws Throwable { try { close(); } catch (SQLiteTableReaderException ex) { //Do nothing, we tried out best to close the connection. } + super.finalize(); } } diff --git a/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReaderException.java b/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReaderException.java index 70ef801673..63f907cc36 100755 --- a/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReaderException.java +++ b/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReaderException.java @@ -27,18 +27,18 @@ public class SQLiteTableReaderException extends Exception { * Accepts both a message and a parent exception. * * @param msg Message detailing the cause - * @param ex Parent exception + * @param parentEx Parent exception */ - public SQLiteTableReaderException(String msg, Throwable ex) { - super(msg, ex); + public SQLiteTableReaderException(String msg, Throwable parentEx) { + super(msg, parentEx); } /** * Accepts only a parent exception. * - * @param ex Parent exception + * @param parentEx Parent exception */ - public SQLiteTableReaderException(Throwable ex) { - super(ex); + public SQLiteTableReaderException(Throwable parentEx) { + super(parentEx); } } From d8e65c70e79cad8911b00dc8677e82bf6c17f753 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Mon, 12 Nov 2018 09:07:29 -0500 Subject: [PATCH 099/129] Did codacy suggestions --- .../autopsy/coreutils/SQLiteTableReader.java | 163 ++++++++---------- .../coreutils/SQLiteTableReaderException.java | 12 +- 2 files changed, 78 insertions(+), 97 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReader.java b/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReader.java index 94306fa258..44a9696eae 100755 --- a/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReader.java +++ b/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReader.java @@ -41,30 +41,30 @@ import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.TskCoreException; /** - * Reads row by row through SQLite tables and performs user-defined actions on the row values. - * Table values are processed by data type. Users configure these actions for certain data types - * in the Builder. Example usage: + * Reads row by row through SQLite tables and performs user-defined actions on + * the row values. Table values are processed by data type. Users configure + * these actions for certain data types in the Builder. Example usage: + * + * SQLiteTableReader reader = new SQLiteTableReader.Builder(file) + * .onInteger((i) + * -> { System.out.println(i); }) + * .build(); * - * SQLiteTableReader reader = new SQLiteTableReader.Builder(file) - * .onInteger((i) -> { - * System.out.println(i); - * }).build(); - * reader.read(tableName); + * reader.read(tableName); + * + * or + * + * SQLiteTableReader reader = new SQLiteTableReader.Builder(file) + * .onInteger(new Consumer() { + * @Override public void accept(Integer i) { + * System.out.println(i); + * } + * }).build(); * - * or - * - * SQLiteTableReader reader = new SQLiteTableReader.Builder(file) - * .onInteger(new Consumer() { - * @Override - * public void accept(Integer i) { - * System.out.println(i); - * } - * }).build(); - * reader.reader(tableName); - * - * Invocation of read(String tableName) causes that table name to be processed row by row. - * When an Integer is encountered, its value will be passed to the Consumer that - * was defined above. + * reader.reader(tableName); + * + * Invocation of read(String tableName) reads row by row. When an Integer is + * encountered, its value will be passed to the Consumer that was defined above. */ public class SQLiteTableReader implements AutoCloseable { @@ -74,14 +74,18 @@ public class SQLiteTableReader implements AutoCloseable { public static class Builder { private final AbstractFile file; + private Consumer onColumnNameAction; - private Consumer onStringAction; private Consumer onLongAction; private Consumer onIntegerAction; private Consumer onFloatAction; private Consumer onBlobAction; private Consumer forAllAction; + + static Consumer doNothing() { + return NOOP -> {}; + } /** * Creates a Builder for this abstract file. @@ -90,6 +94,14 @@ public class SQLiteTableReader implements AutoCloseable { */ public Builder(AbstractFile file) { this.file = file; + + this.onColumnNameAction = Builder.doNothing(); + this.onStringAction = Builder.doNothing(); + this.onLongAction = Builder.doNothing(); + this.onIntegerAction = Builder.doNothing(); + this.onFloatAction = Builder.doNothing(); + this.onBlobAction = Builder.doNothing(); + this.forAllAction = Builder.doNothing(); } /** @@ -196,61 +208,32 @@ public class SQLiteTableReader implements AutoCloseable { } private final AbstractFile file; + private final Builder builder; + + private final String SELECT_ALL_QUERY = "SELECT * FROM \"%s\""; private Connection conn; private PreparedStatement statement; private ResultSet queryResults; + private ResultSetMetaData currentMetadata; - private final Consumer onColumnNameAction; - private final Consumer onStringAction; - private final Consumer onLongAction; - private final Consumer onIntegerAction; - private final Consumer onFloatAction; - private final Consumer onBlobAction; - private final Consumer forAllAction; - - //Iteration state variables + //Iteration state private int currRowColumnIndex; private int columnNameIndex; private int totalColumnCount; private boolean unfinishedRow; - private ResultSetMetaData currentMetadata; - private boolean liveResultSet; private String prevTableName; /** - * Assigns references to each action based on the Builder configuration. + * Holds reference to the builder instance so that we can use its actions + * during iteration. */ private SQLiteTableReader(Builder builder) { - - this.onColumnNameAction = nonNullValue(builder.onColumnNameAction); - this.onStringAction = nonNullValue(builder.onStringAction); - this.onIntegerAction = nonNullValue(builder.onIntegerAction); - this.onLongAction = nonNullValue(builder.onLongAction); - this.onFloatAction = nonNullValue(builder.onFloatAction); - this.onBlobAction = nonNullValue(builder.onBlobAction); - this.forAllAction = nonNullValue(builder.forAllAction); - + this.builder = builder; this.file = builder.file; } - /** - * Ensures the action is null safe. If action is left null, then during - * iteration null checks would be necessary. To mitigate against that, no-op - * lambdas are substituted for null values. - * - * @param Generic type of consumer - * @param action Consumer for generic type, supplied by Builder. - * - * @return If action is null, then a no-op lambda, if not then the action - * itself. - */ - private Consumer nonNullValue(Consumer action) { - return (Objects.nonNull(action)) ? action : NO_OP -> { - }; - } - /** * Fetches all table names from the database. * @@ -306,8 +289,7 @@ public class SQLiteTableReader implements AutoCloseable { public int getColumnCount(String tableName) throws SQLiteTableReaderException { ensureOpen(); try (ResultSet columnCount = conn.createStatement() - .executeQuery("SELECT * FROM " - + "\"" + tableName + "\"")) { + .executeQuery(String.format(SELECT_ALL_QUERY, tableName))) { return columnCount.getMetaData().getColumnCount(); } catch (SQLException ex) { throw new SQLiteTableReaderException(ex); @@ -325,23 +307,24 @@ public class SQLiteTableReader implements AutoCloseable { * @throws SQLiteTableReaderException */ public void read(String tableName) throws SQLiteTableReaderException { - readHelper("SELECT * FROM \"" + tableName + "\"", () -> false); + readHelper(String.format(SELECT_ALL_QUERY, tableName), () -> false); } /** * Reads column names and values from the table. Only actions that were - * configured in the Builder will be invoked during iteration. Iteration will stop - * when the table read has completed or an exception was encountered. + * configured in the Builder will be invoked during iteration. Iteration + * will stop when the table read has completed or an exception was + * encountered. * * @param tableName Source table to perform a read - * @param limit Number of rows to read from the table - * @param offset Starting row to read from in the table + * @param limit Number of rows to read from the table + * @param offset Starting row to read from in the table * * @throws SQLiteTableReaderException * */ public void read(String tableName, int limit, int offset) throws SQLiteTableReaderException { - readHelper("SELECT * FROM \"" + tableName + "\" LIMIT " + limit + readHelper(String.format(SELECT_ALL_QUERY, tableName)+ " LIMIT " + limit + " OFFSET " + offset, () -> false); } @@ -356,18 +339,16 @@ public class SQLiteTableReader implements AutoCloseable { * */ public void read(String tableName, BooleanSupplier condition) throws SQLiteTableReaderException { - if (Objects.nonNull(prevTableName) && prevTableName.equals(tableName)) { - readHelper("SELECT * FROM \"" + tableName + "\"", condition); - } else { + if (Objects.isNull(prevTableName) || !prevTableName.equals(tableName)) { prevTableName = tableName; - closeTableResources(); - readHelper("SELECT * FROM \"" + tableName + "\"", condition); + closeTableResources(); } + readHelper(String.format(SELECT_ALL_QUERY, tableName), condition); } /** - * Performs the result set iteration and is responsible for maintaining state - * of the read over multiple invocations. + * Performs the result set iteration and is responsible for maintaining + * state of the read over multiple invocations. * * @throws SQLiteTableReaderException */ @@ -379,35 +360,35 @@ public class SQLiteTableReader implements AutoCloseable { } //Process column names before reading the database table values - while(columnNameIndex < totalColumnCount) { + while (columnNameIndex < totalColumnCount) { if (condition.getAsBoolean()) { - return; + return; } - this.onColumnNameAction.accept(currentMetadata + builder.onColumnNameAction.accept(currentMetadata .getColumnName(++columnNameIndex)); } while (unfinishedRow || queryResults.next()) { - while(currRowColumnIndex < totalColumnCount) { + while (currRowColumnIndex < totalColumnCount) { if (condition.getAsBoolean()) { unfinishedRow = true; return; } - + Object item = queryResults.getObject(++currRowColumnIndex); if (item instanceof String) { - this.onStringAction.accept((String) item); + builder.onStringAction.accept((String) item); } else if (item instanceof Integer) { - this.onIntegerAction.accept((Integer) item); + builder.onIntegerAction.accept((Integer) item); } else if (item instanceof Double) { - this.onFloatAction.accept((Double) item); + builder.onFloatAction.accept((Double) item); } else if (item instanceof Long) { - this.onLongAction.accept((Long) item); + builder.onLongAction.accept((Long) item); } else if (item instanceof byte[]) { - this.onBlobAction.accept((byte[]) item); + builder.onBlobAction.accept((byte[]) item); } - this.forAllAction.accept(item); + builder.forAllAction.accept(item); } unfinishedRow = false; //Wrap column index back around if we've reached the end of the row @@ -480,18 +461,18 @@ public class SQLiteTableReader implements AutoCloseable { * directory. * * @param file AbstractFile from the data source - * @param id The input files id value + * @param id The input files id value * * @return The path of the file on disk * * @throws IOException Exception writing file contents * @throws NoCurrentCaseException Current case closed during file copying */ - private String copyFileToTempDirectory(AbstractFile file, long id) + private String copyFileToTempDirectory(AbstractFile file, long fileId) throws IOException, NoCurrentCaseException { String localDiskPath = Case.getCurrentCaseThrows().getTempDirectory() - + File.separator + id + file.getName(); + + File.separator + fileId + file.getName(); File localDatabaseFile = new File(localDiskPath); if (!localDatabaseFile.exists()) { ContentUtils.writeToFile(file, localDatabaseFile); @@ -568,12 +549,12 @@ public class SQLiteTableReader implements AutoCloseable { * @throws Throwable */ @Override - public void finalize() throws Throwable { - super.finalize(); + protected void finalize() throws Throwable { try { close(); } catch (SQLiteTableReaderException ex) { //Do nothing, we tried out best to close the connection. } + super.finalize(); } } diff --git a/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReaderException.java b/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReaderException.java index 70ef801673..63f907cc36 100755 --- a/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReaderException.java +++ b/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReaderException.java @@ -27,18 +27,18 @@ public class SQLiteTableReaderException extends Exception { * Accepts both a message and a parent exception. * * @param msg Message detailing the cause - * @param ex Parent exception + * @param parentEx Parent exception */ - public SQLiteTableReaderException(String msg, Throwable ex) { - super(msg, ex); + public SQLiteTableReaderException(String msg, Throwable parentEx) { + super(msg, parentEx); } /** * Accepts only a parent exception. * - * @param ex Parent exception + * @param parentEx Parent exception */ - public SQLiteTableReaderException(Throwable ex) { - super(ex); + public SQLiteTableReaderException(Throwable parentEx) { + super(parentEx); } } From ad9cc4c0266cbb539a3d7b75bc79e0e5e24447f9 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Mon, 12 Nov 2018 09:12:55 -0500 Subject: [PATCH 100/129] Coday changes --- .../sleuthkit/autopsy/keywordsearch/SqliteTextExtractor.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SqliteTextExtractor.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SqliteTextExtractor.java index 455a22e367..cf252111d9 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SqliteTextExtractor.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SqliteTextExtractor.java @@ -25,7 +25,6 @@ import java.util.Iterator; import java.util.Objects; import java.util.function.Consumer; import java.util.logging.Level; -import org.openide.util.Exceptions; import org.sleuthkit.autopsy.coreutils.SQLiteTableReaderException; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.SQLiteTableReader; @@ -147,10 +146,10 @@ class SqliteTextExtractor extends ContentTextExtractor { private int rowIndex = 0; @Override - public void accept(Object t) { + public void accept(Object value) { rowIndex++; //Ignore blobs - String objectStr = (t instanceof byte[]) ? "" : Objects.toString(t, ""); + String objectStr = (value instanceof byte[]) ? "" : Objects.toString(value, ""); if (rowIndex > 1 && rowIndex < totalColumns) { objectStr += " "; From ac54dee5a2d843cc24088016b009a3625d1ccae3 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Mon, 12 Nov 2018 13:09:29 -0500 Subject: [PATCH 101/129] Updated method header --- .../modules/embeddedfileextractor/SevenZipExtractor.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java index 45963d7cdd..c1aec6c81b 100644 --- a/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java +++ b/Core/src/org/sleuthkit/autopsy/modules/embeddedfileextractor/SevenZipExtractor.java @@ -796,11 +796,10 @@ class SevenZipExtractor { /** * UnpackStream used by the SevenZipBindings to do archive extraction. A memory - * leak exists in the SevenZip library that will not let go of the Streams until - * the entire archive extraction is complete. So, to compensate this UnpackStream - * uses a MutableEncodedFileOutputStream, which supports setting a new disk - * location for the contents to be written to (as opposed to creating a new steam - * for each file). + * leak exists in the SevenZip library that will not let go of the streams until + * the entire archive extraction is complete. Instead of creating a new UnpackStream + * for every file in the archive, instead we just rebase our EncodedFileOutputStream pointer + * for every new file. */ private final static class UnpackStream implements ISequentialOutStream { From a1d0519b6ab1d61dfd14f93528c01176a5da9ac3 Mon Sep 17 00:00:00 2001 From: Raman Date: Tue, 13 Nov 2018 17:33:47 -0500 Subject: [PATCH 102/129] 1110: Remove pre-populating phase. --- .../imagegallery/ImageGalleryController.java | 111 ++++++++++++------ .../imagegallery/ImageGalleryModule.java | 91 +++++++++----- .../ImageGalleryTopComponent.java | 10 +- .../imagegallery/actions/OpenAction.java | 46 ++++++-- .../imagegallery/datamodel/DrawableDB.java | 9 +- .../imagegallery/gui/DataSourceCell.java | 25 +++- .../autopsy/imagegallery/gui/Toolbar.java | 4 +- 7 files changed, 205 insertions(+), 91 deletions(-) diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java index 646affbad7..a0300e72c5 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java @@ -22,6 +22,7 @@ import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.ThreadFactoryBuilder; import java.beans.PropertyChangeListener; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -67,8 +68,8 @@ import org.sleuthkit.autopsy.imagegallery.datamodel.HashSetManager; import org.sleuthkit.autopsy.imagegallery.datamodel.grouping.GroupManager; import org.sleuthkit.autopsy.imagegallery.datamodel.grouping.GroupViewState; import org.sleuthkit.autopsy.ingest.IngestManager; -import org.sleuthkit.autopsy.modules.filetypeid.FileTypeDetector; import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.DataSource; import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.SleuthkitCase.CaseDbTransaction; @@ -382,6 +383,51 @@ public final class ImageGalleryController { } + /** + * Returns a map of all data source object ids, along with + * their DB build status. + * + * This includes any data sources already in the table, + * and any data sources that might have been added to the + * case, but are not in the datasources table. + * + * @return map of data source object ids and their Db build status. + */ + public Map getAllDataSourcesDrawableDBStatus() { + + Map dataSourceStatusMap = new HashMap<>(); + + // no current case open to check + if ((null == getDatabase()) || (null == getSleuthKitCase())) { + return dataSourceStatusMap; + } + + try { + Map knownDataSourceIds = getDatabase().getDataSourceDbBuildStatus(); + + List dataSources = getSleuthKitCase().getDataSources(); + Set caseDataSourceIds = new HashSet<>(); + dataSources.stream().map(DataSource::getId).forEach(caseDataSourceIds::add); + + // collect all data sources already in the table + knownDataSourceIds.entrySet().stream().forEach((Map.Entry t) -> { + dataSourceStatusMap.put(t.getKey(), t.getValue()); + }); + + // collect any new data sources in the case. + caseDataSourceIds.forEach((Long id) -> { + if (!knownDataSourceIds.containsKey(id)) { + dataSourceStatusMap.put(id, DrawableDbBuildStatusEnum.UNKNOWN); + } + }); + + return dataSourceStatusMap; + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, "Image Gallery failed to get data source DB status.", ex); + return dataSourceStatusMap; + } + } + public boolean hasTooManyFiles(DataSource datasource) throws TskCoreException { String whereClause = (datasource == null) ? "1 = 1" @@ -390,6 +436,27 @@ public final class ImageGalleryController { return sleuthKitCase.countFilesWhere(whereClause) > FILE_LIMIT; } + + /** + * Checks if the given data source has any files with no mimetype + * + * @param datasource + * @return true if the datasource has any files with no mime type + * @throws TskCoreException + */ + public boolean hasFilesWithNoMimetype(Content datasource) throws TskCoreException { + + // There are some special files/attributes in the root folder, like $BadClus:$Bad and $Security:$SDS + // that do not go through any ingest modules, and hence do not have any assigned mimetype + String whereClause = "data_source_obj_id = " + datasource.getId() + + " AND ( meta_type = " + TskData.TSK_FS_META_TYPE_ENUM.TSK_FS_META_TYPE_REG.getValue() + ")" + + " AND ( mime_type IS NULL )" + + " AND ( meta_addr >= 32 ) " + + " AND ( parent_path <> '/' )" + + " AND ( name NOT like '$%:%' )"; + + return sleuthKitCase.countFilesWhere(whereClause) > 0; + } synchronized private void shutDownDBExecutor() { if (dbExecutor != null) { @@ -763,9 +830,13 @@ public final class ImageGalleryController { } } progressHandle.finish(); - if (taskCompletionStatus) { - taskDB.insertOrUpdateDataSource(dataSourceObjId, DrawableDB.DrawableDbBuildStatusEnum.COMPLETE); - } + + DrawableDB.DrawableDbBuildStatusEnum datasourceDrawableDBStatus = + (taskCompletionStatus) ? + DrawableDB.DrawableDbBuildStatusEnum.COMPLETE : + DrawableDB.DrawableDbBuildStatusEnum.DEFAULT; + taskDB.insertOrUpdateDataSource(dataSourceObjId, datasourceDrawableDBStatus); + updateMessage(""); updateProgress(-1.0); } @@ -832,36 +903,4 @@ public final class ImageGalleryController { } } - /** - * Copy files from a newly added data source into the DB. Get all "drawable" - * files, based on extension and mime-type. After ingest we use file type id - * module and if necessary jpeg/png signature matching to add/remove files - */ - @NbBundle.Messages({"PrePopulateDataSourceFiles.committingDb.status=committing image/video database"}) - static class PrePopulateDataSourceFiles extends BulkTransferTask { - - /** - * @param dataSourceObjId The object ID of the DataSource that is being - * pre-populated into the DrawableDB. - * @param controller The controller for this task. - */ - PrePopulateDataSourceFiles(long dataSourceObjId, ImageGalleryController controller) { - super(dataSourceObjId, controller); - } - - @Override - protected void cleanup(boolean success) { - } - - @Override - void processFile(final AbstractFile f, DrawableDB.DrawableTransaction tr, CaseDbTransaction caseDBTransaction) { - taskDB.insertBasicFileData(DrawableFile.create(f, false, false), tr, caseDBTransaction); - } - - @Override - @NbBundle.Messages({"PrePopulateDataSourceFiles.prepopulatingDb.status=prepopulating image/video database",}) - ProgressHandle getInitialProgressHandle() { - return ProgressHandle.createHandle(Bundle.PrePopulateDataSourceFiles_prepopulatingDb_status(), this); - } - } } diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryModule.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryModule.java index 228b3765e5..88a7d38d70 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryModule.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryModule.java @@ -43,11 +43,14 @@ import org.sleuthkit.autopsy.ingest.IngestManager.IngestJobEvent; import static org.sleuthkit.autopsy.ingest.IngestManager.IngestModuleEvent.DATA_ADDED; import static org.sleuthkit.autopsy.ingest.IngestManager.IngestModuleEvent.FILE_DONE; import org.sleuthkit.autopsy.ingest.ModuleDataEvent; +import org.sleuthkit.autopsy.ingest.events.DataSourceAnalysisCompletedEvent; +import org.sleuthkit.autopsy.ingest.events.DataSourceAnalysisStartedEvent; import org.sleuthkit.autopsy.modules.filetypeid.FileTypeDetector; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE; import org.sleuthkit.datamodel.Content; +import org.sleuthkit.datamodel.DataSource; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskData; @@ -281,8 +284,8 @@ public class ImageGalleryModule { //For a data source added on the local node, prepopulate all file data to drawable database if (((AutopsyEvent) evt).getSourceType() == AutopsyEvent.SourceType.LOCAL) { Content newDataSource = (Content) evt.getNewValue(); - if (con.isListeningEnabled()) { - con.queueDBTask(new ImageGalleryController.PrePopulateDataSourceFiles(newDataSource.getId(), controller)); + if (con.isListeningEnabled()) { + controller.getDatabase().insertOrUpdateDataSource(newDataSource.getId(), DrawableDB.DrawableDbBuildStatusEnum.DEFAULT); } } break; @@ -326,38 +329,66 @@ public class ImageGalleryModule { @Override public void propertyChange(PropertyChangeEvent evt) { IngestJobEvent eventType = IngestJobEvent.valueOf(evt.getPropertyName()); - if (eventType != IngestJobEvent.DATA_SOURCE_ANALYSIS_COMPLETED - || ((AutopsyEvent) evt).getSourceType() != AutopsyEvent.SourceType.REMOTE) { - return; - } - // A remote node added a new data source and just finished ingest on it. - //drawable db is stale, and if ImageGallery is open, ask user what to do - + try { - ImageGalleryController con = getController(); - con.setStale(true); - if (con.isListeningEnabled() && ImageGalleryTopComponent.isImageGalleryOpen()) { - SwingUtilities.invokeLater(() -> { - int showAnswer = JOptionPane.showConfirmDialog(ImageGalleryTopComponent.getTopComponent(), - Bundle.ImageGalleryController_dataSourceAnalyzed_confDlg_msg(), - Bundle.ImageGalleryController_dataSourceAnalyzed_confDlg_title(), - JOptionPane.YES_NO_CANCEL_OPTION, JOptionPane.WARNING_MESSAGE); - - switch (showAnswer) { - case JOptionPane.YES_OPTION: - con.rebuildDB(); - break; - case JOptionPane.NO_OPTION: - case JOptionPane.CANCEL_OPTION: - default: - break; //do nothing + ImageGalleryController controller = getController(); + + if (eventType == IngestJobEvent.DATA_SOURCE_ANALYSIS_STARTED) { + + if (((AutopsyEvent) evt).getSourceType() == AutopsyEvent.SourceType.LOCAL) { + if (controller.isListeningEnabled()) { + DataSourceAnalysisStartedEvent dataSourceAnalysisStartedEvent = (DataSourceAnalysisStartedEvent) evt; + Content dataSource = dataSourceAnalysisStartedEvent.getDataSource(); + + controller.getDatabase().insertOrUpdateDataSource(dataSource.getId(), DrawableDB.DrawableDbBuildStatusEnum.IN_PROGRESS); } - }); + } + } else if (eventType == IngestJobEvent.DATA_SOURCE_ANALYSIS_COMPLETED) { + + if (((AutopsyEvent) evt).getSourceType() == AutopsyEvent.SourceType.LOCAL) { + if (controller.isListeningEnabled()) { + DataSourceAnalysisCompletedEvent dataSourceAnalysisCompletedEvent = (DataSourceAnalysisCompletedEvent) evt; + Content dataSource = dataSourceAnalysisCompletedEvent.getDataSource(); + + DrawableDB.DrawableDbBuildStatusEnum datasourceDrawableDBStatus = + controller.hasFilesWithNoMimetype(dataSource) ? + DrawableDB.DrawableDbBuildStatusEnum.DEFAULT : + DrawableDB.DrawableDbBuildStatusEnum.COMPLETE; + + controller.getDatabase().insertOrUpdateDataSource(dataSource.getId(), datasourceDrawableDBStatus); + } + return; + } + + if (((AutopsyEvent) evt).getSourceType() == AutopsyEvent.SourceType.REMOTE) { + // A remote node added a new data source and just finished ingest on it. + //drawable db is stale, and if ImageGallery is open, ask user what to do + controller.setStale(true); + if (controller.isListeningEnabled() && ImageGalleryTopComponent.isImageGalleryOpen()) { + SwingUtilities.invokeLater(() -> { + int showAnswer = JOptionPane.showConfirmDialog(ImageGalleryTopComponent.getTopComponent(), + Bundle.ImageGalleryController_dataSourceAnalyzed_confDlg_msg(), + Bundle.ImageGalleryController_dataSourceAnalyzed_confDlg_title(), + JOptionPane.YES_NO_CANCEL_OPTION, JOptionPane.WARNING_MESSAGE); + + switch (showAnswer) { + case JOptionPane.YES_OPTION: + controller.rebuildDB(); + break; + case JOptionPane.NO_OPTION: + case JOptionPane.CANCEL_OPTION: + default: + break; //do nothing + } + }); + } + } } - } catch (NoCurrentCaseException ex) { - logger.log(Level.SEVERE, "Attempted to access ImageGallery with no case open.", ex); //NON-NLS + } + catch (NoCurrentCaseException ex) { + logger.log(Level.SEVERE, "Attempted to access ImageGallery with no case open.", ex); //NON-NLS } catch (TskCoreException ex) { - logger.log(Level.SEVERE, "Error getting ImageGalleryController.", ex); //NON-NLS + logger.log(Level.SEVERE, "Error getting ImageGalleryController.", ex); //NON-NLS } } } diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryTopComponent.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryTopComponent.java index e51f27cdef..e9753daf5a 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryTopComponent.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryTopComponent.java @@ -18,13 +18,10 @@ */ package org.sleuthkit.autopsy.imagegallery; -import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Optional; -import java.util.function.Consumer; import java.util.logging.Level; import java.util.stream.Collectors; import javafx.application.Platform; @@ -55,10 +52,8 @@ import javafx.stage.Modality; import javax.swing.SwingUtilities; import static org.apache.commons.collections4.CollectionUtils.isNotEmpty; import static org.apache.commons.lang3.ObjectUtils.notEqual; -import org.apache.commons.lang3.StringUtils; import org.openide.explorer.ExplorerManager; import org.openide.explorer.ExplorerUtils; -import org.openide.util.Exceptions; import org.openide.util.Lookup; import org.openide.util.NbBundle; import org.openide.util.NbBundle.Messages; @@ -70,6 +65,7 @@ import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.ThreadConfined; import org.sleuthkit.autopsy.imagegallery.datamodel.DrawableAttribute; +import org.sleuthkit.autopsy.imagegallery.datamodel.DrawableDB; import org.sleuthkit.autopsy.imagegallery.datamodel.grouping.GroupManager; import org.sleuthkit.autopsy.imagegallery.gui.DataSourceCell; import org.sleuthkit.autopsy.imagegallery.gui.GuiUtils; @@ -232,8 +228,8 @@ public final class ImageGalleryTopComponent extends TopComponent implements Expl @SuppressWarnings(value = "unchecked") ComboBox> comboBox = (ComboBox>) datasourceDialog.getDialogPane().lookup(".combo-box"); //set custom cell renderer - comboBox.setCellFactory((ListView> param) -> new DataSourceCell(dataSourcesTooManyFiles)); - comboBox.setButtonCell(new DataSourceCell(dataSourcesTooManyFiles)); + comboBox.setCellFactory((ListView> param) -> new DataSourceCell(dataSourcesTooManyFiles, controller.getAllDataSourcesDrawableDBStatus())); + comboBox.setButtonCell(new DataSourceCell(dataSourcesTooManyFiles, controller.getAllDataSourcesDrawableDBStatus())); DataSource dataSource = datasourceDialog.showAndWait().orElse(Optional.empty()).orElse(null); try { diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/actions/OpenAction.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/actions/OpenAction.java index 81431b771e..230dd12867 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/actions/OpenAction.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/actions/OpenAction.java @@ -19,11 +19,10 @@ package org.sleuthkit.autopsy.imagegallery.actions; import com.google.common.util.concurrent.ListenableFuture; -import static com.google.common.util.concurrent.MoreExecutors.listeningDecorator; import java.awt.Component; import java.beans.PropertyChangeEvent; import java.beans.PropertyChangeListener; -import static java.util.concurrent.Executors.newSingleThreadExecutor; +import java.util.Map; import java.util.logging.Level; import javafx.application.Platform; import javafx.scene.control.Alert; @@ -32,7 +31,6 @@ import javafx.scene.control.CheckBox; import javafx.scene.control.Label; import javafx.scene.layout.VBox; import javafx.stage.Modality; -import javax.annotation.concurrent.ThreadSafe; import javax.swing.ImageIcon; import javax.swing.JButton; import javax.swing.JMenuItem; @@ -50,11 +48,12 @@ import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; import org.sleuthkit.autopsy.core.Installer; import org.sleuthkit.autopsy.core.RuntimeProperties; import org.sleuthkit.autopsy.coreutils.Logger; -import org.sleuthkit.autopsy.coreutils.ThreadConfined; import org.sleuthkit.autopsy.imagegallery.ImageGalleryController; import org.sleuthkit.autopsy.imagegallery.ImageGalleryModule; import org.sleuthkit.autopsy.imagegallery.ImageGalleryPreferences; import org.sleuthkit.autopsy.imagegallery.ImageGalleryTopComponent; +import org.sleuthkit.autopsy.imagegallery.datamodel.DrawableDB; +import org.sleuthkit.autopsy.imagegallery.datamodel.DrawableDB.DrawableDbBuildStatusEnum; import org.sleuthkit.autopsy.imagegallery.gui.GuiUtils; import org.sleuthkit.autopsy.imagegallery.utils.TaskUtils; import static org.sleuthkit.autopsy.imagegallery.utils.TaskUtils.addFXCallback; @@ -70,6 +69,7 @@ import org.sleuthkit.datamodel.TskCoreException; "OpenAction.stale.confDlg.msg=The image / video database may be out of date. " + "Do you want to update and listen for further ingest results?\n" + "Choosing 'yes' will update the database and enable listening to future ingests.", + "OpenAction.notAnalyzedDlg.msg=No image/video files available to display yet. Please run FileType and EXIF ingest modules.", "OpenAction.stale.confDlg.title=Image Gallery"}) public final class OpenAction extends CallableSystemAction { @@ -184,17 +184,43 @@ public final class OpenAction extends CallableSystemAction { } private void checkDBStale(ImageGalleryController controller) { - //check if db is stale on throw away bg thread and then react back on jfx thread. - ListenableFuture staleFuture = TaskUtils.getExecutorForClass(OpenAction.class) - .submit(controller::isDataSourcesTableStale); - addFXCallback(staleFuture, - dbIsStale -> { + + ListenableFuture> dataSourceStatusMapFuture = TaskUtils.getExecutorForClass(OpenAction.class) + .submit(controller::getAllDataSourcesDrawableDBStatus); + + addFXCallback(dataSourceStatusMapFuture, + dataSourceStatusMap -> { + + boolean dbIsStale = false; + for (Map.Entry entry : dataSourceStatusMap.entrySet()) { + DrawableDbBuildStatusEnum status = entry.getValue(); + if (DrawableDbBuildStatusEnum.COMPLETE != status) { + dbIsStale = true; + } + } + //back on fx thread. if (false == dbIsStale) { //drawable db is not stale, just open it openTopComponent(); } else { - //drawable db is stale, ask what to do + + // If there is only one datasource and it's in DEFAULT State - + // ingest modules need to be run on the data source + if (dataSourceStatusMap.size()== 1) { + Map.Entry entry = dataSourceStatusMap.entrySet().iterator().next(); + if (entry.getValue() == DrawableDbBuildStatusEnum.DEFAULT ) { + Alert alert = new Alert(Alert.AlertType.WARNING, Bundle.OpenAction_notAnalyzedDlg_msg(), ButtonType.OK); + alert.setTitle(Bundle.OpenAction_stale_confDlg_title()); + alert.initModality(Modality.APPLICATION_MODAL); + + alert.showAndWait(); + return; + } + } + + //drawable db is stale, + //ask what to do Alert alert = new Alert(Alert.AlertType.WARNING, Bundle.OpenAction_stale_confDlg_msg(), ButtonType.YES, ButtonType.NO, ButtonType.CANCEL); diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java index 8c27f6cce5..dee725db34 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java @@ -174,11 +174,14 @@ public final class DrawableDB { /** * Enum to track Image gallery db rebuild status for a data source + * + * DO NOT add in the middle. */ public enum DrawableDbBuildStatusEnum { - UNKNOWN, /// no known status - IN_PROGRESS, /// drawable db rebuild has been started for the data source - COMPLETE; /// drawable db rebuild is complete for the data source + UNKNOWN, /// drawable db does not have an entry for this data source + IN_PROGRESS, /// drawable db rebuild has been started for the data source + COMPLETE, /// drawable db rebuild is complete for the data source + DEFAULT; /// drawable db needs to be updated for this data source } //////////////general database logic , mostly borrowed from sleuthkitcase diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/DataSourceCell.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/DataSourceCell.java index 08543a42d2..76190ad1ef 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/DataSourceCell.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/DataSourceCell.java @@ -21,6 +21,8 @@ package org.sleuthkit.autopsy.imagegallery.gui; import java.util.Map; import java.util.Optional; import javafx.scene.control.ListCell; +import org.sleuthkit.autopsy.imagegallery.datamodel.DrawableDB; +import org.sleuthkit.autopsy.imagegallery.datamodel.DrawableDB.DrawableDbBuildStatusEnum; import org.sleuthkit.datamodel.DataSource; /** @@ -29,9 +31,12 @@ import org.sleuthkit.datamodel.DataSource; public class DataSourceCell extends ListCell> { private final Map dataSourcesTooManyFiles; + private final Map dataSourcesDrawableDBStatus; - public DataSourceCell(Map dataSourcesViewable) { + public DataSourceCell(Map dataSourcesViewable, Map dataSourcesDrawableDBStatus) { this.dataSourcesTooManyFiles = dataSourcesViewable; + this.dataSourcesDrawableDBStatus = dataSourcesDrawableDBStatus; + } @Override @@ -43,14 +48,28 @@ public class DataSourceCell extends ListCell> { DataSource dataSource = item.orElse(null); String text = (dataSource == null) ? "All" : dataSource.getName() + " (Id: " + dataSource.getId() + ")"; Boolean tooManyFilesInDataSource = dataSourcesTooManyFiles.getOrDefault(dataSource, false); + + DrawableDbBuildStatusEnum dataSourceDBStatus = (dataSource != null) ? + dataSourcesDrawableDBStatus.get(dataSource.getId()) : DrawableDbBuildStatusEnum.UNKNOWN; + + Boolean dataSourceNotAnalyzed = (dataSourceDBStatus == DrawableDbBuildStatusEnum.DEFAULT); if (tooManyFilesInDataSource) { text += " - Too many files"; + } + if (dataSourceNotAnalyzed) { + text += " - Not Analyzed"; + } + + // check if item should be disabled + if (tooManyFilesInDataSource || dataSourceNotAnalyzed) { + setDisable(true); setStyle("-fx-opacity : .5"); - } else { + } + else { setGraphic(null); setStyle("-fx-opacity : 1"); } - setDisable(tooManyFilesInDataSource); + setText(text); } } diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/Toolbar.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/Toolbar.java index bc0f5e5d84..4bcecbbecb 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/Toolbar.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/Toolbar.java @@ -240,8 +240,8 @@ public class Toolbar extends ToolBar { } private void initDataSourceComboBox() { - dataSourceComboBox.setCellFactory(param -> new DataSourceCell(dataSourcesViewable)); - dataSourceComboBox.setButtonCell(new DataSourceCell(dataSourcesViewable)); + dataSourceComboBox.setCellFactory(param -> new DataSourceCell(dataSourcesViewable, controller.getAllDataSourcesDrawableDBStatus())); + dataSourceComboBox.setButtonCell(new DataSourceCell(dataSourcesViewable, controller.getAllDataSourcesDrawableDBStatus())); dataSourceComboBox.setConverter(new StringConverter>() { @Override public String toString(Optional object) { From 503e1e56ef760157fc3ce67743c9f3436d0ecb87 Mon Sep 17 00:00:00 2001 From: Raman Date: Tue, 13 Nov 2018 18:40:40 -0500 Subject: [PATCH 103/129] Manually applied changes to resolve merge conflict - Push isImageGalleryOpen() check on EDT. --- .../imagegallery/ImageGalleryModule.java | 28 ++++++++++--------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryModule.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryModule.java index 8386999c69..2dd4ba2362 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryModule.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryModule.java @@ -364,21 +364,23 @@ public class ImageGalleryModule { // A remote node added a new data source and just finished ingest on it. //drawable db is stale, and if ImageGallery is open, ask user what to do controller.setStale(true); - if (controller.isListeningEnabled() && ImageGalleryTopComponent.isImageGalleryOpen()) { + if (controller.isListeningEnabled()) { SwingUtilities.invokeLater(() -> { - int showAnswer = JOptionPane.showConfirmDialog(ImageGalleryTopComponent.getTopComponent(), - Bundle.ImageGalleryController_dataSourceAnalyzed_confDlg_msg(), - Bundle.ImageGalleryController_dataSourceAnalyzed_confDlg_title(), - JOptionPane.YES_NO_CANCEL_OPTION, JOptionPane.WARNING_MESSAGE); + if (ImageGalleryTopComponent.isImageGalleryOpen()) { + int showAnswer = JOptionPane.showConfirmDialog(ImageGalleryTopComponent.getTopComponent(), + Bundle.ImageGalleryController_dataSourceAnalyzed_confDlg_msg(), + Bundle.ImageGalleryController_dataSourceAnalyzed_confDlg_title(), + JOptionPane.YES_NO_CANCEL_OPTION, JOptionPane.WARNING_MESSAGE); - switch (showAnswer) { - case JOptionPane.YES_OPTION: - controller.rebuildDB(); - break; - case JOptionPane.NO_OPTION: - case JOptionPane.CANCEL_OPTION: - default: - break; //do nothing + switch (showAnswer) { + case JOptionPane.YES_OPTION: + controller.rebuildDB(); + break; + case JOptionPane.NO_OPTION: + case JOptionPane.CANCEL_OPTION: + default: + break; //do nothing + } } }); } From 8d2326b475b00d9c618b4b43e42915eb9ad7798e Mon Sep 17 00:00:00 2001 From: Raman Date: Wed, 14 Nov 2018 10:21:12 -0500 Subject: [PATCH 104/129] Address review comments and Codacy comments in the previous commit. --- .../imagegallery/ImageGalleryController.java | 3 +- .../imagegallery/ImageGalleryModule.java | 1 - .../ImageGalleryTopComponent.java | 1 - .../imagegallery/actions/OpenAction.java | 3 +- .../imagegallery/datamodel/DrawableDB.java | 8 +- .../imagegallery/gui/DataSourceCell.java | 15 +- release/update_autopsy_version.pl | 268 ++++++++++++++++++ release/update_sleuthkit_version.pl | 199 +++++++++++++ 8 files changed, 488 insertions(+), 10 deletions(-) create mode 100755 release/update_autopsy_version.pl create mode 100755 release/update_sleuthkit_version.pl diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java index bc78be5fd7..6881bb2e88 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java @@ -450,7 +450,8 @@ public final class ImageGalleryController { public boolean hasFilesWithNoMimetype(Content datasource) throws TskCoreException { // There are some special files/attributes in the root folder, like $BadClus:$Bad and $Security:$SDS - // that do not go through any ingest modules, and hence do not have any assigned mimetype + // The IngestTasksScheduler does not push them down to the ingest modules, + // and hence they do not have any assigned mimetype String whereClause = "data_source_obj_id = " + datasource.getId() + " AND ( meta_type = " + TskData.TSK_FS_META_TYPE_ENUM.TSK_FS_META_TYPE_REG.getValue() + ")" + " AND ( mime_type IS NULL )" diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryModule.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryModule.java index 2dd4ba2362..530e3eb7d6 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryModule.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryModule.java @@ -50,7 +50,6 @@ import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE; import org.sleuthkit.datamodel.Content; -import org.sleuthkit.datamodel.DataSource; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskData; diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryTopComponent.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryTopComponent.java index e9753daf5a..927dc6b70e 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryTopComponent.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryTopComponent.java @@ -65,7 +65,6 @@ import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.ThreadConfined; import org.sleuthkit.autopsy.imagegallery.datamodel.DrawableAttribute; -import org.sleuthkit.autopsy.imagegallery.datamodel.DrawableDB; import org.sleuthkit.autopsy.imagegallery.datamodel.grouping.GroupManager; import org.sleuthkit.autopsy.imagegallery.gui.DataSourceCell; import org.sleuthkit.autopsy.imagegallery.gui.GuiUtils; diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/actions/OpenAction.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/actions/OpenAction.java index 230dd12867..5a921e6130 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/actions/OpenAction.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/actions/OpenAction.java @@ -69,7 +69,8 @@ import org.sleuthkit.datamodel.TskCoreException; "OpenAction.stale.confDlg.msg=The image / video database may be out of date. " + "Do you want to update and listen for further ingest results?\n" + "Choosing 'yes' will update the database and enable listening to future ingests.", - "OpenAction.notAnalyzedDlg.msg=No image/video files available to display yet. Please run FileType and EXIF ingest modules.", + "OpenAction.notAnalyzedDlg.msg=No image/video files available to display yet.\n" + + "Please run FileType and EXIF ingest modules.", "OpenAction.stale.confDlg.title=Image Gallery"}) public final class OpenAction extends CallableSystemAction { diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java index dee725db34..13eea6d654 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java @@ -178,10 +178,10 @@ public final class DrawableDB { * DO NOT add in the middle. */ public enum DrawableDbBuildStatusEnum { - UNKNOWN, /// drawable db does not have an entry for this data source - IN_PROGRESS, /// drawable db rebuild has been started for the data source - COMPLETE, /// drawable db rebuild is complete for the data source - DEFAULT; /// drawable db needs to be updated for this data source + UNKNOWN, /// no known status + IN_PROGRESS, /// ingest or db rebuild is in progress + COMPLETE, /// All files in the data source have had file type detected + DEFAULT; /// Not all files in the data source have had file type detected } //////////////general database logic , mostly borrowed from sleuthkitcase diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/DataSourceCell.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/DataSourceCell.java index 76190ad1ef..7626c4fd77 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/DataSourceCell.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/gui/DataSourceCell.java @@ -33,8 +33,19 @@ public class DataSourceCell extends ListCell> { private final Map dataSourcesTooManyFiles; private final Map dataSourcesDrawableDBStatus; - public DataSourceCell(Map dataSourcesViewable, Map dataSourcesDrawableDBStatus) { - this.dataSourcesTooManyFiles = dataSourcesViewable; + /** + * + * @param dataSourcesTooManyFiles: a map of too many files indicator for + * each data source. + * Data sources with too many files may substantially slow down + * the system and hence are disabled for selection. + * @param dataSourcesDrawableDBStatus a map of drawable DB status for + * each data sources. + * Data sources in DEFAULT state are not fully analyzed yet and are + * disabled for selection. + */ + public DataSourceCell(Map dataSourcesTooManyFiles, Map dataSourcesDrawableDBStatus) { + this.dataSourcesTooManyFiles = dataSourcesTooManyFiles; this.dataSourcesDrawableDBStatus = dataSourcesDrawableDBStatus; } diff --git a/release/update_autopsy_version.pl b/release/update_autopsy_version.pl new file mode 100755 index 0000000000..0a115c8daf --- /dev/null +++ b/release/update_autopsy_version.pl @@ -0,0 +1,268 @@ +#!/usr/bin/perl + +# Updates various Autopsy version numbers + +use strict; +use File::Copy; + +# global variables +my $VER; + + +my $TESTING = 0; +print "TESTING MODE (no commits)\n" if ($TESTING); + + + +sub main { + + # Get the Autopsy version argument + if (scalar (@ARGV) != 1) { + print stderr "Missing release version argument (i.e. 4.9.0)\n"; + exit; + } + + $VER = $ARGV[0]; + die "Invalid version number: $VER (1.2.3 or 1.2.3b1 expected)" unless ($VER =~ /^\d+\.\d+\.\d+(b\d+)?$/); + + + my $AUT_RELNAME = "autopsy-${VER}"; + # Verify the tag doesn't already exist + exec_pipe(*OUT, "git tag | grep \"${AUT_RELNAME}\$\""); + my $foo = read_pipe_line(*OUT); + if ($foo ne "") { + print "Tag ${AUT_RELNAME} already exists\n"; + print "Remove with 'git tag -d ${AUT_RELNAME}'\n"; + die "stopping"; + } + close(OUT); + + # Assume we running out of 'release' folder + chdir ".." or die "Error changing directories to root"; + + + # verify_precheckin(); + + + # Update the version info in that tag + update_project_properties(); + update_doxygen_dev(); + update_doxygen_user(); + + print "Files updated. You need to commit and push them\n"; +} + + + + + +###################################################### +# Utility functions + + +# Function to execute a command and send output to pipe +# returns handle +# exec_pipe(HANDLE, CMD); +sub exec_pipe { + my $handle = shift(@_); + my $cmd = shift(@_); + + die "Can't open pipe for exec_pipe" + unless defined(my $pid = open($handle, '-|')); + + if ($pid) { + return $handle; + } + else { + $| = 1; + exec("$cmd") or die "Can't exec program: $!"; + } +} + +# Read a line of text from an open exec_pipe handle +sub read_pipe_line { + my $handle = shift(@_); + my $out; + + for (my $i = 0; $i < 100; $i++) { + $out = <$handle>; + return $out if (defined $out); + } + return $out; +} + + +# Prompt user for argument and return response +sub prompt_user { + my $q = shift(@_); + print "$q: "; + $| = 1; + $_ = ; + chomp; + return $_; +} + + + +############################################# +# File update methods + + + +# Verify that all files in the current source directory +# are checked in. dies if any are modified. +sub verify_precheckin { + + #system ("git pull"); + + print "Verifying everything is checked in\n"; + exec_pipe(*OUT, "git status -s | grep \"^ M\""); + + my $foo = read_pipe_line(*OUT); + if ($foo ne "") { + print "Files not checked in\n"; + while ($foo ne "") { + print "$foo"; + $foo = read_pipe_line(*OUT); + } + die "stopping" unless ($TESTING); + } + close(OUT); + + print "Verifying everything is pushed\n"; + exec_pipe(*OUT, "git status -sb | grep \"^##\" | grep \"ahead \""); + my $foo = read_pipe_line(*OUT); + if ($foo ne "") { + print "$foo"; + print "Files not pushed to remote\n"; + die "stopping" unless ($TESTING); + } + close(OUT); +} + + + +# update the version in nbproject/project.properties +sub update_project_properties { + + my $orig = "project.properties"; + my $temp = "${orig}-bak"; + + print "Updating the version in ${orig}\n"; + + chdir "nbproject" or die "cannot change into nbproject directory"; + + + open (CONF_IN, "<${orig}") or die "Cannot open ${orig}"; + open (CONF_OUT, ">${temp}") or die "Cannot open ${temp}"; + + my $found = 0; + while () { + if (/^app\.version=/) { + print CONF_OUT "app.version=$VER\n"; + $found++; + } + else { + print CONF_OUT $_; + } + } + close (CONF_IN); + close (CONF_OUT); + + if ($found != 1) { + die "$found (instead of 1) occurrences of app.version found in ${orig}"; + } + + unlink ($orig) or die "Error deleting ${orig}"; + rename ($temp, $orig) or die "Error renaming tmp $orig file"; + system("git add ${orig}") unless ($TESTING); + chdir ".." or die "Error changing directories back to root"; +} + + + +# update the dev docs +sub update_doxygen_dev { + + my $orig = "Doxyfile"; + my $temp = "${orig}-bak"; + + print "Updating the version in ${orig} (Dev)\n"; + + chdir "docs/doxygen" or die "cannot change into docs/doxygen directory"; + + + open (CONF_IN, "<${orig}") or die "Cannot open ${orig}"; + open (CONF_OUT, ">${temp}") or die "Cannot open ${temp}"; + + my $found = 0; + while () { + if (/^PROJECT_NUMBER/) { + print CONF_OUT "PROJECT_NUMBER = ${VER}\n"; + $found++; + } + elsif (/^HTML_OUTPUT/) { + print CONF_OUT "HTML_OUTPUT = api-docs/${VER}/\n"; + $found++; + } + else { + print CONF_OUT $_; + } + } + close (CONF_IN); + close (CONF_OUT); + + if ($found != 2) { + die "$found (instead of 2) occurrences of version found in (DEV) ${orig}"; + } + + unlink ($orig) or die "Error deleting ${orig}"; + rename ($temp, $orig) or die "Error renaming tmp $orig file"; + system("git add ${orig}") unless ($TESTING); + chdir "../.." or die "Error changing directories back to root"; +} + + +# update the user docs +sub update_doxygen_user { + + my $orig = "Doxyfile"; + my $temp = "${orig}-bak"; + + print "Updating the version in ${orig} (User)\n"; + + chdir "docs/doxygen-user" or die "cannot change into docs/doxygen-user directory"; + + + open (CONF_IN, "<${orig}") or die "Cannot open ${orig}"; + open (CONF_OUT, ">${temp}") or die "Cannot open ${temp}"; + + my $found = 0; + while () { + if (/^PROJECT_NUMBER/) { + print CONF_OUT "PROJECT_NUMBER = ${VER}\n"; + $found++; + } + elsif (/^HTML_OUTPUT/) { + print CONF_OUT "HTML_OUTPUT = ${VER}\n"; + $found++; + } + else { + print CONF_OUT $_; + } + } + close (CONF_IN); + close (CONF_OUT); + + if ($found != 2) { + die "$found (instead of 2) occurrences of version found in (USER) ${orig}"; + } + + unlink ($orig) or die "Error deleting ${orig}"; + rename ($temp, $orig) or die "Error renaming tmp $orig file"; + system("git add ${orig}") unless ($TESTING); + chdir "../.." or die "Error changing directories back to root"; +} + + +main(); \ No newline at end of file diff --git a/release/update_sleuthkit_version.pl b/release/update_sleuthkit_version.pl new file mode 100755 index 0000000000..e630e4890b --- /dev/null +++ b/release/update_sleuthkit_version.pl @@ -0,0 +1,199 @@ +#!/usr/bin/perl + +# Updates various TSK version numbers +# use this when the version of TSK that Autopsy depends on changes + +use strict; +use File::Copy; + +# global variables +my $VER; + +my $TESTING = 0; +print "TESTING MODE (no commits)\n" if ($TESTING); + + +sub main { + + # Get the TSK version argument + if (scalar (@ARGV) != 1) { + print stderr "Missing release version argument (i.e. 4.9.0)\n"; + exit; + } + + $VER = $ARGV[0]; + die "Invalid version number: $VER (1.2.3 or 1.2.3b1 expected)" unless ($VER =~ /^\d+\.\d+\.\d+(b\d+)?$/); + + # Assume we running out of 'release' folder + chdir ".." or die "Error changing directories to root"; + + # Update the version info in that tag + update_tsk_version(); + update_core_project_properties(); + update_core_project_xml(); + + print "Files updated. You need to commit and push them\n"; +} + + + + + +###################################################### +# Utility functions + + +# Function to execute a command and send output to pipe +# returns handle +# exec_pipe(HANDLE, CMD); +sub exec_pipe { + my $handle = shift(@_); + my $cmd = shift(@_); + + die "Can't open pipe for exec_pipe" + unless defined(my $pid = open($handle, '-|')); + + if ($pid) { + return $handle; + } + else { + $| = 1; + exec("$cmd") or die "Can't exec program: $!"; + } +} + +# Read a line of text from an open exec_pipe handle +sub read_pipe_line { + my $handle = shift(@_); + my $out; + + for (my $i = 0; $i < 100; $i++) { + $out = <$handle>; + return $out if (defined $out); + } + return $out; +} + + + +############################################# +# File update methods + + + +# update the tskversion.xml +sub update_tsk_version { + + my $orig = "TSKVersion.xml"; + my $temp = "${orig}-bak"; + + print "Updating the version in ${orig}\n"; + + open (CONF_IN, "<${orig}") or die "Cannot open ${orig}"; + open (CONF_OUT, ">${temp}") or die "Cannot open ${temp}"; + + my $found = 0; + while () { + if (/name="TSK_VERSION" value=/) { + print CONF_OUT " \n"; + $found++; + } + else { + print CONF_OUT $_; + } + } + close (CONF_IN); + close (CONF_OUT); + + if ($found != 1) { + die "$found (instead of 1) occurrences of app.version found in ${orig}"; + } + + unlink ($orig) or die "Error deleting ${orig}"; + rename ($temp, $orig) or die "Error renaming tmp $orig file"; + system("git add ${orig}") unless ($TESTING); + +} + + + +sub update_core_project_properties { + + my $orig = "project.properties"; + my $temp = "${orig}-bak"; + + print "Updating the version in ${orig}\n"; + + chdir "Core/nbproject" or die "cannot change into Core/nbproject directory"; + + + open (CONF_IN, "<${orig}") or die "Cannot open ${orig}"; + open (CONF_OUT, ">${temp}") or die "Cannot open ${temp}"; + + my $found = 0; + while () { + if (/^file\.reference\.sleuthkit\-postgresql-/) { + print CONF_OUT "file.reference.sleuthkit-postgresql-${VER}.jar=release/modules/ext/sleuthkit-postgresql-${VER}.jar\n"; + $found++; + } + + else { + print CONF_OUT $_; + } + } + close (CONF_IN); + close (CONF_OUT); + + if ($found != 1) { + die "$found (instead of 1) occurrences of version found in ${orig}"; + } + + unlink ($orig) or die "Error deleting ${orig}"; + rename ($temp, $orig) or die "Error renaming tmp $orig file"; + system("git add ${orig}") unless ($TESTING); + chdir "../.." or die "Error changing directories back to root"; +} + +sub update_core_project_xml { + + my $orig = "project.xml"; + my $temp = "${orig}-bak"; + + print "Updating the version in ${orig}\n"; + + chdir "Core/nbproject" or die "cannot change into Core/nbproject directory"; + + open (CONF_IN, "<${orig}") or die "Cannot open ${orig}"; + open (CONF_OUT, ">${temp}") or die "Cannot open ${temp}"; + + my $found = 0; + while () { + if (/ext\/sleuthkit-postgresql/) { + print CONF_OUT " ext/sleuthkit-postgresql-${VER}.jar\n"; + $found++; + } + elsif (/release\/modules\/ext\/sleuthkit-postgresql/) { + print CONF_OUT " release/modules/ext/sleuthkit-postgresql-${VER}.jar\n"; + $found++; + } + else { + print CONF_OUT $_; + } + } + close (CONF_IN); + close (CONF_OUT); + + if ($found != 2) { + die "$found (instead of 2) occurrences of version found in ${orig}"; + } + + unlink ($orig) or die "Error deleting ${orig}"; + rename ($temp, $orig) or die "Error renaming tmp $orig file"; + system("git add ${orig}") unless ($TESTING); + chdir "../.." or die "Error changing directories back to root"; +} + + + + +main(); \ No newline at end of file From cf2dfe899ee50b0296f94a17e045909c85ce8ba8 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Wed, 14 Nov 2018 12:54:12 -0500 Subject: [PATCH 105/129] Implemented all of Richards code review comments --- .../autopsy/core/UserPreferences.java | 4 +- .../autopsy/corecomponents/Bundle.properties | 2 +- .../corecomponents/ViewPreferencesPanel.form | 10 +- .../corecomponents/ViewPreferencesPanel.java | 29 +- .../datamodel/AbstractAbstractFileNode.java | 561 ++++++++++-------- .../datamodel/AbstractFsContentNode.java | 2 +- .../autopsy/datamodel/FileNodeUtil.java | 244 -------- .../datamodel/SCOAndTranslationTask.java | 41 +- .../datamodel/ToggleableNodeProperty.java | 48 -- 9 files changed, 352 insertions(+), 589 deletions(-) delete mode 100755 Core/src/org/sleuthkit/autopsy/datamodel/FileNodeUtil.java delete mode 100755 Core/src/org/sleuthkit/autopsy/datamodel/ToggleableNodeProperty.java diff --git a/Core/src/org/sleuthkit/autopsy/core/UserPreferences.java b/Core/src/org/sleuthkit/autopsy/core/UserPreferences.java index 3dc230cf21..bc62391076 100644 --- a/Core/src/org/sleuthkit/autopsy/core/UserPreferences.java +++ b/Core/src/org/sleuthkit/autopsy/core/UserPreferences.java @@ -246,11 +246,11 @@ public final class UserPreferences { preferences.putBoolean(HIDE_CENTRAL_REPO_COMMENTS_AND_OCCURRENCES, value); } - public static void setDisplayTranslationFileNames(boolean value) { + public static void setDisplayTranslatedFileNames(boolean value) { preferences.putBoolean(DISPLAY_TRANSLATED_NAMES, value); } - public static boolean displayTranslationFileNames() { + public static boolean displayTranslatedFileNames() { return preferences.getBoolean(DISPLAY_TRANSLATED_NAMES, false); } diff --git a/Core/src/org/sleuthkit/autopsy/corecomponents/Bundle.properties b/Core/src/org/sleuthkit/autopsy/corecomponents/Bundle.properties index 3e8e75dad7..38be797a22 100644 --- a/Core/src/org/sleuthkit/autopsy/corecomponents/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/corecomponents/Bundle.properties @@ -193,4 +193,4 @@ ViewPreferencesPanel.commentsOccurencesColumnsCheckbox.text=C(omments) and O(ccu ViewPreferencesPanel.deletedFilesLimitCheckbox.text=Limit to 10,000 ViewPreferencesPanel.deletedFilesLimitLabel.text=Limit number of deleted files displayed: ViewPreferencesPanel.fileDisplayLabel.text=Translate text in the: -ViewPreferencesPanel.translatedNamesButton.text=Table +ViewPreferencesPanel.translateNamesRadioButton.text=Table diff --git a/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.form b/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.form index 1f76dde4dd..7b8f1e49b2 100755 --- a/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.form +++ b/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.form @@ -129,7 +129,7 @@ - + @@ -183,7 +183,7 @@ - + @@ -364,14 +364,14 @@ - + - + - + diff --git a/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.java b/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.java index af9a940537..6f2a0ab68c 100755 --- a/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.java +++ b/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.java @@ -49,10 +49,7 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { //If there is not Text Translator implementation, then hide these buttons //from the user. TextTranslationService tts = TextTranslationService.getInstance(); - if(!tts.hasProvider()) { - translatedNamesButton.setVisible(false); - fileDisplayLabel.setVisible(false); - } + translateNamesRadioButton.setEnabled(tts.hasProvider()); } @Override @@ -76,7 +73,7 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { commentsOccurencesColumnsCheckbox.setSelected(UserPreferences.hideCentralRepoCommentsAndOccurrences()); deletedFilesLimitCheckbox.setSelected(DeletedFilePreferences.getDefault().getShouldLimitDeletedFiles()); - translatedNamesButton.setSelected(UserPreferences.displayTranslationFileNames()); + translateNamesRadioButton.setSelected(UserPreferences.displayTranslatedFileNames()); // Current Case Settings boolean caseIsOpen = Case.isCaseOpen(); @@ -100,7 +97,7 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { UserPreferences.setHideSlackFilesInViewsTree(viewsHideSlackCheckbox.isSelected()); UserPreferences.setShowOnlyCurrentUserTags(hideOtherUsersTagsCheckbox.isSelected()); UserPreferences.setHideCentralRepoCommentsAndOccurrences(commentsOccurencesColumnsCheckbox.isSelected()); - UserPreferences.setDisplayTranslationFileNames(translatedNamesButton.isSelected()); + UserPreferences.setDisplayTranslatedFileNames(translateNamesRadioButton.isSelected()); storeGroupItemsInTreeByDataSource(); @@ -153,7 +150,7 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { centralRepoLabel = new javax.swing.JLabel(); deletedFilesLimitCheckbox = new javax.swing.JCheckBox(); deletedFilesLimitLabel = new javax.swing.JLabel(); - translatedNamesButton = new javax.swing.JRadioButton(); + translateNamesRadioButton = new javax.swing.JRadioButton(); fileDisplayLabel = new javax.swing.JLabel(); currentCaseSettingsPanel = new javax.swing.JPanel(); groupByDataSourceCheckbox = new javax.swing.JCheckBox(); @@ -257,10 +254,10 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { org.openide.awt.Mnemonics.setLocalizedText(deletedFilesLimitLabel, org.openide.util.NbBundle.getMessage(ViewPreferencesPanel.class, "ViewPreferencesPanel.deletedFilesLimitLabel.text")); // NOI18N - org.openide.awt.Mnemonics.setLocalizedText(translatedNamesButton, org.openide.util.NbBundle.getMessage(ViewPreferencesPanel.class, "ViewPreferencesPanel.translatedNamesButton.text")); // NOI18N - translatedNamesButton.addActionListener(new java.awt.event.ActionListener() { + org.openide.awt.Mnemonics.setLocalizedText(translateNamesRadioButton, org.openide.util.NbBundle.getMessage(ViewPreferencesPanel.class, "ViewPreferencesPanel.translateNamesRadioButton.text")); // NOI18N + translateNamesRadioButton.addActionListener(new java.awt.event.ActionListener() { public void actionPerformed(java.awt.event.ActionEvent evt) { - translatedNamesButtonActionPerformed(evt); + translateNamesRadioButtonActionPerformed(evt); } }); @@ -311,7 +308,7 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { .addComponent(useBestViewerRadioButton) .addComponent(useGMTTimeRadioButton) .addComponent(useLocalTimeRadioButton) - .addComponent(translatedNamesButton))) + .addComponent(translateNamesRadioButton))) .addComponent(selectFileLabel)))) .addGap(0, 10, Short.MAX_VALUE))) .addContainerGap()) @@ -352,7 +349,7 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) .addGroup(globalSettingsPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) .addComponent(hideOtherUsersTagsCheckbox) - .addComponent(translatedNamesButton)) + .addComponent(translateNamesRadioButton)) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) .addComponent(centralRepoLabel) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) @@ -563,13 +560,13 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { } }//GEN-LAST:event_deletedFilesLimitCheckboxActionPerformed - private void translatedNamesButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_translatedNamesButtonActionPerformed + private void translateNamesRadioButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_translateNamesRadioButtonActionPerformed if (immediateUpdates) { - UserPreferences.setDisplayTranslationFileNames(translatedNamesButton.isSelected()); + UserPreferences.setDisplayTranslatedFileNames(translateNamesRadioButton.isSelected()); } else { firePropertyChange(OptionsPanelController.PROP_CHANGED, null, null); } - }//GEN-LAST:event_translatedNamesButtonActionPerformed + }//GEN-LAST:event_translateNamesRadioButtonActionPerformed // Variables declaration - do not modify//GEN-BEGIN:variables @@ -592,7 +589,7 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { private javax.swing.JLabel hideSlackFilesLabel; private javax.swing.JRadioButton keepCurrentViewerRadioButton; private javax.swing.JLabel selectFileLabel; - private javax.swing.JRadioButton translatedNamesButton; + private javax.swing.JRadioButton translateNamesRadioButton; private javax.swing.JRadioButton useBestViewerRadioButton; private javax.swing.JRadioButton useGMTTimeRadioButton; private javax.swing.JRadioButton useLocalTimeRadioButton; diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java index 7e207eb281..3bd51c5f8a 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java @@ -18,12 +18,12 @@ */ package org.sleuthkit.autopsy.datamodel; +import com.google.common.util.concurrent.ThreadFactoryBuilder; import java.beans.PropertyChangeEvent; import java.beans.PropertyChangeListener; import java.lang.ref.WeakReference; -import java.util.ArrayList; +import java.util.LinkedList; import java.util.EnumSet; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -31,6 +31,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.logging.Level; import java.util.stream.Collectors; +import org.apache.commons.io.FilenameUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; import org.openide.nodes.Children; @@ -43,7 +44,14 @@ import org.sleuthkit.autopsy.casemodule.events.CommentChangedEvent; import org.sleuthkit.autopsy.casemodule.events.ContentTagAddedEvent; import org.sleuthkit.autopsy.casemodule.events.ContentTagDeletedEvent; import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeInstance; +import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeNormalizationException; +import org.sleuthkit.autopsy.centralrepository.datamodel.EamArtifactUtil; +import org.sleuthkit.autopsy.centralrepository.datamodel.EamDb; +import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbException; +import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbUtil; import org.sleuthkit.autopsy.core.UserPreferences; +import org.sleuthkit.autopsy.corecomponents.DataResultViewerTable; +import org.sleuthkit.autopsy.corecomponents.DataResultViewerTable.HasCommentStatus; import org.sleuthkit.autopsy.corecomponents.DataResultViewerTable.Score; import org.sleuthkit.autopsy.coreutils.Logger; import static org.sleuthkit.autopsy.datamodel.Bundle.*; @@ -51,10 +59,15 @@ import static org.sleuthkit.autopsy.datamodel.AbstractAbstractFileNode.AbstractF import org.sleuthkit.autopsy.datamodel.SCOAndTranslationTask.SCOResults; import org.sleuthkit.autopsy.ingest.IngestManager; import org.sleuthkit.autopsy.ingest.ModuleContentEvent; +import org.sleuthkit.autopsy.texttranslation.NoServiceProviderException; +import org.sleuthkit.autopsy.texttranslation.TextTranslationService; +import org.sleuthkit.autopsy.texttranslation.TranslationException; import org.sleuthkit.datamodel.AbstractFile; +import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.ContentTag; import org.sleuthkit.datamodel.TskCoreException; +import org.sleuthkit.datamodel.TskData; /** * An abstract node that encapsulates AbstractFile data @@ -70,7 +83,7 @@ public abstract class AbstractAbstractFileNode extends A private static final Set CASE_EVENTS_OF_INTEREST = EnumSet.of(Case.Events.CURRENT_CASE, Case.Events.CONTENT_TAG_ADDED, Case.Events.CONTENT_TAG_DELETED, Case.Events.CR_COMMENT_CHANGED); - private static final ExecutorService pool; + private static final ExecutorService SCOAndTranslationPool; private static final Integer MAX_POOL_SIZE = 10; /** @@ -95,7 +108,8 @@ public abstract class AbstractAbstractFileNode extends A static { //Initialize this pool only once! This will be used by every instance of AAFN //to do their heavy duty SCO column and translation updates. - pool = Executors.newFixedThreadPool(MAX_POOL_SIZE); + SCOAndTranslationPool = Executors.newFixedThreadPool(MAX_POOL_SIZE, + new ThreadFactoryBuilder().setNameFormat("SCOandTranslation-thread-%d").build()); } /** @@ -127,7 +141,7 @@ public abstract class AbstractAbstractFileNode extends A */ enum NodeSpecificEvents { TRANSLATION_AVAILABLE, - DABABASE_CONTENT_AVAILABLE; + SCO_AVAILABLE; } private final PropertyChangeListener pcl = (PropertyChangeEvent evt) -> { @@ -166,96 +180,55 @@ public abstract class AbstractAbstractFileNode extends A // case was closed. Remove listeners so that we don't get called with a stale case handle removeListeners(); } + /* + * No need to do any asynchrony around tag added, deleted or CR + * change events, they are so infrequent and user driven that we can + * just keep a simple blocking approach, where we go out to the + * database ourselves. + */ } else if (eventType.equals(Case.Events.CONTENT_TAG_ADDED.toString())) { ContentTagAddedEvent event = (ContentTagAddedEvent) evt; if (event.getAddedTag().getContent().equals(content)) { - //No need to do any asynchrony around these events, they are so infrequent - //and user driven that we can just keep a simple blocking approach, where we - //go out to the database ourselves! - List tags = FileNodeUtil.getContentTagsFromDatabase(content); - Pair scorePropertyAndDescription - = FileNodeUtil.getScorePropertyAndDescription(content, tags); - CorrelationAttributeInstance attribute = - FileNodeUtil.getCorrelationAttributeInstance(content); - updateProperty( - new ToggleableNodeProperty( - SCORE.toString(), - scorePropertyAndDescription.getRight(), - scorePropertyAndDescription.getLeft()), - new ToggleableNodeProperty( - COMMENT.toString(), - NO_DESCR, - FileNodeUtil.getCommentProperty(tags, attribute)) { - }); + List tags = getContentTagsFromDatabase(); + Pair scorePropAndDescr = getScorePropertyAndDescription(tags); + Score value = scorePropAndDescr.getLeft(); + String descr = scorePropAndDescr.getRight(); + CorrelationAttributeInstance attribute = getCorrelationAttributeInstance(); + updateSheet(new NodeProperty<>(SCORE.toString(),SCORE.toString(),descr,value), + new NodeProperty<>(COMMENT.toString(),COMMENT.toString(),NO_DESCR,getCommentProperty(tags, attribute)) + ); } } else if (eventType.equals(Case.Events.CONTENT_TAG_DELETED.toString())) { ContentTagDeletedEvent event = (ContentTagDeletedEvent) evt; if (event.getDeletedTagInfo().getContentID() == content.getId()) { - //No need to do any asynchrony around these events, they are so infrequent - //and user driven that we can just keep a simple blocking approach, where we - //go out to the database ourselves! - List tags = FileNodeUtil.getContentTagsFromDatabase(content); - Pair scorePropertyAndDescription - = FileNodeUtil.getScorePropertyAndDescription(content, tags); - CorrelationAttributeInstance attribute = - FileNodeUtil.getCorrelationAttributeInstance(content); - updateProperty( - new ToggleableNodeProperty( - SCORE.toString(), - scorePropertyAndDescription.getRight(), - scorePropertyAndDescription.getLeft()), - new ToggleableNodeProperty( - COMMENT.toString(), - NO_DESCR, - FileNodeUtil.getCommentProperty(tags, attribute)) + List tags = getContentTagsFromDatabase(); + Pair scorePropAndDescr = getScorePropertyAndDescription(tags); + Score value = scorePropAndDescr.getLeft(); + String descr = scorePropAndDescr.getRight(); + CorrelationAttributeInstance attribute = getCorrelationAttributeInstance(); + updateSheet(new NodeProperty<>(SCORE.toString(), SCORE.toString(),descr,value), + new NodeProperty<>(COMMENT.toString(), COMMENT.toString(),NO_DESCR,getCommentProperty(tags, attribute)) ); } } else if (eventType.equals(Case.Events.CR_COMMENT_CHANGED.toString())) { CommentChangedEvent event = (CommentChangedEvent) evt; if (event.getContentID() == content.getId()) { - //No need to do any asynchrony around these events, they are so infrequent - //and user driven that we can just keep a simple blocking approach, where we - //go out to the database ourselves! - List tags = FileNodeUtil.getContentTagsFromDatabase(content); - CorrelationAttributeInstance attribute - = FileNodeUtil.getCorrelationAttributeInstance(content); - updateProperty( - new ToggleableNodeProperty( - COMMENT.toString(), - NO_DESCR, - FileNodeUtil.getCommentProperty(tags, attribute))); + List tags = getContentTagsFromDatabase(); + CorrelationAttributeInstance attribute = getCorrelationAttributeInstance(); + updateSheet(new NodeProperty<>(COMMENT.toString(), COMMENT.toString(),NO_DESCR,getCommentProperty(tags, attribute))); } + /* + * Data that was being computed in the background task. Kicked off by a + * call to createSheet(). + */ } else if (eventType.equals(NodeSpecificEvents.TRANSLATION_AVAILABLE.toString())) { - updateProperty( - new ToggleableNodeProperty( - TRANSLATION.toString(), - NO_DESCR, - evt.getNewValue()) { - @Override - public boolean isEnabled() { - return UserPreferences.displayTranslationFileNames(); - } - }); - } else if (eventType.equals(NodeSpecificEvents.DABABASE_CONTENT_AVAILABLE.toString())) { - SCOResults results = (SCOResults) evt.getNewValue(); - updateProperty( - new ToggleableNodeProperty( - SCORE.toString(), - results.getScoreDescription(), - results.getScore()), - new ToggleableNodeProperty( - COMMENT.toString(), - NO_DESCR, - results.getComment()), - new ToggleableNodeProperty( - OCCURRENCES.toString(), - results.getCountDescription(), - results.getCount()) { - @Override - public boolean isEnabled() { - return !UserPreferences.hideCentralRepoCommentsAndOccurrences(); - } - }); + updateSheet(new NodeProperty<>(TRANSLATION.toString(),TRANSLATION.toString(),NO_DESCR,evt.getNewValue())); + } else if (eventType.equals(NodeSpecificEvents.SCO_AVAILABLE.toString())) { + SCOResults res = (SCOResults) evt.getNewValue(); + updateSheet(new NodeProperty<>(SCORE.toString(),SCORE.toString(),res.getScoreDescription(),res.getScore()), + new NodeProperty<>(COMMENT.toString(),COMMENT.toString(),NO_DESCR,res.getComment()), + new NodeProperty<>(OCCURRENCES.toString(),OCCURRENCES.toString(),res.getCountDescription(),res.getCount()) + ); } }; /** @@ -270,51 +243,41 @@ public abstract class AbstractAbstractFileNode extends A /** * Updates the values of the properties in the current property sheet with - * the new properties being passed in! Only if that property exists in the + * the new properties being passed in. Only if that property exists in the * current sheet will it be applied. That way, we allow for subclasses to * add their own (or omit some!) properties and we will not accidentally * disrupt their UI. * * Race condition if not synchronized. Only one update should be applied at - * a time. The timing of currSheetSet.getProperties() could result in - * wrong/stale data being shown! + * a time. * * @param newProps New file property instances to be updated in the current * sheet. */ - private synchronized void updateProperty(ToggleableNodeProperty... newProps) { - + private synchronized void updateSheet(NodeProperty... newProps) { //Refresh ONLY those properties in the sheet currently. Subclasses may have - //only added a subset of our properties or their own props! Let's keep their UI correct. - Sheet currSheet = this.getSheet(); - Sheet.Set currSheetSet = currSheet.get(Sheet.PROPERTIES); - Property[] currProps = currSheetSet.getProperties(); - - Map newPropsMap = new HashMap<>(); - for(ToggleableNodeProperty property: newProps) { - newPropsMap.put(property.getName(), property); - } - - for (int i = 0; i < currProps.length; i++) { - String currentPropertyName = currProps[i].getName(); - if (newPropsMap.containsKey(currentPropertyName) && - newPropsMap.get(currentPropertyName).isEnabled()) { - currProps[i] = newPropsMap.get(currentPropertyName); + //only added a subset of our properties or their own props. Let's keep their UI correct. + Sheet visibleSheet = this.getSheet(); + Sheet.Set visibleSheetSet = visibleSheet.get(Sheet.PROPERTIES); + Property[] visibleProps = visibleSheetSet.getProperties(); + for(NodeProperty newProp: newProps) { + for(int i = 0; i < visibleProps.length; i++) { + if(visibleProps[i].getName().equals(newProp.getName())) { + visibleProps[i] = newProp; + } } } - - currSheetSet.put(currProps); - currSheet.put(currSheetSet); - - //setSheet() will notify Netbeans to update this node in the UI! - this.setSheet(currSheet); + visibleSheetSet.put(visibleProps); + visibleSheet.put(visibleSheetSet); + //setSheet() will notify Netbeans to update this node in the UI. + this.setSheet(visibleSheet); } /* * This is called when the node is first initialized. Any new updates or * changes happen by directly manipulating the sheet. That means we can fire * off background events everytime this method is called and not worry about - * duplicated jobs! + * duplicated jobs. */ @Override protected synchronized Sheet createSheet() { @@ -323,14 +286,17 @@ public abstract class AbstractAbstractFileNode extends A sheet.put(sheetSet); //This will fire off fresh background tasks. - List newProperties = getProperties(); - - //Add only the enabled properties to the sheet! - for (ToggleableNodeProperty property : newProperties) { - if (property.isEnabled()) { - sheetSet.put(property); - } - } + List> newProperties = getProperties(); + newProperties.forEach((property) -> { + sheetSet.put(property); + }); + + /* + * Submit the database queries ASAP. We want updated SCO columns without + * blocking the UI as soon as we can get it! Keep all weak references so + * this task doesn't block the ability of this node to be GC'd. + */ + SCOAndTranslationPool.submit(new SCOAndTranslationTask(new WeakReference<>(this), weakPcl)); return sheet; } @@ -401,9 +367,9 @@ public abstract class AbstractAbstractFileNode extends A } /** - * Creates a list of properties for this file node. ToggleableNodeProperty + * Creates a list of properties for this file node. NodeProperty * is a subclass of NodeProperty, with the added functionality of being able to be - * enabled and disabled. Disabled properties don't get added to the sheet! + * enabled and disabled. Disabled properties don't get added to the sheet. * Additionally, with a return value of a list, any children classes of this * node may reorder or omit any of these properties as they see fit for their use case. * @@ -412,130 +378,44 @@ public abstract class AbstractAbstractFileNode extends A * * @return List of file properties associated with this file node's content. */ - List getProperties() { - List properties = new ArrayList<>(); - properties.add(new ToggleableNodeProperty( - NAME.toString(), - NO_DESCR, - FileNodeUtil.getContentDisplayName(content))); - - //Initialize dummy place holder properties! These obviously do no work - //to get their property values, but at the bottom we kick off a background - //task that promises to update these values. - final String NO_OP = ""; - properties.add(new ToggleableNodeProperty( - TRANSLATION.toString(), - NO_DESCR, - NO_OP) { - @Override - public boolean isEnabled() { - return UserPreferences.displayTranslationFileNames(); + private List> getProperties() { + List> properties = new LinkedList>() {{ + add(new NodeProperty<>(NAME.toString(), NAME.toString(), NO_DESCR, getContentDisplayName(content))); + /* + * Initialize dummy place holder properties for Translation, + * Score, Comment, and Occurrences). At the bottom, we kick off a + * background task that promises to update these values. + */ + if (UserPreferences.displayTranslatedFileNames()) { + add(new NodeProperty<>(TRANSLATION.toString(), TRANSLATION.toString(), NO_DESCR, "")); } - }); - properties.add(new ToggleableNodeProperty( - SCORE.toString(), - NO_DESCR, - NO_OP)); - properties.add(new ToggleableNodeProperty( - COMMENT.toString(), - NO_DESCR, - NO_OP) { - }); - - properties.add(new ToggleableNodeProperty( - OCCURRENCES.toString(), - NO_DESCR, - NO_OP) { - @Override - public boolean isEnabled() { - return !UserPreferences.hideCentralRepoCommentsAndOccurrences(); + add(new NodeProperty<>(SCORE.toString(), SCORE.toString(), NO_DESCR, "")); + add(new NodeProperty<>(COMMENT.toString(), COMMENT.toString(), NO_DESCR, "")); + if (!UserPreferences.hideCentralRepoCommentsAndOccurrences()) { + add(new NodeProperty<>(OCCURRENCES.toString(), OCCURRENCES.toString(), NO_DESCR, "")); } - }); - properties.add(new ToggleableNodeProperty( - LOCATION.toString(), - NO_DESCR, - FileNodeUtil.getContentPath(content))); - properties.add(new ToggleableNodeProperty( - MOD_TIME.toString(), - NO_DESCR, - ContentUtils.getStringTime(content.getMtime(), content))); - properties.add(new ToggleableNodeProperty( - CHANGED_TIME.toString(), - NO_DESCR, - ContentUtils.getStringTime(content.getCtime(), content))); - properties.add(new ToggleableNodeProperty( - ACCESS_TIME.toString(), - NO_DESCR, - ContentUtils.getStringTime(content.getAtime(), content))); - properties.add(new ToggleableNodeProperty( - CREATED_TIME.toString(), - NO_DESCR, - ContentUtils.getStringTime(content.getCrtime(), content))); - properties.add(new ToggleableNodeProperty( - SIZE.toString(), - NO_DESCR, - StringUtils.defaultString(content.getMIMEType()))); - properties.add(new ToggleableNodeProperty( - FLAGS_DIR.toString(), - NO_DESCR, - content.getSize())); - properties.add(new ToggleableNodeProperty( - FLAGS_META.toString(), - NO_DESCR, - content.getMetaFlagsAsString())); - properties.add(new ToggleableNodeProperty( - MODE.toString(), - NO_DESCR, - content.getModesAsString())); - properties.add(new ToggleableNodeProperty( - USER_ID.toString(), - NO_DESCR, - content.getUid())); - properties.add(new ToggleableNodeProperty( - GROUP_ID.toString(), - NO_DESCR, - content.getGid())); - properties.add(new ToggleableNodeProperty( - META_ADDR.toString(), - NO_DESCR, - content.getMetaAddr())); - properties.add(new ToggleableNodeProperty( - ATTR_ADDR.toString(), - NO_DESCR, - content.getAttrType().getValue() + "-" + content.getAttributeId())); - properties.add(new ToggleableNodeProperty( - TYPE_DIR.toString(), - NO_DESCR, - content.getDirType().getLabel())); - properties.add(new ToggleableNodeProperty( - TYPE_META.toString(), - NO_DESCR, - content.getMetaType().toString())); - properties.add(new ToggleableNodeProperty( - KNOWN.toString(), - NO_DESCR, - content.getKnown().getName())); - properties.add(new ToggleableNodeProperty( - MD5HASH.toString(), - NO_DESCR, - StringUtils.defaultString(content.getMd5Hash()))); - properties.add(new ToggleableNodeProperty( - ObjectID.toString(), - NO_DESCR, - content.getId())); - properties.add(new ToggleableNodeProperty( - MIMETYPE.toString(), - NO_DESCR, - StringUtils.defaultString(content.getMIMEType()))); - properties.add(new ToggleableNodeProperty( - EXTENSION.toString(), - NO_DESCR, - content.getNameExtension())); - - //Submit the database queries ASAP! We want updated SCO columns - //without blocking the UI as soon as we can get it! Keep all weak references - //so this task doesn't block the ability of this node to be GC'd. - pool.submit(new SCOAndTranslationTask(new WeakReference<>(content), weakPcl)); + add(new NodeProperty<>(LOCATION.toString(), LOCATION.toString(), NO_DESCR, getContentPath(content))); + add(new NodeProperty<>(MOD_TIME.toString(), MOD_TIME.toString(), NO_DESCR, ContentUtils.getStringTime(content.getMtime(), content))); + add(new NodeProperty<>(CHANGED_TIME.toString(), CHANGED_TIME.toString(), NO_DESCR, ContentUtils.getStringTime(content.getCtime(), content))); + add(new NodeProperty<>(ACCESS_TIME.toString(), ACCESS_TIME.toString(), NO_DESCR, ContentUtils.getStringTime(content.getAtime(), content))); + add(new NodeProperty<>(CREATED_TIME.toString(), CREATED_TIME.toString(), NO_DESCR, ContentUtils.getStringTime(content.getCrtime(), content))); + add(new NodeProperty<>(SIZE.toString(), SIZE.toString(), NO_DESCR, content.getSize())); + add(new NodeProperty<>(FLAGS_DIR.toString(), FLAGS_DIR.toString(), NO_DESCR, content.getDirFlagAsString())); + add(new NodeProperty<>(FLAGS_META.toString(), FLAGS_META.toString(), NO_DESCR, content.getMetaFlagsAsString())); + add(new NodeProperty<>(MODE.toString(), MODE.toString(), NO_DESCR, content.getModesAsString())); + add(new NodeProperty<>(USER_ID.toString(), USER_ID.toString(), NO_DESCR, content.getUid())); + add(new NodeProperty<>(GROUP_ID.toString(), GROUP_ID.toString(), NO_DESCR, content.getGid())); + add(new NodeProperty<>(META_ADDR.toString(), META_ADDR.toString(), NO_DESCR, content.getMetaAddr())); + add(new NodeProperty<>(ATTR_ADDR.toString(), ATTR_ADDR.toString(), NO_DESCR, content.getAttrType().getValue() + "-" + content.getAttributeId())); + add(new NodeProperty<>(TYPE_DIR.toString(), TYPE_DIR.toString(), NO_DESCR, content.getDirType().getLabel())); + add(new NodeProperty<>(TYPE_META.toString(), TYPE_META.toString(), NO_DESCR, content.getMetaType().toString())); + add(new NodeProperty<>(KNOWN.toString(), KNOWN.toString(), NO_DESCR, content.getKnown().getName())); + add(new NodeProperty<>(MD5HASH.toString(), MD5HASH.toString(), NO_DESCR, StringUtils.defaultString(content.getMd5Hash()))); + add(new NodeProperty<>(ObjectID.toString(), ObjectID.toString(), NO_DESCR, content.getId())); + add(new NodeProperty<>(MIMETYPE.toString(), MIMETYPE.toString(), NO_DESCR, StringUtils.defaultString(content.getMIMEType()))); + add(new NodeProperty<>(EXTENSION.toString(), EXTENSION.toString(), NO_DESCR, content.getNameExtension())); + }}; + return properties; } @@ -551,12 +431,7 @@ public abstract class AbstractAbstractFileNode extends A @NbBundle.Messages("AbstractAbstractFileNode.tagsProperty.displayName=Tags") @Deprecated protected void addTagProperty(Sheet.Set sheetSet) { - List tags = new ArrayList<>(); - try { - tags.addAll(Case.getCurrentCaseThrows().getServices().getTagsManager().getContentTagsByContent(content)); - } catch (TskCoreException | NoCurrentCaseException ex) { - logger.log(Level.SEVERE, "Failed to get tags for content " + content.getName(), ex); - } + List tags = getContentTagsFromDatabase(); sheetSet.put(new NodeProperty<>("Tags", AbstractAbstractFileNode_tagsProperty_displayName(), NO_DESCR, tags.stream().map(t -> t.getName().getDisplayName()) .distinct() @@ -582,6 +457,190 @@ public abstract class AbstractAbstractFileNode extends A return ""; } } + + @NbBundle.Messages({ + "AbstractAbstractFileNode.createSheet.count.displayName=O", + "AbstractAbstractFileNode.createSheet.count.noCentralRepo.description=Central repository was not enabled when this column was populated", + "AbstractAbstractFileNode.createSheet.count.hashLookupNotRun.description=Hash lookup had not been run on this file when the column was populated", + "# {0} - occuranceCount", + "AbstractAbstractFileNode.createSheet.count.description=There were {0} datasource(s) found with occurances of the correlation value"}) + Pair getCountPropertyAndDescription(CorrelationAttributeInstance attribute) { + Long count = -1L; //The column renderer will not display negative values, negative value used when count unavailble to preserve sorting + String description = Bundle.AbstractAbstractFileNode_createSheet_count_noCentralRepo_description(); + try { + //don't perform the query if there is no correlation value + if (attribute != null && StringUtils.isNotBlank(attribute.getCorrelationValue())) { + count = EamDb.getInstance().getCountUniqueCaseDataSourceTuplesHavingTypeValue(attribute.getCorrelationType(), attribute.getCorrelationValue()); + description = Bundle.AbstractAbstractFileNode_createSheet_count_description(count); + } else if (attribute != null) { + description = Bundle.AbstractAbstractFileNode_createSheet_count_hashLookupNotRun_description(); + } + } catch (EamDbException ex) { + logger.log(Level.WARNING, "Error getting count of datasources with correlation attribute", ex); + } catch (CorrelationAttributeNormalizationException ex) { + logger.log(Level.WARNING, "Unable to normalize data to get count of datasources with correlation attribute", ex); + } + + return Pair.of(count, description); + } + + /** + * Used by subclasses of AbstractAbstractFileNode to add the Score property + * to their sheets. + * + * @param sheetSet the modifiable Sheet.Set returned by + * Sheet.get(Sheet.PROPERTIES) + * @param tags the list of tags associated with the file + */ + @NbBundle.Messages({ + "AbstractAbstractFileNode.createSheet.score.displayName=S", + "AbstractAbstractFileNode.createSheet.notableFile.description=File recognized as notable.", + "AbstractAbstractFileNode.createSheet.interestingResult.description=File has interesting result associated with it.", + "AbstractAbstractFileNode.createSheet.taggedFile.description=File has been tagged.", + "AbstractAbstractFileNode.createSheet.notableTaggedFile.description=File tagged with notable tag.", + "AbstractAbstractFileNode.createSheet.noScore.description=No score"}) + Pair getScorePropertyAndDescription(List tags) { + DataResultViewerTable.Score score = DataResultViewerTable.Score.NO_SCORE; + String description = ""; + if (content.getKnown() == TskData.FileKnown.BAD) { + score = DataResultViewerTable.Score.NOTABLE_SCORE; + description = Bundle.AbstractAbstractFileNode_createSheet_notableFile_description(); + } + try { + if (score == DataResultViewerTable.Score.NO_SCORE && !content.getArtifacts(BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT).isEmpty()) { + score = DataResultViewerTable.Score.INTERESTING_SCORE; + description = Bundle.AbstractAbstractFileNode_createSheet_interestingResult_description(); + } + } catch (TskCoreException ex) { + logger.log(Level.WARNING, "Error getting artifacts for file: " + content.getName(), ex); + } + if (!tags.isEmpty() && (score == DataResultViewerTable.Score.NO_SCORE || score == DataResultViewerTable.Score.INTERESTING_SCORE)) { + score = DataResultViewerTable.Score.INTERESTING_SCORE; + description = Bundle.AbstractAbstractFileNode_createSheet_taggedFile_description(); + for (ContentTag tag : tags) { + if (tag.getName().getKnownStatus() == TskData.FileKnown.BAD) { + score = DataResultViewerTable.Score.NOTABLE_SCORE; + description = Bundle.AbstractAbstractFileNode_createSheet_notableTaggedFile_description(); + break; + } + } + } + return Pair.of(score, description); + } + + /** + * Used by subclasses of AbstractAbstractFileNode to add the comment + * property to their sheets. + * + * @param sheetSet the modifiable Sheet.Set returned by + * Sheet.get(Sheet.PROPERTIES) + * @param tags the list of tags associated with the file + * @param attribute the correlation attribute associated with this file, + * null if central repo is not enabled + */ + @NbBundle.Messages({ + "AbstractAbstractFileNode.createSheet.comment.displayName=C"}) + HasCommentStatus getCommentProperty(List tags, CorrelationAttributeInstance attribute) { + + DataResultViewerTable.HasCommentStatus status = !tags.isEmpty() ? DataResultViewerTable.HasCommentStatus.TAG_NO_COMMENT : DataResultViewerTable.HasCommentStatus.NO_COMMENT; + + for (ContentTag tag : tags) { + if (!StringUtils.isBlank(tag.getComment())) { + //if the tag is null or empty or contains just white space it will indicate there is not a comment + status = DataResultViewerTable.HasCommentStatus.TAG_COMMENT; + break; + } + } + if (attribute != null && !StringUtils.isBlank(attribute.getComment())) { + if (status == DataResultViewerTable.HasCommentStatus.TAG_COMMENT) { + status = DataResultViewerTable.HasCommentStatus.CR_AND_TAG_COMMENTS; + } else { + status = DataResultViewerTable.HasCommentStatus.CR_COMMENT; + } + } + return status; + } + + /** + * Attempts translation of the content name being passed in. + * + * @return The file names translation. + */ + String getTranslatedFileName() { + //If already in complete English, don't translate. + if (content.getName().matches("^\\p{ASCII}+$")) { + return ""; + } + TextTranslationService tts = TextTranslationService.getInstance(); + if (tts.hasProvider()) { + //Seperate out the base and ext from the contents file name. + String base = FilenameUtils.getBaseName(content.getName()); + try { + String translation = tts.translate(base); + String ext = FilenameUtils.getExtension(content.getName()); + + //If we have no extension, then we shouldn't add the . + String extensionDelimiter = (ext.isEmpty()) ? "" : "."; + + //Talk directly to this nodes pcl, fire an update when the translation + //is complete. + if (!translation.isEmpty()) { + return translation + extensionDelimiter + ext; + } + } catch (NoServiceProviderException noServiceEx) { + logger.log(Level.WARNING, "Translate unsuccessful because no TextTranslator " + + "implementation was provided.", noServiceEx.getMessage()); + } catch (TranslationException noTranslationEx) { + logger.log(Level.WARNING, "Could not successfully translate file name " + + content.getName(), noTranslationEx.getMessage()); + } + } + return ""; + } + + /** + * Get all tags from the case database that are associated with the file + * + * @return a list of tags that are associated with the file + */ + List getContentTagsFromDatabase() { + List tags = new LinkedList<>(); + try { + tags.addAll(Case.getCurrentCaseThrows().getServices().getTagsManager().getContentTagsByContent(content)); + } catch (TskCoreException | NoCurrentCaseException ex) { + logger.log(Level.SEVERE, "Failed to get tags for content " + content.getName(), ex); + } + return tags; + } + + CorrelationAttributeInstance getCorrelationAttributeInstance() { + CorrelationAttributeInstance attribute = null; + if (EamDbUtil.useCentralRepo()) { + attribute = EamArtifactUtil.getInstanceFromContent(content); + } + return attribute; + } + + static String getContentPath(AbstractFile file) { + try { + return file.getUniquePath(); + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, "Except while calling Content.getUniquePath() on " + file.getName(), ex); //NON-NLS + return ""; //NON-NLS + } + } + + static String getContentDisplayName(AbstractFile file) { + String name = file.getName(); + switch (name) { + case "..": + return DirectoryNode.DOTDOTDIR; + case ".": + return DirectoryNode.DOTDIR; + default: + return name; + } + } /** * Fill map with AbstractFile properties @@ -589,11 +648,13 @@ public abstract class AbstractAbstractFileNode extends A * @param map map with preserved ordering, where property names/values * are put * @param content The content to get properties for. + * + * TODO JIRA-4421: Deprecate this method and resolve warnings that appear + * in other locations. */ - @Deprecated static public void fillPropertyMap(Map map, AbstractFile content) { - map.put(NAME.toString(), FileNodeUtil.getContentDisplayName(content)); - map.put(LOCATION.toString(), FileNodeUtil.getContentPath(content)); + map.put(NAME.toString(), getContentDisplayName(content)); + map.put(LOCATION.toString(), getContentPath(content)); map.put(MOD_TIME.toString(), ContentUtils.getStringTime(content.getMtime(), content)); map.put(CHANGED_TIME.toString(), ContentUtils.getStringTime(content.getCtime(), content)); map.put(ACCESS_TIME.toString(), ContentUtils.getStringTime(content.getAtime(), content)); @@ -614,4 +675,4 @@ public abstract class AbstractAbstractFileNode extends A map.put(MIMETYPE.toString(), StringUtils.defaultString(content.getMIMEType())); map.put(EXTENSION.toString(), content.getNameExtension()); } -} +} \ No newline at end of file diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractFsContentNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractFsContentNode.java index 559acb451f..e4af5e24ef 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractFsContentNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractFsContentNode.java @@ -51,7 +51,7 @@ public abstract class AbstractFsContentNode extends Abst */ AbstractFsContentNode(T content, boolean directoryBrowseMode) { super(content); - this.setDisplayName(FileNodeUtil.getContentDisplayName(content)); + this.setDisplayName(getContentDisplayName(content)); this.directoryBrowseMode = directoryBrowseMode; } diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/FileNodeUtil.java b/Core/src/org/sleuthkit/autopsy/datamodel/FileNodeUtil.java deleted file mode 100755 index 64261c56b0..0000000000 --- a/Core/src/org/sleuthkit/autopsy/datamodel/FileNodeUtil.java +++ /dev/null @@ -1,244 +0,0 @@ -/* - * Autopsy Forensic Browser - * - * Copyright 2018-2018 Basis Technology Corp. - * Contact: carrier sleuthkit org - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.sleuthkit.autopsy.datamodel; - -import java.util.ArrayList; -import java.util.List; -import java.util.logging.Level; -import org.apache.commons.io.FilenameUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.tuple.Pair; -import org.openide.util.NbBundle; -import org.sleuthkit.autopsy.casemodule.Case; -import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; -import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeInstance; -import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeNormalizationException; -import org.sleuthkit.autopsy.centralrepository.datamodel.EamArtifactUtil; -import org.sleuthkit.autopsy.centralrepository.datamodel.EamDb; -import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbException; -import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbUtil; -import org.sleuthkit.autopsy.corecomponents.DataResultViewerTable; -import org.sleuthkit.autopsy.coreutils.Logger; -import org.sleuthkit.autopsy.texttranslation.NoServiceProviderException; -import org.sleuthkit.autopsy.texttranslation.TextTranslationService; -import org.sleuthkit.autopsy.texttranslation.TranslationException; -import org.sleuthkit.datamodel.AbstractFile; -import org.sleuthkit.datamodel.BlackboardArtifact; -import org.sleuthkit.datamodel.ContentTag; -import org.sleuthkit.datamodel.TskCoreException; -import org.sleuthkit.datamodel.TskData; - -/** - * Utility class for getting common data about an AbstractFile, such as content tags - * correlation attributes, content paths and SCO values, to name a few. - */ -class FileNodeUtil { - - private static final String NO_TRANSLATION = ""; - private static final Logger logger = Logger.getLogger(FileNodeUtil.class.getName()); - - @NbBundle.Messages({ - "AbstractAbstractFileNode.createSheet.count.displayName=O", - "AbstractAbstractFileNode.createSheet.count.noCentralRepo.description=Central repository was not enabled when this column was populated", - "AbstractAbstractFileNode.createSheet.count.hashLookupNotRun.description=Hash lookup had not been run on this file when the column was populated", - "# {0} - occuranceCount", - "AbstractAbstractFileNode.createSheet.count.description=There were {0} datasource(s) found with occurances of the correlation value"}) - static Pair getCountPropertyAndDescription(CorrelationAttributeInstance attribute) { - Long count = -1L; //The column renderer will not display negative values, negative value used when count unavailble to preserve sorting - String description = Bundle.AbstractAbstractFileNode_createSheet_count_noCentralRepo_description(); - try { - //don't perform the query if there is no correlation value - if (attribute != null && StringUtils.isNotBlank(attribute.getCorrelationValue())) { - count = EamDb.getInstance().getCountUniqueCaseDataSourceTuplesHavingTypeValue(attribute.getCorrelationType(), attribute.getCorrelationValue()); - description = Bundle.AbstractAbstractFileNode_createSheet_count_description(count); - } else if (attribute != null) { - description = Bundle.AbstractAbstractFileNode_createSheet_count_hashLookupNotRun_description(); - } - } catch (EamDbException ex) { - logger.log(Level.WARNING, "Error getting count of datasources with correlation attribute", ex); - } catch (CorrelationAttributeNormalizationException ex) { - logger.log(Level.WARNING, "Unable to normalize data to get count of datasources with correlation attribute", ex); - } - - return Pair.of(count, description); - } - - - /** - * Used by subclasses of AbstractAbstractFileNode to add the Score property - * to their sheets. - * - * @param sheetSet the modifiable Sheet.Set returned by - * Sheet.get(Sheet.PROPERTIES) - * @param tags the list of tags associated with the file - */ - @NbBundle.Messages({ - "AbstractAbstractFileNode.createSheet.score.displayName=S", - "AbstractAbstractFileNode.createSheet.notableFile.description=File recognized as notable.", - "AbstractAbstractFileNode.createSheet.interestingResult.description=File has interesting result associated with it.", - "AbstractAbstractFileNode.createSheet.taggedFile.description=File has been tagged.", - "AbstractAbstractFileNode.createSheet.notableTaggedFile.description=File tagged with notable tag.", - "AbstractAbstractFileNode.createSheet.noScore.description=No score"}) - static Pair getScorePropertyAndDescription(AbstractFile content, List tags) { - DataResultViewerTable.Score score = DataResultViewerTable.Score.NO_SCORE; - String description = ""; - if (content.getKnown() == TskData.FileKnown.BAD) { - score = DataResultViewerTable.Score.NOTABLE_SCORE; - description = Bundle.AbstractAbstractFileNode_createSheet_notableFile_description(); - } - try { - if (score == DataResultViewerTable.Score.NO_SCORE && !content.getArtifacts(BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT).isEmpty()) { - score = DataResultViewerTable.Score.INTERESTING_SCORE; - description = Bundle.AbstractAbstractFileNode_createSheet_interestingResult_description(); - } - } catch (TskCoreException ex) { - logger.log(Level.WARNING, "Error getting artifacts for file: " + content.getName(), ex); - } - if (tags.size() > 0 && (score == DataResultViewerTable.Score.NO_SCORE || score == DataResultViewerTable.Score.INTERESTING_SCORE)) { - score = DataResultViewerTable.Score.INTERESTING_SCORE; - description = Bundle.AbstractAbstractFileNode_createSheet_taggedFile_description(); - for (ContentTag tag : tags) { - if (tag.getName().getKnownStatus() == TskData.FileKnown.BAD) { - score = DataResultViewerTable.Score.NOTABLE_SCORE; - description = Bundle.AbstractAbstractFileNode_createSheet_notableTaggedFile_description(); - break; - } - } - } - return Pair.of(score, description); - } - - /** - * Used by subclasses of AbstractAbstractFileNode to add the comment - * property to their sheets. - * - * @param sheetSet the modifiable Sheet.Set returned by - * Sheet.get(Sheet.PROPERTIES) - * @param tags the list of tags associated with the file - * @param attribute the correlation attribute associated with this file, - * null if central repo is not enabled - */ - @NbBundle.Messages({ - "AbstractAbstractFileNode.createSheet.comment.displayName=C"}) - static DataResultViewerTable.HasCommentStatus getCommentProperty(List tags, CorrelationAttributeInstance attribute) { - - DataResultViewerTable.HasCommentStatus status = tags.size() > 0 ? DataResultViewerTable.HasCommentStatus.TAG_NO_COMMENT : DataResultViewerTable.HasCommentStatus.NO_COMMENT; - - for (ContentTag tag : tags) { - if (!StringUtils.isBlank(tag.getComment())) { - //if the tag is null or empty or contains just white space it will indicate there is not a comment - status = DataResultViewerTable.HasCommentStatus.TAG_COMMENT; - break; - } - } - if (attribute != null && !StringUtils.isBlank(attribute.getComment())) { - if (status == DataResultViewerTable.HasCommentStatus.TAG_COMMENT) { - status = DataResultViewerTable.HasCommentStatus.CR_AND_TAG_COMMENTS; - } else { - status = DataResultViewerTable.HasCommentStatus.CR_COMMENT; - } - } - return status; - } - - /** - * Attempts translation of the content name being passed in. - * - * @return The file names translation. - */ - static String getTranslatedFileName(AbstractFile content) { - //If already in complete English, don't translate. - if (content.getName().matches("^\\p{ASCII}+$")) { - return NO_TRANSLATION; - } - - TextTranslationService tts = TextTranslationService.getInstance(); - if (tts.hasProvider()) { - //Seperate out the base and ext from the contents file name. - String base = FilenameUtils.getBaseName(content.getName()); - - try { - String translation = tts.translate(base); - String ext = FilenameUtils.getExtension(content.getName()); - - //If we have no extension, then we shouldn't add the . - String extensionDelimiter = (ext.isEmpty()) ? "" : "."; - - //Talk directly to this nodes pcl, fire an update when the translation - //is complete. - if (!translation.isEmpty()) { - return translation + extensionDelimiter + ext; - } - } catch (NoServiceProviderException noServiceEx) { - logger.log(Level.WARNING, "Translate unsuccessful because no TextTranslator " - + "implementation was provided.", noServiceEx.getMessage()); - } catch (TranslationException noTranslationEx) { - logger.log(Level.WARNING, "Could not successfully translate file name " - + content.getName(), noTranslationEx.getMessage()); - } - } - - return NO_TRANSLATION; - } - - /** - * Get all tags from the case database that are associated with the file - * - * @return a list of tags that are associated with the file - */ - static List getContentTagsFromDatabase(AbstractFile content) { - List tags = new ArrayList<>(); - try { - tags.addAll(Case.getCurrentCaseThrows().getServices().getTagsManager().getContentTagsByContent(content)); - } catch (TskCoreException | NoCurrentCaseException ex) { - logger.log(Level.SEVERE, "Failed to get tags for content " + content.getName(), ex); - } - return tags; - } - - static CorrelationAttributeInstance getCorrelationAttributeInstance(AbstractFile content) { - CorrelationAttributeInstance attribute = null; - if (EamDbUtil.useCentralRepo()) { - attribute = EamArtifactUtil.getInstanceFromContent(content); - } - return attribute; - } - - static String getContentPath(AbstractFile file) { - try { - return file.getUniquePath(); - } catch (TskCoreException ex) { - logger.log(Level.SEVERE, "Except while calling Content.getUniquePath() on " + file, ex); //NON-NLS - return ""; //NON-NLS - } - } - - static String getContentDisplayName(AbstractFile file) { - String name = file.getName(); - switch (name) { - case "..": - return DirectoryNode.DOTDOTDIR; - - case ".": - return DirectoryNode.DOTDIR; - default: - return name; - } - } -} diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/SCOAndTranslationTask.java b/Core/src/org/sleuthkit/autopsy/datamodel/SCOAndTranslationTask.java index 43dfa783d7..2dc3f4b9bb 100755 --- a/Core/src/org/sleuthkit/autopsy/datamodel/SCOAndTranslationTask.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/SCOAndTranslationTask.java @@ -28,7 +28,6 @@ import org.sleuthkit.autopsy.corecomponents.DataResultViewerTable; import org.sleuthkit.autopsy.corecomponents.DataResultViewerTable.HasCommentStatus; import org.sleuthkit.autopsy.corecomponents.DataResultViewerTable.Score; import org.sleuthkit.autopsy.events.AutopsyEvent; -import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.ContentTag; @@ -36,34 +35,29 @@ import org.sleuthkit.datamodel.ContentTag; * Completes the tasks needed to populate the Score, Comment, Occurrences and Translation * columns in the background so that the UI is not blocked while waiting for responses from the database or * translation service. Once these events are done, it fires a PropertyChangeEvent - * to let the AbstractAbstractFileNode know it's time to update! + * to let the AbstractAbstractFileNode know it's time to update. */ class SCOAndTranslationTask implements Runnable { - private final WeakReference weakContentRef; + private final WeakReference> weakNodeRef; private final PropertyChangeListener listener; - public SCOAndTranslationTask(WeakReference weakContentRef, PropertyChangeListener listener) { - this.weakContentRef = weakContentRef; + public SCOAndTranslationTask(WeakReference> weakContentRef, PropertyChangeListener listener) { + this.weakNodeRef = weakContentRef; this.listener = listener; } @Override public void run() { try { - AbstractFile content = weakContentRef.get(); + AbstractAbstractFileNode fileNode = weakNodeRef.get(); //Long DB queries - List tags = FileNodeUtil.getContentTagsFromDatabase(content); - CorrelationAttributeInstance attribute = - FileNodeUtil.getCorrelationAttributeInstance(content); - - Pair scoreAndDescription = - FileNodeUtil.getScorePropertyAndDescription(content, tags); - DataResultViewerTable.HasCommentStatus comment = - FileNodeUtil.getCommentProperty(tags, attribute); - Pair countAndDescription = - FileNodeUtil.getCountPropertyAndDescription(attribute); + List tags = fileNode.getContentTagsFromDatabase(); + CorrelationAttributeInstance attribute = fileNode.getCorrelationAttributeInstance(); + Pair scoreAndDescription = fileNode.getScorePropertyAndDescription(tags); + DataResultViewerTable.HasCommentStatus comment = fileNode.getCommentProperty(tags, attribute); + Pair countAndDescription = fileNode.getCountPropertyAndDescription(attribute); //Load the results from the SCO column operations into a wrapper object to be passed //back to the listener so that the node can internally update it's propertySheet. @@ -77,18 +71,21 @@ class SCOAndTranslationTask implements Runnable { listener.propertyChange(new PropertyChangeEvent( AutopsyEvent.SourceType.LOCAL.toString(), - AbstractAbstractFileNode.NodeSpecificEvents.DABABASE_CONTENT_AVAILABLE.toString(), + AbstractAbstractFileNode.NodeSpecificEvents.SCO_AVAILABLE.toString(), null, results)); //Once we've got the SCO columns, then lets fire the translation result. //Updating of this column is significantly lower priority than //getting results to the SCO columns! - listener.propertyChange(new PropertyChangeEvent( - AutopsyEvent.SourceType.LOCAL.toString(), - AbstractAbstractFileNode.NodeSpecificEvents.TRANSLATION_AVAILABLE.toString(), - null, - FileNodeUtil.getTranslatedFileName(content))); + String translatedFileName = fileNode.getTranslatedFileName(); + if(!translatedFileName.isEmpty()) { + //Only fire if the result is meaningful. + listener.propertyChange(new PropertyChangeEvent( + AutopsyEvent.SourceType.LOCAL.toString(), + AbstractAbstractFileNode.NodeSpecificEvents.TRANSLATION_AVAILABLE.toString(), + null, translatedFileName)); + } } catch (NullPointerException ex) { //If we are here, that means our weakPcl or content pointer has gone stale (aka //has been garbage collected). There's no recovery. Netbeans has diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/ToggleableNodeProperty.java b/Core/src/org/sleuthkit/autopsy/datamodel/ToggleableNodeProperty.java deleted file mode 100755 index 5272c8b205..0000000000 --- a/Core/src/org/sleuthkit/autopsy/datamodel/ToggleableNodeProperty.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Autopsy Forensic Browser - * - * Copyright 2018-2018 Basis Technology Corp. - * Contact: carrier sleuthkit org - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.sleuthkit.autopsy.datamodel; - -/** - * Adds the functionality of enabling and disabling a NodeProperty (column in the UI). - */ -class ToggleableNodeProperty extends NodeProperty { - - /** - * Wraps the super constructor. In our use cases, we want the name and display - * name of the column to be the exact same, so to avoid redundancy we accept the name - * just once and pass it twice to the NodeProperty. - * - * @param name Name of the property to be displayed - * @param desc Description of the property when hovering over the column - * @param value Value to be displayed in that column - */ - public ToggleableNodeProperty(String name, String desc, Object value) { - super(name, name, desc, value); - } - - /** - * Allows a property to be either enabled or disabled. When creating a sheet, - * this method is used to filter out from displaying in the UI. - * - * @return boolean denoting the availiability of this property. True by default. - */ - public boolean isEnabled() { - return true; - } -} From d597eea4a86734fa021d3a99611ed50cf9207393 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Wed, 14 Nov 2018 12:59:38 -0500 Subject: [PATCH 106/129] Adjusted some comments --- .../datamodel/AbstractAbstractFileNode.java | 36 +++---------------- 1 file changed, 4 insertions(+), 32 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java index 3bd51c5f8a..c9d4895068 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java @@ -293,7 +293,7 @@ public abstract class AbstractAbstractFileNode extends A /* * Submit the database queries ASAP. We want updated SCO columns without - * blocking the UI as soon as we can get it! Keep all weak references so + * blocking the UI. Keep all weak references so * this task doesn't block the ability of this node to be GC'd. */ SCOAndTranslationPool.submit(new SCOAndTranslationTask(new WeakReference<>(this), weakPcl)); @@ -367,16 +367,7 @@ public abstract class AbstractAbstractFileNode extends A } /** - * Creates a list of properties for this file node. NodeProperty - * is a subclass of NodeProperty, with the added functionality of being able to be - * enabled and disabled. Disabled properties don't get added to the sheet. - * Additionally, with a return value of a list, any children classes of this - * node may reorder or omit any of these properties as they see fit for their use case. - * - * Note: subclasses that use this, please synchronize your createSheet method, so that the - * updates don't come in while you haven't finished creating your sheet. - * - * @return List of file properties associated with this file node's content. + * Creates and populates a list of properties for this nodes property sheet. */ private List> getProperties() { List> properties = new LinkedList>() {{ @@ -484,14 +475,6 @@ public abstract class AbstractAbstractFileNode extends A return Pair.of(count, description); } - /** - * Used by subclasses of AbstractAbstractFileNode to add the Score property - * to their sheets. - * - * @param sheetSet the modifiable Sheet.Set returned by - * Sheet.get(Sheet.PROPERTIES) - * @param tags the list of tags associated with the file - */ @NbBundle.Messages({ "AbstractAbstractFileNode.createSheet.score.displayName=S", "AbstractAbstractFileNode.createSheet.notableFile.description=File recognized as notable.", @@ -528,16 +511,6 @@ public abstract class AbstractAbstractFileNode extends A return Pair.of(score, description); } - /** - * Used by subclasses of AbstractAbstractFileNode to add the comment - * property to their sheets. - * - * @param sheetSet the modifiable Sheet.Set returned by - * Sheet.get(Sheet.PROPERTIES) - * @param tags the list of tags associated with the file - * @param attribute the correlation attribute associated with this file, - * null if central repo is not enabled - */ @NbBundle.Messages({ "AbstractAbstractFileNode.createSheet.comment.displayName=C"}) HasCommentStatus getCommentProperty(List tags, CorrelationAttributeInstance attribute) { @@ -562,9 +535,8 @@ public abstract class AbstractAbstractFileNode extends A } /** - * Attempts translation of the content name being passed in. - * - * @return The file names translation. + * Translates this nodes content name. Doesn't attempt translation if + * the name is in english or if there is now translation service available. */ String getTranslatedFileName() { //If already in complete English, don't translate. From 5c59c8937c20dc58c462836f929c896d67ab41c4 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Wed, 14 Nov 2018 13:05:37 -0500 Subject: [PATCH 107/129] Changed variable names and minor tweaks --- .../sleuthkit/autopsy/corecomponents/Bundle.properties | 2 +- .../autopsy/corecomponents/ViewPreferencesPanel.form | 8 ++++---- .../autopsy/corecomponents/ViewPreferencesPanel.java | 10 +++++----- .../autopsy/datamodel/AbstractAbstractFileNode.java | 6 +++--- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/corecomponents/Bundle.properties b/Core/src/org/sleuthkit/autopsy/corecomponents/Bundle.properties index 38be797a22..93626ee62a 100644 --- a/Core/src/org/sleuthkit/autopsy/corecomponents/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/corecomponents/Bundle.properties @@ -192,5 +192,5 @@ ViewPreferencesPanel.centralRepoLabel.text=Do not use Central Repository for: ViewPreferencesPanel.commentsOccurencesColumnsCheckbox.text=C(omments) and O(ccurences) columns to reduce loading times ViewPreferencesPanel.deletedFilesLimitCheckbox.text=Limit to 10,000 ViewPreferencesPanel.deletedFilesLimitLabel.text=Limit number of deleted files displayed: -ViewPreferencesPanel.fileDisplayLabel.text=Translate text in the: ViewPreferencesPanel.translateNamesRadioButton.text=Table +ViewPreferencesPanel.translateTextLabel.text=Translate text in the: diff --git a/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.form b/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.form index 7b8f1e49b2..89ffab91d8 100755 --- a/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.form +++ b/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.form @@ -120,7 +120,7 @@ - + @@ -177,7 +177,7 @@ - + @@ -374,10 +374,10 @@ - + - + diff --git a/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.java b/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.java index 6f2a0ab68c..b96e97d133 100755 --- a/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.java +++ b/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.java @@ -151,7 +151,7 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { deletedFilesLimitCheckbox = new javax.swing.JCheckBox(); deletedFilesLimitLabel = new javax.swing.JLabel(); translateNamesRadioButton = new javax.swing.JRadioButton(); - fileDisplayLabel = new javax.swing.JLabel(); + translateTextLabel = new javax.swing.JLabel(); currentCaseSettingsPanel = new javax.swing.JPanel(); groupByDataSourceCheckbox = new javax.swing.JCheckBox(); currentSessionSettingsPanel = new javax.swing.JPanel(); @@ -261,7 +261,7 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { } }); - org.openide.awt.Mnemonics.setLocalizedText(fileDisplayLabel, org.openide.util.NbBundle.getMessage(ViewPreferencesPanel.class, "ViewPreferencesPanel.fileDisplayLabel.text")); // NOI18N + org.openide.awt.Mnemonics.setLocalizedText(translateTextLabel, org.openide.util.NbBundle.getMessage(ViewPreferencesPanel.class, "ViewPreferencesPanel.translateTextLabel.text")); // NOI18N javax.swing.GroupLayout globalSettingsPanelLayout = new javax.swing.GroupLayout(globalSettingsPanel); globalSettingsPanel.setLayout(globalSettingsPanelLayout); @@ -299,7 +299,7 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { .addComponent(hideOtherUsersTagsLabel)) .addGap(18, 18, 18) .addGroup(globalSettingsPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addComponent(fileDisplayLabel) + .addComponent(translateTextLabel) .addComponent(displayTimeLabel) .addGroup(globalSettingsPanelLayout.createSequentialGroup() .addGap(10, 10, 10) @@ -344,7 +344,7 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { .addComponent(useGMTTimeRadioButton))) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) .addGroup(globalSettingsPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addComponent(fileDisplayLabel) + .addComponent(translateTextLabel) .addComponent(hideOtherUsersTagsLabel)) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) .addGroup(globalSettingsPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) @@ -579,7 +579,6 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { private javax.swing.JCheckBox deletedFilesLimitCheckbox; private javax.swing.JLabel deletedFilesLimitLabel; private javax.swing.JLabel displayTimeLabel; - private javax.swing.JLabel fileDisplayLabel; private javax.swing.JPanel globalSettingsPanel; private javax.swing.JCheckBox groupByDataSourceCheckbox; private javax.swing.JLabel hideKnownFilesLabel; @@ -590,6 +589,7 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { private javax.swing.JRadioButton keepCurrentViewerRadioButton; private javax.swing.JLabel selectFileLabel; private javax.swing.JRadioButton translateNamesRadioButton; + private javax.swing.JLabel translateTextLabel; private javax.swing.JRadioButton useBestViewerRadioButton; private javax.swing.JRadioButton useGMTTimeRadioButton; private javax.swing.JRadioButton useLocalTimeRadioButton; diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java index c9d4895068..5890fea0cf 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java @@ -22,7 +22,7 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder; import java.beans.PropertyChangeEvent; import java.beans.PropertyChangeListener; import java.lang.ref.WeakReference; -import java.util.LinkedList; +import java.util.ArrayList; import java.util.EnumSet; import java.util.List; import java.util.Map; @@ -370,7 +370,7 @@ public abstract class AbstractAbstractFileNode extends A * Creates and populates a list of properties for this nodes property sheet. */ private List> getProperties() { - List> properties = new LinkedList>() {{ + List> properties = new ArrayList>() {{ add(new NodeProperty<>(NAME.toString(), NAME.toString(), NO_DESCR, getContentDisplayName(content))); /* * Initialize dummy place holder properties for Translation, @@ -576,7 +576,7 @@ public abstract class AbstractAbstractFileNode extends A * @return a list of tags that are associated with the file */ List getContentTagsFromDatabase() { - List tags = new LinkedList<>(); + List tags = new ArrayList<>(); try { tags.addAll(Case.getCurrentCaseThrows().getServices().getTagsManager().getContentTagsByContent(content)); } catch (TskCoreException | NoCurrentCaseException ex) { From 75bb46f0f4044696b7b1d385a0671241d4c4bdda Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Wed, 14 Nov 2018 13:18:07 -0500 Subject: [PATCH 108/129] Reverted Prefences panel and will try the merge conflict resolution again --- .../corecomponents/ViewPreferencesPanel.form | 118 ++++-------------- .../corecomponents/ViewPreferencesPanel.java | 107 +++------------- 2 files changed, 41 insertions(+), 184 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.form b/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.form index 60366083cb..89ffab91d8 100755 --- a/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.form +++ b/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.form @@ -31,19 +31,11 @@ - - - - - - - - @@ -95,10 +87,7 @@ - - - - + @@ -106,34 +95,25 @@ - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + @@ -150,30 +130,16 @@ - - - - - - - - - - - - - - - + - - + + @@ -194,16 +160,6 @@ - - - - - - - - - - @@ -215,10 +171,8 @@ - - - - + + @@ -349,14 +303,14 @@ - + - + - + @@ -427,28 +381,6 @@ - - - - - - - - - - - - - - - - - - - - - - diff --git a/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.java b/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.java index a9f5c6c865..b96e97d133 100755 --- a/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.java +++ b/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.java @@ -19,14 +19,12 @@ package org.sleuthkit.autopsy.corecomponents; import java.util.Objects; -import java.util.TimeZone; import javax.swing.JPanel; import org.netbeans.spi.options.OptionsPanelController; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.CasePreferences; import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbUtil; import org.sleuthkit.autopsy.core.UserPreferences; -import org.sleuthkit.autopsy.coreutils.TimeZoneUtils; import org.sleuthkit.autopsy.deletedFiles.DeletedFilePreferences; import org.sleuthkit.autopsy.directorytree.DirectoryTreeTopComponent; import org.sleuthkit.autopsy.texttranslation.TextTranslationService; @@ -52,8 +50,6 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { //from the user. TextTranslationService tts = TextTranslationService.getInstance(); translateNamesRadioButton.setEnabled(tts.hasProvider()); - - this.timeZoneList.setListData(TimeZoneUtils.createTimeZoneList().stream().toArray(String[]::new)); } @Override @@ -64,10 +60,8 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { useBestViewerRadioButton.setSelected(!keepPreferredViewer); boolean useLocalTime = UserPreferences.displayTimesInLocalTime(); - timeZoneList.setEnabled(!useLocalTime); - timeZoneList.setSelectedValue(TimeZoneUtils.createTimeZoneString(TimeZone.getTimeZone(UserPreferences.getTimeZoneForDisplays())), true); useLocalTimeRadioButton.setSelected(useLocalTime); - useAnotherTimeRadioButton.setSelected(!useLocalTime); + useGMTTimeRadioButton.setSelected(!useLocalTime); dataSourcesHideKnownCheckbox.setSelected(UserPreferences.hideKnownFilesInDataSourcesTree()); viewsHideKnownCheckbox.setSelected(UserPreferences.hideKnownFilesInViewsTree()); @@ -97,9 +91,6 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { public void store() { UserPreferences.setKeepPreferredContentViewer(keepCurrentViewerRadioButton.isSelected()); UserPreferences.setDisplayTimesInLocalTime(useLocalTimeRadioButton.isSelected()); - if (useAnotherTimeRadioButton.isSelected()) { - UserPreferences.setTimeZoneForDisplays(timeZoneList.getSelectedValue().substring(11).trim()); - } UserPreferences.setHideKnownFilesInDataSourcesTree(dataSourcesHideKnownCheckbox.isSelected()); UserPreferences.setHideKnownFilesInViewsTree(viewsHideKnownCheckbox.isSelected()); UserPreferences.setHideSlackFilesInDataSourcesTree(dataSourcesHideSlackCheckbox.isSelected()); @@ -152,7 +143,7 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { viewsHideSlackCheckbox = new javax.swing.JCheckBox(); displayTimeLabel = new javax.swing.JLabel(); useLocalTimeRadioButton = new javax.swing.JRadioButton(); - useAnotherTimeRadioButton = new javax.swing.JRadioButton(); + useGMTTimeRadioButton = new javax.swing.JRadioButton(); hideOtherUsersTagsCheckbox = new javax.swing.JCheckBox(); hideOtherUsersTagsLabel = new javax.swing.JLabel(); commentsOccurencesColumnsCheckbox = new javax.swing.JCheckBox(); @@ -161,17 +152,12 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { deletedFilesLimitLabel = new javax.swing.JLabel(); translateNamesRadioButton = new javax.swing.JRadioButton(); translateTextLabel = new javax.swing.JLabel(); - jScrollPane1 = new javax.swing.JScrollPane(); - timeZoneList = new javax.swing.JList<>(); currentCaseSettingsPanel = new javax.swing.JPanel(); groupByDataSourceCheckbox = new javax.swing.JCheckBox(); currentSessionSettingsPanel = new javax.swing.JPanel(); hideRejectedResultsCheckbox = new javax.swing.JCheckBox(); viewPreferencesScrollPane.setBorder(null); - viewPreferencesScrollPane.setPreferredSize(new java.awt.Dimension(625, 452)); - - viewPreferencesPanel.setPreferredSize(new java.awt.Dimension(625, 452)); globalSettingsPanel.setBorder(javax.swing.BorderFactory.createTitledBorder(org.openide.util.NbBundle.getMessage(ViewPreferencesPanel.class, "ViewPreferencesPanel.globalSettingsPanel.border.title"))); // NOI18N @@ -234,10 +220,10 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { } }); - org.openide.awt.Mnemonics.setLocalizedText(useAnotherTimeRadioButton, org.openide.util.NbBundle.getMessage(ViewPreferencesPanel.class, "ViewPreferencesPanel.useAnotherTimeRadioButton.text")); // NOI18N - useAnotherTimeRadioButton.addActionListener(new java.awt.event.ActionListener() { + org.openide.awt.Mnemonics.setLocalizedText(useGMTTimeRadioButton, org.openide.util.NbBundle.getMessage(ViewPreferencesPanel.class, "ViewPreferencesPanel.useGMTTimeRadioButton.text")); // NOI18N + useGMTTimeRadioButton.addActionListener(new java.awt.event.ActionListener() { public void actionPerformed(java.awt.event.ActionEvent evt) { - useAnotherTimeRadioButtonActionPerformed(evt); + useGMTTimeRadioButtonActionPerformed(evt); } }); @@ -276,12 +262,6 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { }); org.openide.awt.Mnemonics.setLocalizedText(translateTextLabel, org.openide.util.NbBundle.getMessage(ViewPreferencesPanel.class, "ViewPreferencesPanel.translateTextLabel.text")); // NOI18N - timeZoneList.addListSelectionListener(new javax.swing.event.ListSelectionListener() { - public void valueChanged(javax.swing.event.ListSelectionEvent evt) { - timeZoneListValueChanged(evt); - } - }); - jScrollPane1.setViewportView(timeZoneList); javax.swing.GroupLayout globalSettingsPanelLayout = new javax.swing.GroupLayout(globalSettingsPanel); globalSettingsPanel.setLayout(globalSettingsPanelLayout); @@ -295,38 +275,16 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { .addGroup(globalSettingsPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addComponent(commentsOccurencesColumnsCheckbox) .addComponent(hideOtherUsersTagsCheckbox) - .addGroup(globalSettingsPanelLayout.createSequentialGroup() - .addComponent(deletedFilesLimitCheckbox, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) - .addGap(399, 399, 399)))) + .addComponent(deletedFilesLimitCheckbox, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE))) .addGroup(globalSettingsPanelLayout.createSequentialGroup() .addGroup(globalSettingsPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addComponent(centralRepoLabel) .addComponent(deletedFilesLimitLabel, javax.swing.GroupLayout.PREFERRED_SIZE, 215, javax.swing.GroupLayout.PREFERRED_SIZE) - .addGroup(globalSettingsPanelLayout.createSequentialGroup() - .addComponent(centralRepoLabel) - .addGap(135, 135, 135) - .addComponent(jScrollPane1, javax.swing.GroupLayout.PREFERRED_SIZE, 272, javax.swing.GroupLayout.PREFERRED_SIZE)) .addGroup(globalSettingsPanelLayout.createSequentialGroup() .addGroup(globalSettingsPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addGroup(globalSettingsPanelLayout.createSequentialGroup() + .addComponent(hideKnownFilesLabel) + .addGroup(globalSettingsPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.TRAILING) .addGroup(globalSettingsPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addComponent(hideKnownFilesLabel) - .addGroup(globalSettingsPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.TRAILING) - .addGroup(globalSettingsPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addGroup(globalSettingsPanelLayout.createSequentialGroup() - .addGap(10, 10, 10) - .addGroup(globalSettingsPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addComponent(dataSourcesHideSlackCheckbox) - .addComponent(viewsHideSlackCheckbox))) - .addComponent(hideSlackFilesLabel)) - .addGroup(globalSettingsPanelLayout.createSequentialGroup() - .addGap(10, 10, 10) - .addGroup(globalSettingsPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addComponent(dataSourcesHideKnownCheckbox) - .addComponent(viewsHideKnownCheckbox))))) - .addGap(18, 18, 18) - .addGroup(globalSettingsPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addComponent(displayTimeLabel) .addGroup(globalSettingsPanelLayout.createSequentialGroup() .addGap(10, 10, 10) .addGroup(globalSettingsPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) @@ -354,15 +312,6 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { .addComponent(selectFileLabel)))) .addGap(0, 10, Short.MAX_VALUE))) .addContainerGap()) - .addComponent(keepCurrentViewerRadioButton) - .addComponent(useBestViewerRadioButton) - .addComponent(useLocalTimeRadioButton) - .addComponent(useAnotherTimeRadioButton))) - .addComponent(selectFileLabel))) - .addComponent(hideOtherUsersTagsLabel) - .addComponent(deletedFilesLimitLabel, javax.swing.GroupLayout.PREFERRED_SIZE, 215, javax.swing.GroupLayout.PREFERRED_SIZE)) - .addGap(0, 0, Short.MAX_VALUE))) - .addContainerGap()))) ); globalSettingsPanelLayout.setVerticalGroup( globalSettingsPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) @@ -380,17 +329,7 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) .addComponent(dataSourcesHideSlackCheckbox) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) - .addComponent(viewsHideSlackCheckbox) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) - .addComponent(hideOtherUsersTagsLabel) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) - .addComponent(hideOtherUsersTagsCheckbox) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) - .addComponent(centralRepoLabel) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) - .addComponent(commentsOccurencesColumnsCheckbox) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) - .addComponent(deletedFilesLimitLabel)) + .addComponent(viewsHideSlackCheckbox)) .addGroup(globalSettingsPanelLayout.createSequentialGroup() .addComponent(selectFileLabel) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) @@ -417,9 +356,6 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { .addComponent(commentsOccurencesColumnsCheckbox) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) .addComponent(deletedFilesLimitLabel) - .addComponent(useAnotherTimeRadioButton) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) - .addComponent(jScrollPane1, javax.swing.GroupLayout.PREFERRED_SIZE, 67, javax.swing.GroupLayout.PREFERRED_SIZE))) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) .addComponent(deletedFilesLimitCheckbox, javax.swing.GroupLayout.PREFERRED_SIZE, 33, javax.swing.GroupLayout.PREFERRED_SIZE) .addGap(0, 0, 0)) @@ -504,11 +440,11 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { this.setLayout(layout); layout.setHorizontalGroup( layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addComponent(viewPreferencesScrollPane, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) + .addComponent(viewPreferencesScrollPane) ); layout.setVerticalGroup( layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addComponent(viewPreferencesScrollPane, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) + .addComponent(viewPreferencesScrollPane) ); }// //GEN-END:initComponents @@ -534,8 +470,7 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { private void useLocalTimeRadioButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_useLocalTimeRadioButtonActionPerformed useLocalTimeRadioButton.setSelected(true); - useAnotherTimeRadioButton.setSelected(false); - timeZoneList.setEnabled(false); + useGMTTimeRadioButton.setSelected(false); if (immediateUpdates) { UserPreferences.setDisplayTimesInLocalTime(true); } else { @@ -543,16 +478,15 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { } }//GEN-LAST:event_useLocalTimeRadioButtonActionPerformed - private void useAnotherTimeRadioButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_useAnotherTimeRadioButtonActionPerformed + private void useGMTTimeRadioButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_useGMTTimeRadioButtonActionPerformed useLocalTimeRadioButton.setSelected(false); - useAnotherTimeRadioButton.setSelected(true); - timeZoneList.setEnabled(true); + useGMTTimeRadioButton.setSelected(true); if (immediateUpdates) { UserPreferences.setDisplayTimesInLocalTime(false); } else { firePropertyChange(OptionsPanelController.PROP_CHANGED, null, null); } - }//GEN-LAST:event_useAnotherTimeRadioButtonActionPerformed + }//GEN-LAST:event_useGMTTimeRadioButtonActionPerformed private void dataSourcesHideKnownCheckboxActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_dataSourcesHideKnownCheckboxActionPerformed if (immediateUpdates) { @@ -633,13 +567,6 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { firePropertyChange(OptionsPanelController.PROP_CHANGED, null, null); } }//GEN-LAST:event_translateNamesRadioButtonActionPerformed - private void timeZoneListValueChanged(javax.swing.event.ListSelectionEvent evt) {//GEN-FIRST:event_timeZoneListValueChanged - if (immediateUpdates && useAnotherTimeRadioButton.isSelected()) { - UserPreferences.setTimeZoneForDisplays(timeZoneList.getSelectedValue().substring(11).trim()); - } else { - firePropertyChange(OptionsPanelController.PROP_CHANGED, null, null); - } - }//GEN-LAST:event_timeZoneListValueChanged // Variables declaration - do not modify//GEN-BEGIN:variables @@ -659,14 +586,12 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { private javax.swing.JLabel hideOtherUsersTagsLabel; private javax.swing.JCheckBox hideRejectedResultsCheckbox; private javax.swing.JLabel hideSlackFilesLabel; - private javax.swing.JScrollPane jScrollPane1; private javax.swing.JRadioButton keepCurrentViewerRadioButton; private javax.swing.JLabel selectFileLabel; private javax.swing.JRadioButton translateNamesRadioButton; private javax.swing.JLabel translateTextLabel; - private javax.swing.JList timeZoneList; - private javax.swing.JRadioButton useAnotherTimeRadioButton; private javax.swing.JRadioButton useBestViewerRadioButton; + private javax.swing.JRadioButton useGMTTimeRadioButton; private javax.swing.JRadioButton useLocalTimeRadioButton; private javax.swing.JPanel viewPreferencesPanel; private javax.swing.JScrollPane viewPreferencesScrollPane; From ed802b0c3b0e23c305470a46a3b775010b346746 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Thu, 15 Nov 2018 15:35:58 -0500 Subject: [PATCH 109/129] Fixed ViewPreferncesPanel to cooperate --- .../autopsy/corecomponents/Bundle.properties | 44 +- .../corecomponents/Bundle_ja.properties | 10 +- .../corecomponents/ViewPreferencesPanel.form | 224 ++++++----- .../corecomponents/ViewPreferencesPanel.java | 376 ++++++++++-------- 4 files changed, 374 insertions(+), 280 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/corecomponents/Bundle.properties b/Core/src/org/sleuthkit/autopsy/corecomponents/Bundle.properties index ac35c7a680..2267791cbd 100644 --- a/Core/src/org/sleuthkit/autopsy/corecomponents/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/corecomponents/Bundle.properties @@ -165,32 +165,34 @@ AutopsyOptionsPanel.runtimePanel.border.title=Runtime DataResultPanel.matchLabel.text=Results DataResultPanel.numberOfChildNodesLabel.text=0 DataResultPanel.descriptionLabel.text=directoryPath -ViewPreferencesPanel.selectFileLabel.text=When selecting a file: -ViewPreferencesPanel.globalSettingsPanel.border.title=Global Settings -ViewPreferencesPanel.displayTimeLabel.text=When displaying times: -ViewPreferencesPanel.hideSlackFilesLabel.text=Hide slack files in the: ViewPreferencesPanel.groupByDataSourceCheckbox.text=Group by data source -ViewPreferencesPanel.hideKnownFilesLabel.text=Hide known files (i.e. those in the NIST NSRL) in the: -ViewPreferencesPanel.hideOtherUsersTagsCheckbox.text=Tags area in the tree ViewPreferencesPanel.currentCaseSettingsPanel.border.title=Current Case Settings OptionsCategory_Name_View=View OptionsCategory_Keywords_View=View -ViewPreferencesPanel.useBestViewerRadioButton.toolTipText=For example, change from Hex to Media when a JPEG is selected. -ViewPreferencesPanel.useBestViewerRadioButton.text=Change to the most specific file viewer -ViewPreferencesPanel.keepCurrentViewerRadioButton.toolTipText=For example, stay in Hex view when a JPEG is selected. -ViewPreferencesPanel.keepCurrentViewerRadioButton.text=Stay on the same file viewer -ViewPreferencesPanel.useLocalTimeRadioButton.text=Use local time zone -ViewPreferencesPanel.dataSourcesHideKnownCheckbox.text=Data Sources area (the directory hierarchy) -ViewPreferencesPanel.viewsHideKnownCheckbox.text=Views area -ViewPreferencesPanel.dataSourcesHideSlackCheckbox.text=Data Sources area (the directory hierarchy) -ViewPreferencesPanel.viewsHideSlackCheckbox.text=Views area ViewPreferencesPanel.currentSessionSettingsPanel.border.title=Current Session Settings ViewPreferencesPanel.hideRejectedResultsCheckbox.text=Hide rejected results -ViewPreferencesPanel.hideOtherUsersTagsLabel.text=Hide other users' tags in the: -ViewPreferencesPanel.centralRepoLabel.text=Do not use Central Repository for: -ViewPreferencesPanel.commentsOccurencesColumnsCheckbox.text=C(omments) and O(ccurences) columns to reduce loading times -ViewPreferencesPanel.deletedFilesLimitCheckbox.text=Limit to 10,000 -ViewPreferencesPanel.deletedFilesLimitLabel.text=Limit number of deleted files displayed: -ViewPreferencesPanel.translateNamesRadioButton.text=Table ViewPreferencesPanel.translateTextLabel.text=Translate text in the: +ViewPreferencesPanel.globalSettingsPanel.border.title=Global Settings +ViewPreferencesPanel.translateNamesInTableRadioButton.text=Table +ViewPreferencesPanel.TranslateTextLabel.text=Translate text in the: +ViewPreferencesPanel.deletedFilesLimitLabel.text=Limit number of deleted files displayed: +ViewPreferencesPanel.deletedFilesLimitCheckbox.text=Limit to 10,000 +ViewPreferencesPanel.commentsOccurencesColumnsCheckbox.text=C(omments) and O(ccurences) columns +ViewPreferencesPanel.centralRepoLabel.text=Do not use Central Repository for: +ViewPreferencesPanel.hideOtherUsersTagsLabel.text=Hide other users' tags in the: +ViewPreferencesPanel.hideOtherUsersTagsCheckbox.text=Tags area in the tree ViewPreferencesPanel.useAnotherTimeRadioButton.text=Use another time zone +ViewPreferencesPanel.useLocalTimeRadioButton.text=Use local time zone +ViewPreferencesPanel.displayTimeLabel.text=When displaying times: +ViewPreferencesPanel.viewsHideSlackCheckbox.text=Views area +ViewPreferencesPanel.dataSourcesHideSlackCheckbox.text=Data Sources area (the directory hierarchy) +ViewPreferencesPanel.hideSlackFilesLabel.text=Hide slack files in the: +ViewPreferencesPanel.viewsHideKnownCheckbox.text=Views area +ViewPreferencesPanel.dataSourcesHideKnownCheckbox.text=Data Sources area (the directory hierarchy) +ViewPreferencesPanel.hideKnownFilesLabel.text=Hide known files (i.e. those in the NIST NSRL) in the: +ViewPreferencesPanel.keepCurrentViewerRadioButton.toolTipText=For example, stay in Hex view when a JPEG is selected. +ViewPreferencesPanel.keepCurrentViewerRadioButton.text=Stay on the same file viewer +ViewPreferencesPanel.useBestViewerRadioButton.toolTipText=For example, change from Hex to Media when a JPEG is selected. +ViewPreferencesPanel.useBestViewerRadioButton.text=Change to the most specific file viewer +ViewPreferencesPanel.selectFileLabel.text=When selecting a file: +ViewPreferencesPanel.hideColumnWrapAroundText.text=to reduce loading times diff --git a/Core/src/org/sleuthkit/autopsy/corecomponents/Bundle_ja.properties b/Core/src/org/sleuthkit/autopsy/corecomponents/Bundle_ja.properties index c9172f03a0..7eba327056 100644 --- a/Core/src/org/sleuthkit/autopsy/corecomponents/Bundle_ja.properties +++ b/Core/src/org/sleuthkit/autopsy/corecomponents/Bundle_ja.properties @@ -121,12 +121,12 @@ DataResultPanel.matchLabel.text=\u7d50\u679c DataResultPanel.numberOfChildNodesLabel.text=0 DataResultPanel.descriptionLabel.text=\u30c7\u30a3\u30ec\u30af\u30c8\u30ea\u30d1\u30b9 ViewPreferencesPanel.selectFileLabel.text=\u30d5\u30a1\u30a4\u30eb\u3092\u9078\u629e\u3059\u308b\u5834\u5408\uff1a +ViewPreferencesPanel.useLocalTimeRadioButton.text=\u30ed\u30fc\u30ab\u30eb\u30bf\u30a4\u30e0\u30be\u30fc\u30f3\u3092\u4f7f\u7528 ViewPreferencesPanel.displayTimeLabel.text=\u6642\u9593\u3092\u8868\u793a\u3059\u308b\u5834\u5408\uff1a +ViewPreferencesPanel.viewsHideKnownCheckbox.text=\u30d3\u30e5\u30fc\u30a8\u30ea\u30a2 +ViewPreferencesPanel.dataSourcesHideKnownCheckbox.text=\u30c7\u30fc\u30bf\u30bd\u30fc\u30b9\u30a8\u30ea\u30a2\uff08\u30c7\u30a3\u30ec\u30af\u30c8\u30ea\u968e\u5c64\uff09 ViewPreferencesPanel.hideKnownFilesLabel.text=\u65e2\u77e5\u30d5\u30a1\u30a4\u30eb\uff08NIST NSRL\u5185\u306e\uff09\u3092\u6b21\u306b\u96a0\u3059\uff1a +ViewPreferencesPanel.keepCurrentViewerRadioButton.toolTipText=\u4f8b\u3048\u3070\u3001JPEG\u304c\u9078\u629e\u3055\u308c\u305f\u5834\u5408\u306b\u305d\u306e\u307e\u307eHEX\u30d3\u30e5\u30fc\u3092\u4f7f\u7528\u3002 +ViewPreferencesPanel.keepCurrentViewerRadioButton.text=\u305d\u306e\u307e\u307e\u540c\u3058\u30d5\u30a1\u30a4\u30eb\u30d3\u30e5\u30fc\u30a2\u3092\u4f7f\u7528 ViewPreferencesPanel.useBestViewerRadioButton.toolTipText=\u4f8b\u3048\u3070\u3001JPEG\u304c\u9078\u629e\u3055\u308c\u305f\u5834\u5408\u306b\u306fHEX\u304b\u3089\u30e1\u30c7\u30a3\u30a2\u306b\u5909\u66f4\u3059\u308b\u3002 ViewPreferencesPanel.useBestViewerRadioButton.text=\u6700\u3082\u5c02\u9580\u7684\u306a\u30d5\u30a1\u30a4\u30eb\u30d3\u30e5\u30fc\u30a2\u306b\u5909\u66f4 -ViewPreferencesPanel.keepCurrentViewerRadioButton.text=\u305d\u306e\u307e\u307e\u540c\u3058\u30d5\u30a1\u30a4\u30eb\u30d3\u30e5\u30fc\u30a2\u3092\u4f7f\u7528 -ViewPreferencesPanel.keepCurrentViewerRadioButton.toolTipText=\u4f8b\u3048\u3070\u3001JPEG\u304c\u9078\u629e\u3055\u308c\u305f\u5834\u5408\u306b\u305d\u306e\u307e\u307eHEX\u30d3\u30e5\u30fc\u3092\u4f7f\u7528\u3002 -ViewPreferencesPanel.useLocalTimeRadioButton.text=\u30ed\u30fc\u30ab\u30eb\u30bf\u30a4\u30e0\u30be\u30fc\u30f3\u3092\u4f7f\u7528 -ViewPreferencesPanel.dataSourcesHideKnownCheckbox.text=\u30c7\u30fc\u30bf\u30bd\u30fc\u30b9\u30a8\u30ea\u30a2\uff08\u30c7\u30a3\u30ec\u30af\u30c8\u30ea\u968e\u5c64\uff09 -ViewPreferencesPanel.viewsHideKnownCheckbox.text=\u30d3\u30e5\u30fc\u30a8\u30ea\u30a2 diff --git a/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.form b/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.form index 89ffab91d8..3e2ab0eca0 100755 --- a/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.form +++ b/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.form @@ -31,11 +31,19 @@ + + + + + + + + @@ -82,61 +90,67 @@ - - - - - - - + + + + - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + - + - - - - - + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + + + + @@ -160,6 +174,18 @@ + + + + + + + + + + + + @@ -171,29 +197,19 @@ - - + + + + + + + + - - - - - - - - - - - - - - - - - - + + @@ -303,14 +319,14 @@ - + - + - + @@ -330,16 +346,6 @@ - - - - - - - - - - @@ -347,6 +353,17 @@ + + + + + + + + + + + @@ -364,23 +381,52 @@ - + + + + + + + + + + + + + + + + + + + + + + + - + + + + + + + + + + + + + + + - + - - - - - - - diff --git a/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.java b/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.java index b96e97d133..1302067c3d 100755 --- a/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.java +++ b/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.java @@ -19,12 +19,14 @@ package org.sleuthkit.autopsy.corecomponents; import java.util.Objects; +import java.util.TimeZone; import javax.swing.JPanel; import org.netbeans.spi.options.OptionsPanelController; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.CasePreferences; import org.sleuthkit.autopsy.centralrepository.datamodel.EamDbUtil; import org.sleuthkit.autopsy.core.UserPreferences; +import org.sleuthkit.autopsy.coreutils.TimeZoneUtils; import org.sleuthkit.autopsy.deletedFiles.DeletedFilePreferences; import org.sleuthkit.autopsy.directorytree.DirectoryTreeTopComponent; import org.sleuthkit.autopsy.texttranslation.TextTranslationService; @@ -46,10 +48,7 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { initComponents(); this.immediateUpdates = immediateUpdates; - //If there is not Text Translator implementation, then hide these buttons - //from the user. - TextTranslationService tts = TextTranslationService.getInstance(); - translateNamesRadioButton.setEnabled(tts.hasProvider()); + this.timeZoneList.setListData(TimeZoneUtils.createTimeZoneList().stream().toArray(String[]::new)); } @Override @@ -60,8 +59,10 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { useBestViewerRadioButton.setSelected(!keepPreferredViewer); boolean useLocalTime = UserPreferences.displayTimesInLocalTime(); + timeZoneList.setEnabled(!useLocalTime); + timeZoneList.setSelectedValue(TimeZoneUtils.createTimeZoneString(TimeZone.getTimeZone(UserPreferences.getTimeZoneForDisplays())), true); useLocalTimeRadioButton.setSelected(useLocalTime); - useGMTTimeRadioButton.setSelected(!useLocalTime); + useAnotherTimeRadioButton.setSelected(!useLocalTime); dataSourcesHideKnownCheckbox.setSelected(UserPreferences.hideKnownFilesInDataSourcesTree()); viewsHideKnownCheckbox.setSelected(UserPreferences.hideKnownFilesInViewsTree()); @@ -70,10 +71,14 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { viewsHideSlackCheckbox.setSelected(UserPreferences.hideSlackFilesInViewsTree()); commentsOccurencesColumnsCheckbox.setEnabled(EamDbUtil.useCentralRepo()); + hideColumnWrapAroundText.setEnabled(EamDbUtil.useCentralRepo()); commentsOccurencesColumnsCheckbox.setSelected(UserPreferences.hideCentralRepoCommentsAndOccurrences()); deletedFilesLimitCheckbox.setSelected(DeletedFilePreferences.getDefault().getShouldLimitDeletedFiles()); - translateNamesRadioButton.setSelected(UserPreferences.displayTranslatedFileNames()); + translateNamesInTableRadioButton.setSelected(UserPreferences.displayTranslatedFileNames()); + + TextTranslationService tts = TextTranslationService.getInstance(); + translateNamesInTableRadioButton.setEnabled(tts.hasProvider()); // Current Case Settings boolean caseIsOpen = Case.isCaseOpen(); @@ -91,13 +96,16 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { public void store() { UserPreferences.setKeepPreferredContentViewer(keepCurrentViewerRadioButton.isSelected()); UserPreferences.setDisplayTimesInLocalTime(useLocalTimeRadioButton.isSelected()); + if (useAnotherTimeRadioButton.isSelected()) { + UserPreferences.setTimeZoneForDisplays(timeZoneList.getSelectedValue().substring(11).trim()); + } UserPreferences.setHideKnownFilesInDataSourcesTree(dataSourcesHideKnownCheckbox.isSelected()); UserPreferences.setHideKnownFilesInViewsTree(viewsHideKnownCheckbox.isSelected()); UserPreferences.setHideSlackFilesInDataSourcesTree(dataSourcesHideSlackCheckbox.isSelected()); UserPreferences.setHideSlackFilesInViewsTree(viewsHideSlackCheckbox.isSelected()); UserPreferences.setShowOnlyCurrentUserTags(hideOtherUsersTagsCheckbox.isSelected()); UserPreferences.setHideCentralRepoCommentsAndOccurrences(commentsOccurencesColumnsCheckbox.isSelected()); - UserPreferences.setDisplayTranslatedFileNames(translateNamesRadioButton.isSelected()); + UserPreferences.setDisplayTranslatedFileNames(translateNamesInTableRadioButton.isSelected()); storeGroupItemsInTreeByDataSource(); @@ -143,21 +151,27 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { viewsHideSlackCheckbox = new javax.swing.JCheckBox(); displayTimeLabel = new javax.swing.JLabel(); useLocalTimeRadioButton = new javax.swing.JRadioButton(); - useGMTTimeRadioButton = new javax.swing.JRadioButton(); + useAnotherTimeRadioButton = new javax.swing.JRadioButton(); hideOtherUsersTagsCheckbox = new javax.swing.JCheckBox(); hideOtherUsersTagsLabel = new javax.swing.JLabel(); - commentsOccurencesColumnsCheckbox = new javax.swing.JCheckBox(); centralRepoLabel = new javax.swing.JLabel(); + commentsOccurencesColumnsCheckbox = new javax.swing.JCheckBox(); deletedFilesLimitCheckbox = new javax.swing.JCheckBox(); deletedFilesLimitLabel = new javax.swing.JLabel(); - translateNamesRadioButton = new javax.swing.JRadioButton(); - translateTextLabel = new javax.swing.JLabel(); + jScrollPane1 = new javax.swing.JScrollPane(); + timeZoneList = new javax.swing.JList<>(); + TranslateTextLabel = new javax.swing.JLabel(); + hideColumnWrapAroundText = new javax.swing.JLabel(); + translateNamesInTableRadioButton = new javax.swing.JRadioButton(); currentCaseSettingsPanel = new javax.swing.JPanel(); groupByDataSourceCheckbox = new javax.swing.JCheckBox(); currentSessionSettingsPanel = new javax.swing.JPanel(); hideRejectedResultsCheckbox = new javax.swing.JCheckBox(); viewPreferencesScrollPane.setBorder(null); + viewPreferencesScrollPane.setPreferredSize(new java.awt.Dimension(625, 452)); + + viewPreferencesPanel.setPreferredSize(new java.awt.Dimension(625, 452)); globalSettingsPanel.setBorder(javax.swing.BorderFactory.createTitledBorder(org.openide.util.NbBundle.getMessage(ViewPreferencesPanel.class, "ViewPreferencesPanel.globalSettingsPanel.border.title"))); // NOI18N @@ -220,10 +234,10 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { } }); - org.openide.awt.Mnemonics.setLocalizedText(useGMTTimeRadioButton, org.openide.util.NbBundle.getMessage(ViewPreferencesPanel.class, "ViewPreferencesPanel.useGMTTimeRadioButton.text")); // NOI18N - useGMTTimeRadioButton.addActionListener(new java.awt.event.ActionListener() { + org.openide.awt.Mnemonics.setLocalizedText(useAnotherTimeRadioButton, org.openide.util.NbBundle.getMessage(ViewPreferencesPanel.class, "ViewPreferencesPanel.useAnotherTimeRadioButton.text")); // NOI18N + useAnotherTimeRadioButton.addActionListener(new java.awt.event.ActionListener() { public void actionPerformed(java.awt.event.ActionEvent evt) { - useGMTTimeRadioButtonActionPerformed(evt); + useAnotherTimeRadioButtonActionPerformed(evt); } }); @@ -236,15 +250,16 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { org.openide.awt.Mnemonics.setLocalizedText(hideOtherUsersTagsLabel, org.openide.util.NbBundle.getMessage(ViewPreferencesPanel.class, "ViewPreferencesPanel.hideOtherUsersTagsLabel.text")); // NOI18N + org.openide.awt.Mnemonics.setLocalizedText(centralRepoLabel, org.openide.util.NbBundle.getMessage(ViewPreferencesPanel.class, "ViewPreferencesPanel.centralRepoLabel.text")); // NOI18N + org.openide.awt.Mnemonics.setLocalizedText(commentsOccurencesColumnsCheckbox, org.openide.util.NbBundle.getMessage(ViewPreferencesPanel.class, "ViewPreferencesPanel.commentsOccurencesColumnsCheckbox.text")); // NOI18N + commentsOccurencesColumnsCheckbox.setHorizontalAlignment(javax.swing.SwingConstants.TRAILING); commentsOccurencesColumnsCheckbox.addActionListener(new java.awt.event.ActionListener() { public void actionPerformed(java.awt.event.ActionEvent evt) { commentsOccurencesColumnsCheckboxActionPerformed(evt); } }); - org.openide.awt.Mnemonics.setLocalizedText(centralRepoLabel, org.openide.util.NbBundle.getMessage(ViewPreferencesPanel.class, "ViewPreferencesPanel.centralRepoLabel.text")); // NOI18N - org.openide.awt.Mnemonics.setLocalizedText(deletedFilesLimitCheckbox, org.openide.util.NbBundle.getMessage(ViewPreferencesPanel.class, "ViewPreferencesPanel.deletedFilesLimitCheckbox.text")); // NOI18N deletedFilesLimitCheckbox.addActionListener(new java.awt.event.ActionListener() { public void actionPerformed(java.awt.event.ActionEvent evt) { @@ -254,14 +269,23 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { org.openide.awt.Mnemonics.setLocalizedText(deletedFilesLimitLabel, org.openide.util.NbBundle.getMessage(ViewPreferencesPanel.class, "ViewPreferencesPanel.deletedFilesLimitLabel.text")); // NOI18N - org.openide.awt.Mnemonics.setLocalizedText(translateNamesRadioButton, org.openide.util.NbBundle.getMessage(ViewPreferencesPanel.class, "ViewPreferencesPanel.translateNamesRadioButton.text")); // NOI18N - translateNamesRadioButton.addActionListener(new java.awt.event.ActionListener() { - public void actionPerformed(java.awt.event.ActionEvent evt) { - translateNamesRadioButtonActionPerformed(evt); + timeZoneList.addListSelectionListener(new javax.swing.event.ListSelectionListener() { + public void valueChanged(javax.swing.event.ListSelectionEvent evt) { + timeZoneListValueChanged(evt); } }); + jScrollPane1.setViewportView(timeZoneList); - org.openide.awt.Mnemonics.setLocalizedText(translateTextLabel, org.openide.util.NbBundle.getMessage(ViewPreferencesPanel.class, "ViewPreferencesPanel.translateTextLabel.text")); // NOI18N + org.openide.awt.Mnemonics.setLocalizedText(TranslateTextLabel, org.openide.util.NbBundle.getMessage(ViewPreferencesPanel.class, "ViewPreferencesPanel.TranslateTextLabel.text")); // NOI18N + + org.openide.awt.Mnemonics.setLocalizedText(hideColumnWrapAroundText, org.openide.util.NbBundle.getMessage(ViewPreferencesPanel.class, "ViewPreferencesPanel.hideColumnWrapAroundText.text")); // NOI18N + + org.openide.awt.Mnemonics.setLocalizedText(translateNamesInTableRadioButton, org.openide.util.NbBundle.getMessage(ViewPreferencesPanel.class, "ViewPreferencesPanel.translateNamesInTableRadioButton.text")); // NOI18N + translateNamesInTableRadioButton.addActionListener(new java.awt.event.ActionListener() { + public void actionPerformed(java.awt.event.ActionEvent evt) { + translateNamesInTableRadioButtonActionPerformed(evt); + } + }); javax.swing.GroupLayout globalSettingsPanelLayout = new javax.swing.GroupLayout(globalSettingsPanel); globalSettingsPanel.setLayout(globalSettingsPanelLayout); @@ -271,46 +295,51 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { .addContainerGap() .addGroup(globalSettingsPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addGroup(globalSettingsPanelLayout.createSequentialGroup() - .addGap(10, 10, 10) - .addGroup(globalSettingsPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addComponent(commentsOccurencesColumnsCheckbox) - .addComponent(hideOtherUsersTagsCheckbox) - .addComponent(deletedFilesLimitCheckbox, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE))) + .addComponent(centralRepoLabel) + .addGap(135, 135, 135) + .addComponent(jScrollPane1, javax.swing.GroupLayout.PREFERRED_SIZE, 272, javax.swing.GroupLayout.PREFERRED_SIZE)) + .addComponent(hideOtherUsersTagsLabel) .addGroup(globalSettingsPanelLayout.createSequentialGroup() .addGroup(globalSettingsPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addComponent(centralRepoLabel) - .addComponent(deletedFilesLimitLabel, javax.swing.GroupLayout.PREFERRED_SIZE, 215, javax.swing.GroupLayout.PREFERRED_SIZE) - .addGroup(globalSettingsPanelLayout.createSequentialGroup() + .addComponent(hideKnownFilesLabel) + .addGroup(globalSettingsPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.TRAILING) .addGroup(globalSettingsPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addComponent(hideKnownFilesLabel) - .addGroup(globalSettingsPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.TRAILING) - .addGroup(globalSettingsPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addGroup(globalSettingsPanelLayout.createSequentialGroup() - .addGap(10, 10, 10) - .addGroup(globalSettingsPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addComponent(dataSourcesHideSlackCheckbox) - .addComponent(viewsHideSlackCheckbox))) - .addComponent(hideSlackFilesLabel)) - .addGroup(globalSettingsPanelLayout.createSequentialGroup() - .addGap(10, 10, 10) - .addGroup(globalSettingsPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addComponent(dataSourcesHideKnownCheckbox) - .addComponent(viewsHideKnownCheckbox)))) - .addComponent(hideOtherUsersTagsLabel)) - .addGap(18, 18, 18) - .addGroup(globalSettingsPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addComponent(translateTextLabel) - .addComponent(displayTimeLabel) .addGroup(globalSettingsPanelLayout.createSequentialGroup() .addGap(10, 10, 10) .addGroup(globalSettingsPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addComponent(keepCurrentViewerRadioButton) - .addComponent(useBestViewerRadioButton) - .addComponent(useGMTTimeRadioButton) - .addComponent(useLocalTimeRadioButton) - .addComponent(translateNamesRadioButton))) - .addComponent(selectFileLabel)))) - .addGap(0, 10, Short.MAX_VALUE))) + .addComponent(dataSourcesHideSlackCheckbox) + .addComponent(viewsHideSlackCheckbox))) + .addComponent(hideSlackFilesLabel)) + .addGroup(globalSettingsPanelLayout.createSequentialGroup() + .addGap(10, 10, 10) + .addGroup(globalSettingsPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addComponent(dataSourcesHideKnownCheckbox) + .addComponent(viewsHideKnownCheckbox)))) + .addGroup(globalSettingsPanelLayout.createSequentialGroup() + .addGap(10, 10, 10) + .addComponent(commentsOccurencesColumnsCheckbox)) + .addGroup(globalSettingsPanelLayout.createSequentialGroup() + .addGap(30, 30, 30) + .addComponent(hideColumnWrapAroundText))) + .addGap(18, 18, 18) + .addGroup(globalSettingsPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addComponent(displayTimeLabel) + .addComponent(selectFileLabel) + .addComponent(TranslateTextLabel, javax.swing.GroupLayout.PREFERRED_SIZE, 120, javax.swing.GroupLayout.PREFERRED_SIZE) + .addGroup(globalSettingsPanelLayout.createSequentialGroup() + .addGap(10, 10, 10) + .addGroup(globalSettingsPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addComponent(keepCurrentViewerRadioButton) + .addComponent(useBestViewerRadioButton) + .addComponent(useLocalTimeRadioButton) + .addComponent(useAnotherTimeRadioButton) + .addComponent(translateNamesInTableRadioButton))))) + .addComponent(deletedFilesLimitLabel, javax.swing.GroupLayout.PREFERRED_SIZE, 215, javax.swing.GroupLayout.PREFERRED_SIZE) + .addGroup(globalSettingsPanelLayout.createSequentialGroup() + .addGap(10, 10, 10) + .addGroup(globalSettingsPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addComponent(hideOtherUsersTagsCheckbox) + .addComponent(deletedFilesLimitCheckbox, javax.swing.GroupLayout.DEFAULT_SIZE, 567, Short.MAX_VALUE)))) .addContainerGap()) ); globalSettingsPanelLayout.setVerticalGroup( @@ -329,7 +358,19 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) .addComponent(dataSourcesHideSlackCheckbox) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) - .addComponent(viewsHideSlackCheckbox)) + .addComponent(viewsHideSlackCheckbox) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) + .addComponent(hideOtherUsersTagsLabel) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addComponent(hideOtherUsersTagsCheckbox) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) + .addComponent(centralRepoLabel) + .addGap(3, 3, 3) + .addComponent(commentsOccurencesColumnsCheckbox, javax.swing.GroupLayout.PREFERRED_SIZE, 18, javax.swing.GroupLayout.PREFERRED_SIZE) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addComponent(hideColumnWrapAroundText) + .addGap(11, 11, 11) + .addComponent(deletedFilesLimitLabel)) .addGroup(globalSettingsPanelLayout.createSequentialGroup() .addComponent(selectFileLabel) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) @@ -341,24 +382,16 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) .addComponent(useLocalTimeRadioButton) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) - .addComponent(useGMTTimeRadioButton))) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) - .addGroup(globalSettingsPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addComponent(translateTextLabel) - .addComponent(hideOtherUsersTagsLabel)) + .addComponent(useAnotherTimeRadioButton) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addComponent(jScrollPane1, javax.swing.GroupLayout.PREFERRED_SIZE, 67, javax.swing.GroupLayout.PREFERRED_SIZE) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addComponent(TranslateTextLabel) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addComponent(translateNamesInTableRadioButton))) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) - .addGroup(globalSettingsPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) - .addComponent(hideOtherUsersTagsCheckbox) - .addComponent(translateNamesRadioButton)) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) - .addComponent(centralRepoLabel) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) - .addComponent(commentsOccurencesColumnsCheckbox) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) - .addComponent(deletedFilesLimitLabel) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) - .addComponent(deletedFilesLimitCheckbox, javax.swing.GroupLayout.PREFERRED_SIZE, 33, javax.swing.GroupLayout.PREFERRED_SIZE) - .addGap(0, 0, 0)) + .addComponent(deletedFilesLimitCheckbox, javax.swing.GroupLayout.PREFERRED_SIZE, 26, javax.swing.GroupLayout.PREFERRED_SIZE) + .addContainerGap(8, Short.MAX_VALUE)) ); currentCaseSettingsPanel.setBorder(javax.swing.BorderFactory.createTitledBorder(org.openide.util.NbBundle.getMessage(ViewPreferencesPanel.class, "ViewPreferencesPanel.currentCaseSettingsPanel.border.title"))); // NOI18N @@ -440,94 +473,14 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { this.setLayout(layout); layout.setHorizontalGroup( layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addComponent(viewPreferencesScrollPane) + .addComponent(viewPreferencesScrollPane, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) ); layout.setVerticalGroup( layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addComponent(viewPreferencesScrollPane) + .addComponent(viewPreferencesScrollPane, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) ); }// //GEN-END:initComponents - private void useBestViewerRadioButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_useBestViewerRadioButtonActionPerformed - useBestViewerRadioButton.setSelected(true); - keepCurrentViewerRadioButton.setSelected(false); - if (immediateUpdates) { - UserPreferences.setKeepPreferredContentViewer(false); - } else { - firePropertyChange(OptionsPanelController.PROP_CHANGED, null, null); - } - }//GEN-LAST:event_useBestViewerRadioButtonActionPerformed - - private void keepCurrentViewerRadioButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_keepCurrentViewerRadioButtonActionPerformed - useBestViewerRadioButton.setSelected(false); - keepCurrentViewerRadioButton.setSelected(true); - if (immediateUpdates) { - UserPreferences.setKeepPreferredContentViewer(true); - } else { - firePropertyChange(OptionsPanelController.PROP_CHANGED, null, null); - } - }//GEN-LAST:event_keepCurrentViewerRadioButtonActionPerformed - - private void useLocalTimeRadioButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_useLocalTimeRadioButtonActionPerformed - useLocalTimeRadioButton.setSelected(true); - useGMTTimeRadioButton.setSelected(false); - if (immediateUpdates) { - UserPreferences.setDisplayTimesInLocalTime(true); - } else { - firePropertyChange(OptionsPanelController.PROP_CHANGED, null, null); - } - }//GEN-LAST:event_useLocalTimeRadioButtonActionPerformed - - private void useGMTTimeRadioButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_useGMTTimeRadioButtonActionPerformed - useLocalTimeRadioButton.setSelected(false); - useGMTTimeRadioButton.setSelected(true); - if (immediateUpdates) { - UserPreferences.setDisplayTimesInLocalTime(false); - } else { - firePropertyChange(OptionsPanelController.PROP_CHANGED, null, null); - } - }//GEN-LAST:event_useGMTTimeRadioButtonActionPerformed - - private void dataSourcesHideKnownCheckboxActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_dataSourcesHideKnownCheckboxActionPerformed - if (immediateUpdates) { - UserPreferences.setHideKnownFilesInDataSourcesTree(dataSourcesHideKnownCheckbox.isSelected()); - } else { - firePropertyChange(OptionsPanelController.PROP_CHANGED, null, null); - } - }//GEN-LAST:event_dataSourcesHideKnownCheckboxActionPerformed - - private void viewsHideKnownCheckboxActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_viewsHideKnownCheckboxActionPerformed - if (immediateUpdates) { - UserPreferences.setHideKnownFilesInViewsTree(viewsHideKnownCheckbox.isSelected()); - } else { - firePropertyChange(OptionsPanelController.PROP_CHANGED, null, null); - } - }//GEN-LAST:event_viewsHideKnownCheckboxActionPerformed - - private void dataSourcesHideSlackCheckboxActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_dataSourcesHideSlackCheckboxActionPerformed - if (immediateUpdates) { - UserPreferences.setHideSlackFilesInDataSourcesTree(dataSourcesHideSlackCheckbox.isSelected()); - } else { - firePropertyChange(OptionsPanelController.PROP_CHANGED, null, null); - } - }//GEN-LAST:event_dataSourcesHideSlackCheckboxActionPerformed - - private void viewsHideSlackCheckboxActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_viewsHideSlackCheckboxActionPerformed - if (immediateUpdates) { - UserPreferences.setHideSlackFilesInViewsTree(viewsHideSlackCheckbox.isSelected()); - } else { - firePropertyChange(OptionsPanelController.PROP_CHANGED, null, null); - } - }//GEN-LAST:event_viewsHideSlackCheckboxActionPerformed - - private void hideOtherUsersTagsCheckboxActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_hideOtherUsersTagsCheckboxActionPerformed - if (immediateUpdates) { - UserPreferences.setShowOnlyCurrentUserTags(hideOtherUsersTagsCheckbox.isSelected()); - } else { - firePropertyChange(OptionsPanelController.PROP_CHANGED, null, null); - } - }//GEN-LAST:event_hideOtherUsersTagsCheckboxActionPerformed - private void groupByDataSourceCheckboxActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_groupByDataSourceCheckboxActionPerformed if (immediateUpdates) { storeGroupItemsInTreeByDataSource(); @@ -544,13 +497,21 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { } }//GEN-LAST:event_hideRejectedResultsCheckboxActionPerformed - private void commentsOccurencesColumnsCheckboxActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_commentsOccurencesColumnsCheckboxActionPerformed + private void translateNamesInTableRadioButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_translateNamesInTableRadioButtonActionPerformed if (immediateUpdates) { - UserPreferences.setHideCentralRepoCommentsAndOccurrences(commentsOccurencesColumnsCheckbox.isSelected()); + UserPreferences.setDisplayTranslatedFileNames(translateNamesInTableRadioButton.isSelected()); } else { firePropertyChange(OptionsPanelController.PROP_CHANGED, null, null); } - }//GEN-LAST:event_commentsOccurencesColumnsCheckboxActionPerformed + }//GEN-LAST:event_translateNamesInTableRadioButtonActionPerformed + + private void timeZoneListValueChanged(javax.swing.event.ListSelectionEvent evt) {//GEN-FIRST:event_timeZoneListValueChanged + if (immediateUpdates && useAnotherTimeRadioButton.isSelected()) { + UserPreferences.setTimeZoneForDisplays(timeZoneList.getSelectedValue().substring(11).trim()); + } else { + firePropertyChange(OptionsPanelController.PROP_CHANGED, null, null); + } + }//GEN-LAST:event_timeZoneListValueChanged private void deletedFilesLimitCheckboxActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_deletedFilesLimitCheckboxActionPerformed if (immediateUpdates) { @@ -560,16 +521,99 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { } }//GEN-LAST:event_deletedFilesLimitCheckboxActionPerformed - private void translateNamesRadioButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_translateNamesRadioButtonActionPerformed + private void commentsOccurencesColumnsCheckboxActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_commentsOccurencesColumnsCheckboxActionPerformed if (immediateUpdates) { - UserPreferences.setDisplayTranslatedFileNames(translateNamesRadioButton.isSelected()); + UserPreferences.setHideCentralRepoCommentsAndOccurrences(commentsOccurencesColumnsCheckbox.isSelected()); } else { firePropertyChange(OptionsPanelController.PROP_CHANGED, null, null); } - }//GEN-LAST:event_translateNamesRadioButtonActionPerformed + }//GEN-LAST:event_commentsOccurencesColumnsCheckboxActionPerformed + + private void hideOtherUsersTagsCheckboxActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_hideOtherUsersTagsCheckboxActionPerformed + if (immediateUpdates) { + UserPreferences.setShowOnlyCurrentUserTags(hideOtherUsersTagsCheckbox.isSelected()); + } else { + firePropertyChange(OptionsPanelController.PROP_CHANGED, null, null); + } + }//GEN-LAST:event_hideOtherUsersTagsCheckboxActionPerformed + + private void useAnotherTimeRadioButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_useAnotherTimeRadioButtonActionPerformed + useLocalTimeRadioButton.setSelected(false); + useAnotherTimeRadioButton.setSelected(true); + timeZoneList.setEnabled(true); + if (immediateUpdates) { + UserPreferences.setDisplayTimesInLocalTime(false); + } else { + firePropertyChange(OptionsPanelController.PROP_CHANGED, null, null); + } + }//GEN-LAST:event_useAnotherTimeRadioButtonActionPerformed + + private void useLocalTimeRadioButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_useLocalTimeRadioButtonActionPerformed + useLocalTimeRadioButton.setSelected(true); + useAnotherTimeRadioButton.setSelected(false); + timeZoneList.setEnabled(false); + if (immediateUpdates) { + UserPreferences.setDisplayTimesInLocalTime(true); + } else { + firePropertyChange(OptionsPanelController.PROP_CHANGED, null, null); + } + }//GEN-LAST:event_useLocalTimeRadioButtonActionPerformed + + private void viewsHideSlackCheckboxActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_viewsHideSlackCheckboxActionPerformed + if (immediateUpdates) { + UserPreferences.setHideSlackFilesInViewsTree(viewsHideSlackCheckbox.isSelected()); + } else { + firePropertyChange(OptionsPanelController.PROP_CHANGED, null, null); + } + }//GEN-LAST:event_viewsHideSlackCheckboxActionPerformed + + private void dataSourcesHideSlackCheckboxActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_dataSourcesHideSlackCheckboxActionPerformed + if (immediateUpdates) { + UserPreferences.setHideSlackFilesInDataSourcesTree(dataSourcesHideSlackCheckbox.isSelected()); + } else { + firePropertyChange(OptionsPanelController.PROP_CHANGED, null, null); + } + }//GEN-LAST:event_dataSourcesHideSlackCheckboxActionPerformed + + private void viewsHideKnownCheckboxActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_viewsHideKnownCheckboxActionPerformed + if (immediateUpdates) { + UserPreferences.setHideKnownFilesInViewsTree(viewsHideKnownCheckbox.isSelected()); + } else { + firePropertyChange(OptionsPanelController.PROP_CHANGED, null, null); + } + }//GEN-LAST:event_viewsHideKnownCheckboxActionPerformed + + private void dataSourcesHideKnownCheckboxActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_dataSourcesHideKnownCheckboxActionPerformed + if (immediateUpdates) { + UserPreferences.setHideKnownFilesInDataSourcesTree(dataSourcesHideKnownCheckbox.isSelected()); + } else { + firePropertyChange(OptionsPanelController.PROP_CHANGED, null, null); + } + }//GEN-LAST:event_dataSourcesHideKnownCheckboxActionPerformed + + private void keepCurrentViewerRadioButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_keepCurrentViewerRadioButtonActionPerformed + useBestViewerRadioButton.setSelected(false); + keepCurrentViewerRadioButton.setSelected(true); + if (immediateUpdates) { + UserPreferences.setKeepPreferredContentViewer(true); + } else { + firePropertyChange(OptionsPanelController.PROP_CHANGED, null, null); + } + }//GEN-LAST:event_keepCurrentViewerRadioButtonActionPerformed + + private void useBestViewerRadioButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_useBestViewerRadioButtonActionPerformed + useBestViewerRadioButton.setSelected(true); + keepCurrentViewerRadioButton.setSelected(false); + if (immediateUpdates) { + UserPreferences.setKeepPreferredContentViewer(false); + } else { + firePropertyChange(OptionsPanelController.PROP_CHANGED, null, null); + } + }//GEN-LAST:event_useBestViewerRadioButtonActionPerformed // Variables declaration - do not modify//GEN-BEGIN:variables + private javax.swing.JLabel TranslateTextLabel; private javax.swing.JLabel centralRepoLabel; private javax.swing.JCheckBox commentsOccurencesColumnsCheckbox; private javax.swing.JPanel currentCaseSettingsPanel; @@ -581,21 +625,23 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { private javax.swing.JLabel displayTimeLabel; private javax.swing.JPanel globalSettingsPanel; private javax.swing.JCheckBox groupByDataSourceCheckbox; + private javax.swing.JLabel hideColumnWrapAroundText; private javax.swing.JLabel hideKnownFilesLabel; private javax.swing.JCheckBox hideOtherUsersTagsCheckbox; private javax.swing.JLabel hideOtherUsersTagsLabel; private javax.swing.JCheckBox hideRejectedResultsCheckbox; private javax.swing.JLabel hideSlackFilesLabel; + private javax.swing.JScrollPane jScrollPane1; private javax.swing.JRadioButton keepCurrentViewerRadioButton; private javax.swing.JLabel selectFileLabel; - private javax.swing.JRadioButton translateNamesRadioButton; - private javax.swing.JLabel translateTextLabel; + private javax.swing.JList timeZoneList; + private javax.swing.JRadioButton translateNamesInTableRadioButton; + private javax.swing.JRadioButton useAnotherTimeRadioButton; private javax.swing.JRadioButton useBestViewerRadioButton; - private javax.swing.JRadioButton useGMTTimeRadioButton; private javax.swing.JRadioButton useLocalTimeRadioButton; private javax.swing.JPanel viewPreferencesPanel; private javax.swing.JScrollPane viewPreferencesScrollPane; private javax.swing.JCheckBox viewsHideKnownCheckbox; private javax.swing.JCheckBox viewsHideSlackCheckbox; // End of variables declaration//GEN-END:variables -} +} \ No newline at end of file From 645ab33f609bc1b9ae75fed66d0eaa73f6ad3b8f Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Thu, 15 Nov 2018 15:38:34 -0500 Subject: [PATCH 110/129] Removed bundle text not being used --- .../sleuthkit/autopsy/corecomponents/Bundle.properties | 1 - .../autopsy/corecomponents/ViewPreferencesPanel.form | 8 ++++---- .../autopsy/corecomponents/ViewPreferencesPanel.java | 10 +++++----- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/corecomponents/Bundle.properties b/Core/src/org/sleuthkit/autopsy/corecomponents/Bundle.properties index 2267791cbd..8da774be7e 100644 --- a/Core/src/org/sleuthkit/autopsy/corecomponents/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/corecomponents/Bundle.properties @@ -174,7 +174,6 @@ ViewPreferencesPanel.hideRejectedResultsCheckbox.text=Hide rejected results ViewPreferencesPanel.translateTextLabel.text=Translate text in the: ViewPreferencesPanel.globalSettingsPanel.border.title=Global Settings ViewPreferencesPanel.translateNamesInTableRadioButton.text=Table -ViewPreferencesPanel.TranslateTextLabel.text=Translate text in the: ViewPreferencesPanel.deletedFilesLimitLabel.text=Limit number of deleted files displayed: ViewPreferencesPanel.deletedFilesLimitCheckbox.text=Limit to 10,000 ViewPreferencesPanel.commentsOccurencesColumnsCheckbox.text=C(omments) and O(ccurences) columns diff --git a/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.form b/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.form index 3e2ab0eca0..e9f6f51b04 100755 --- a/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.form +++ b/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.form @@ -131,7 +131,7 @@ - + @@ -202,7 +202,7 @@ - + @@ -403,10 +403,10 @@ - + - + diff --git a/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.java b/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.java index 1302067c3d..3f6d3e6a34 100755 --- a/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.java +++ b/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.java @@ -160,7 +160,7 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { deletedFilesLimitLabel = new javax.swing.JLabel(); jScrollPane1 = new javax.swing.JScrollPane(); timeZoneList = new javax.swing.JList<>(); - TranslateTextLabel = new javax.swing.JLabel(); + translateTextLabel = new javax.swing.JLabel(); hideColumnWrapAroundText = new javax.swing.JLabel(); translateNamesInTableRadioButton = new javax.swing.JRadioButton(); currentCaseSettingsPanel = new javax.swing.JPanel(); @@ -276,7 +276,7 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { }); jScrollPane1.setViewportView(timeZoneList); - org.openide.awt.Mnemonics.setLocalizedText(TranslateTextLabel, org.openide.util.NbBundle.getMessage(ViewPreferencesPanel.class, "ViewPreferencesPanel.TranslateTextLabel.text")); // NOI18N + org.openide.awt.Mnemonics.setLocalizedText(translateTextLabel, org.openide.util.NbBundle.getMessage(ViewPreferencesPanel.class, "ViewPreferencesPanel.translateTextLabel.text")); // NOI18N org.openide.awt.Mnemonics.setLocalizedText(hideColumnWrapAroundText, org.openide.util.NbBundle.getMessage(ViewPreferencesPanel.class, "ViewPreferencesPanel.hideColumnWrapAroundText.text")); // NOI18N @@ -325,7 +325,7 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { .addGroup(globalSettingsPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addComponent(displayTimeLabel) .addComponent(selectFileLabel) - .addComponent(TranslateTextLabel, javax.swing.GroupLayout.PREFERRED_SIZE, 120, javax.swing.GroupLayout.PREFERRED_SIZE) + .addComponent(translateTextLabel, javax.swing.GroupLayout.PREFERRED_SIZE, 120, javax.swing.GroupLayout.PREFERRED_SIZE) .addGroup(globalSettingsPanelLayout.createSequentialGroup() .addGap(10, 10, 10) .addGroup(globalSettingsPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) @@ -386,7 +386,7 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) .addComponent(jScrollPane1, javax.swing.GroupLayout.PREFERRED_SIZE, 67, javax.swing.GroupLayout.PREFERRED_SIZE) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) - .addComponent(TranslateTextLabel) + .addComponent(translateTextLabel) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) .addComponent(translateNamesInTableRadioButton))) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) @@ -613,7 +613,6 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { // Variables declaration - do not modify//GEN-BEGIN:variables - private javax.swing.JLabel TranslateTextLabel; private javax.swing.JLabel centralRepoLabel; private javax.swing.JCheckBox commentsOccurencesColumnsCheckbox; private javax.swing.JPanel currentCaseSettingsPanel; @@ -636,6 +635,7 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { private javax.swing.JLabel selectFileLabel; private javax.swing.JList timeZoneList; private javax.swing.JRadioButton translateNamesInTableRadioButton; + private javax.swing.JLabel translateTextLabel; private javax.swing.JRadioButton useAnotherTimeRadioButton; private javax.swing.JRadioButton useBestViewerRadioButton; private javax.swing.JRadioButton useLocalTimeRadioButton; From c71db47b49669a74ee44aec39cd141f93c2ddb71 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Thu, 15 Nov 2018 15:47:18 -0500 Subject: [PATCH 111/129] Codacy suggestions --- .../autopsy/corecomponents/Bundle.properties | 2 +- .../corecomponents/ViewPreferencesPanel.form | 8 +-- .../corecomponents/ViewPreferencesPanel.java | 12 ++-- .../datamodel/AbstractAbstractFileNode.java | 71 +++++++++---------- 4 files changed, 46 insertions(+), 47 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/corecomponents/Bundle.properties b/Core/src/org/sleuthkit/autopsy/corecomponents/Bundle.properties index 8da774be7e..f07d7b911f 100644 --- a/Core/src/org/sleuthkit/autopsy/corecomponents/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/corecomponents/Bundle.properties @@ -194,4 +194,4 @@ ViewPreferencesPanel.keepCurrentViewerRadioButton.text=Stay on the same file vie ViewPreferencesPanel.useBestViewerRadioButton.toolTipText=For example, change from Hex to Media when a JPEG is selected. ViewPreferencesPanel.useBestViewerRadioButton.text=Change to the most specific file viewer ViewPreferencesPanel.selectFileLabel.text=When selecting a file: -ViewPreferencesPanel.hideColumnWrapAroundText.text=to reduce loading times +ViewPreferencesPanel.commentsOccurencesColumnWrapAroundText.text=to reduce loading times diff --git a/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.form b/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.form index e9f6f51b04..4644a3db6c 100755 --- a/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.form +++ b/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.form @@ -124,7 +124,7 @@ - + @@ -183,7 +183,7 @@ - + @@ -410,10 +410,10 @@ - + - + diff --git a/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.java b/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.java index 3f6d3e6a34..8d7326f913 100755 --- a/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.java +++ b/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.java @@ -71,7 +71,7 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { viewsHideSlackCheckbox.setSelected(UserPreferences.hideSlackFilesInViewsTree()); commentsOccurencesColumnsCheckbox.setEnabled(EamDbUtil.useCentralRepo()); - hideColumnWrapAroundText.setEnabled(EamDbUtil.useCentralRepo()); + commentsOccurencesColumnWrapAroundText.setEnabled(EamDbUtil.useCentralRepo()); commentsOccurencesColumnsCheckbox.setSelected(UserPreferences.hideCentralRepoCommentsAndOccurrences()); deletedFilesLimitCheckbox.setSelected(DeletedFilePreferences.getDefault().getShouldLimitDeletedFiles()); @@ -161,7 +161,7 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { jScrollPane1 = new javax.swing.JScrollPane(); timeZoneList = new javax.swing.JList<>(); translateTextLabel = new javax.swing.JLabel(); - hideColumnWrapAroundText = new javax.swing.JLabel(); + commentsOccurencesColumnWrapAroundText = new javax.swing.JLabel(); translateNamesInTableRadioButton = new javax.swing.JRadioButton(); currentCaseSettingsPanel = new javax.swing.JPanel(); groupByDataSourceCheckbox = new javax.swing.JCheckBox(); @@ -278,7 +278,7 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { org.openide.awt.Mnemonics.setLocalizedText(translateTextLabel, org.openide.util.NbBundle.getMessage(ViewPreferencesPanel.class, "ViewPreferencesPanel.translateTextLabel.text")); // NOI18N - org.openide.awt.Mnemonics.setLocalizedText(hideColumnWrapAroundText, org.openide.util.NbBundle.getMessage(ViewPreferencesPanel.class, "ViewPreferencesPanel.hideColumnWrapAroundText.text")); // NOI18N + org.openide.awt.Mnemonics.setLocalizedText(commentsOccurencesColumnWrapAroundText, org.openide.util.NbBundle.getMessage(ViewPreferencesPanel.class, "ViewPreferencesPanel.commentsOccurencesColumnWrapAroundText.text")); // NOI18N org.openide.awt.Mnemonics.setLocalizedText(translateNamesInTableRadioButton, org.openide.util.NbBundle.getMessage(ViewPreferencesPanel.class, "ViewPreferencesPanel.translateNamesInTableRadioButton.text")); // NOI18N translateNamesInTableRadioButton.addActionListener(new java.awt.event.ActionListener() { @@ -320,7 +320,7 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { .addComponent(commentsOccurencesColumnsCheckbox)) .addGroup(globalSettingsPanelLayout.createSequentialGroup() .addGap(30, 30, 30) - .addComponent(hideColumnWrapAroundText))) + .addComponent(commentsOccurencesColumnWrapAroundText))) .addGap(18, 18, 18) .addGroup(globalSettingsPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) .addComponent(displayTimeLabel) @@ -368,7 +368,7 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { .addGap(3, 3, 3) .addComponent(commentsOccurencesColumnsCheckbox, javax.swing.GroupLayout.PREFERRED_SIZE, 18, javax.swing.GroupLayout.PREFERRED_SIZE) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) - .addComponent(hideColumnWrapAroundText) + .addComponent(commentsOccurencesColumnWrapAroundText) .addGap(11, 11, 11) .addComponent(deletedFilesLimitLabel)) .addGroup(globalSettingsPanelLayout.createSequentialGroup() @@ -614,6 +614,7 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { // Variables declaration - do not modify//GEN-BEGIN:variables private javax.swing.JLabel centralRepoLabel; + private javax.swing.JLabel commentsOccurencesColumnWrapAroundText; private javax.swing.JCheckBox commentsOccurencesColumnsCheckbox; private javax.swing.JPanel currentCaseSettingsPanel; private javax.swing.JPanel currentSessionSettingsPanel; @@ -624,7 +625,6 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { private javax.swing.JLabel displayTimeLabel; private javax.swing.JPanel globalSettingsPanel; private javax.swing.JCheckBox groupByDataSourceCheckbox; - private javax.swing.JLabel hideColumnWrapAroundText; private javax.swing.JLabel hideKnownFilesLabel; private javax.swing.JCheckBox hideOtherUsersTagsCheckbox; private javax.swing.JLabel hideOtherUsersTagsLabel; diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java index 5890fea0cf..03a733faaa 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java @@ -370,42 +370,41 @@ public abstract class AbstractAbstractFileNode extends A * Creates and populates a list of properties for this nodes property sheet. */ private List> getProperties() { - List> properties = new ArrayList>() {{ - add(new NodeProperty<>(NAME.toString(), NAME.toString(), NO_DESCR, getContentDisplayName(content))); - /* - * Initialize dummy place holder properties for Translation, - * Score, Comment, and Occurrences). At the bottom, we kick off a - * background task that promises to update these values. - */ - if (UserPreferences.displayTranslatedFileNames()) { - add(new NodeProperty<>(TRANSLATION.toString(), TRANSLATION.toString(), NO_DESCR, "")); - } - add(new NodeProperty<>(SCORE.toString(), SCORE.toString(), NO_DESCR, "")); - add(new NodeProperty<>(COMMENT.toString(), COMMENT.toString(), NO_DESCR, "")); - if (!UserPreferences.hideCentralRepoCommentsAndOccurrences()) { - add(new NodeProperty<>(OCCURRENCES.toString(), OCCURRENCES.toString(), NO_DESCR, "")); - } - add(new NodeProperty<>(LOCATION.toString(), LOCATION.toString(), NO_DESCR, getContentPath(content))); - add(new NodeProperty<>(MOD_TIME.toString(), MOD_TIME.toString(), NO_DESCR, ContentUtils.getStringTime(content.getMtime(), content))); - add(new NodeProperty<>(CHANGED_TIME.toString(), CHANGED_TIME.toString(), NO_DESCR, ContentUtils.getStringTime(content.getCtime(), content))); - add(new NodeProperty<>(ACCESS_TIME.toString(), ACCESS_TIME.toString(), NO_DESCR, ContentUtils.getStringTime(content.getAtime(), content))); - add(new NodeProperty<>(CREATED_TIME.toString(), CREATED_TIME.toString(), NO_DESCR, ContentUtils.getStringTime(content.getCrtime(), content))); - add(new NodeProperty<>(SIZE.toString(), SIZE.toString(), NO_DESCR, content.getSize())); - add(new NodeProperty<>(FLAGS_DIR.toString(), FLAGS_DIR.toString(), NO_DESCR, content.getDirFlagAsString())); - add(new NodeProperty<>(FLAGS_META.toString(), FLAGS_META.toString(), NO_DESCR, content.getMetaFlagsAsString())); - add(new NodeProperty<>(MODE.toString(), MODE.toString(), NO_DESCR, content.getModesAsString())); - add(new NodeProperty<>(USER_ID.toString(), USER_ID.toString(), NO_DESCR, content.getUid())); - add(new NodeProperty<>(GROUP_ID.toString(), GROUP_ID.toString(), NO_DESCR, content.getGid())); - add(new NodeProperty<>(META_ADDR.toString(), META_ADDR.toString(), NO_DESCR, content.getMetaAddr())); - add(new NodeProperty<>(ATTR_ADDR.toString(), ATTR_ADDR.toString(), NO_DESCR, content.getAttrType().getValue() + "-" + content.getAttributeId())); - add(new NodeProperty<>(TYPE_DIR.toString(), TYPE_DIR.toString(), NO_DESCR, content.getDirType().getLabel())); - add(new NodeProperty<>(TYPE_META.toString(), TYPE_META.toString(), NO_DESCR, content.getMetaType().toString())); - add(new NodeProperty<>(KNOWN.toString(), KNOWN.toString(), NO_DESCR, content.getKnown().getName())); - add(new NodeProperty<>(MD5HASH.toString(), MD5HASH.toString(), NO_DESCR, StringUtils.defaultString(content.getMd5Hash()))); - add(new NodeProperty<>(ObjectID.toString(), ObjectID.toString(), NO_DESCR, content.getId())); - add(new NodeProperty<>(MIMETYPE.toString(), MIMETYPE.toString(), NO_DESCR, StringUtils.defaultString(content.getMIMEType()))); - add(new NodeProperty<>(EXTENSION.toString(), EXTENSION.toString(), NO_DESCR, content.getNameExtension())); - }}; + List> properties = new ArrayList<>(); + properties.add(new NodeProperty<>(NAME.toString(), NAME.toString(), NO_DESCR, getContentDisplayName(content))); + /* + * Initialize dummy place holder properties for Translation, + * Score, Comment, and Occurrences). At the bottom, we kick off a + * background task that promises to update these values. + */ + if (UserPreferences.displayTranslatedFileNames()) { + properties.add(new NodeProperty<>(TRANSLATION.toString(), TRANSLATION.toString(), NO_DESCR, "")); + } + properties.add(new NodeProperty<>(SCORE.toString(), SCORE.toString(), NO_DESCR, "")); + properties.add(new NodeProperty<>(COMMENT.toString(), COMMENT.toString(), NO_DESCR, "")); + if (!UserPreferences.hideCentralRepoCommentsAndOccurrences()) { + properties.add(new NodeProperty<>(OCCURRENCES.toString(), OCCURRENCES.toString(), NO_DESCR, "")); + } + properties.add(new NodeProperty<>(LOCATION.toString(), LOCATION.toString(), NO_DESCR, getContentPath(content))); + properties.add(new NodeProperty<>(MOD_TIME.toString(), MOD_TIME.toString(), NO_DESCR, ContentUtils.getStringTime(content.getMtime(), content))); + properties.add(new NodeProperty<>(CHANGED_TIME.toString(), CHANGED_TIME.toString(), NO_DESCR, ContentUtils.getStringTime(content.getCtime(), content))); + properties.add(new NodeProperty<>(ACCESS_TIME.toString(), ACCESS_TIME.toString(), NO_DESCR, ContentUtils.getStringTime(content.getAtime(), content))); + properties.add(new NodeProperty<>(CREATED_TIME.toString(), CREATED_TIME.toString(), NO_DESCR, ContentUtils.getStringTime(content.getCrtime(), content))); + properties.add(new NodeProperty<>(SIZE.toString(), SIZE.toString(), NO_DESCR, content.getSize())); + properties.add(new NodeProperty<>(FLAGS_DIR.toString(), FLAGS_DIR.toString(), NO_DESCR, content.getDirFlagAsString())); + properties.add(new NodeProperty<>(FLAGS_META.toString(), FLAGS_META.toString(), NO_DESCR, content.getMetaFlagsAsString())); + properties.add(new NodeProperty<>(MODE.toString(), MODE.toString(), NO_DESCR, content.getModesAsString())); + properties.add(new NodeProperty<>(USER_ID.toString(), USER_ID.toString(), NO_DESCR, content.getUid())); + properties.add(new NodeProperty<>(GROUP_ID.toString(), GROUP_ID.toString(), NO_DESCR, content.getGid())); + properties.add(new NodeProperty<>(META_ADDR.toString(), META_ADDR.toString(), NO_DESCR, content.getMetaAddr())); + properties.add(new NodeProperty<>(ATTR_ADDR.toString(), ATTR_ADDR.toString(), NO_DESCR, content.getAttrType().getValue() + "-" + content.getAttributeId())); + properties.add(new NodeProperty<>(TYPE_DIR.toString(), TYPE_DIR.toString(), NO_DESCR, content.getDirType().getLabel())); + properties.add(new NodeProperty<>(TYPE_META.toString(), TYPE_META.toString(), NO_DESCR, content.getMetaType().toString())); + properties.add(new NodeProperty<>(KNOWN.toString(), KNOWN.toString(), NO_DESCR, content.getKnown().getName())); + properties.add(new NodeProperty<>(MD5HASH.toString(), MD5HASH.toString(), NO_DESCR, StringUtils.defaultString(content.getMd5Hash()))); + properties.add(new NodeProperty<>(ObjectID.toString(), ObjectID.toString(), NO_DESCR, content.getId())); + properties.add(new NodeProperty<>(MIMETYPE.toString(), MIMETYPE.toString(), NO_DESCR, StringUtils.defaultString(content.getMIMEType()))); + properties.add(new NodeProperty<>(EXTENSION.toString(), EXTENSION.toString(), NO_DESCR, content.getNameExtension())); return properties; } From 6fe69623c1dc021660e3288aa3f431aa7a69d4ee Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Thu, 15 Nov 2018 15:50:56 -0500 Subject: [PATCH 112/129] Lined up some txt in the PreferencesPanel --- .../sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.form | 2 +- .../sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.form b/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.form index 4644a3db6c..1f9dfc4ceb 100755 --- a/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.form +++ b/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.form @@ -123,7 +123,7 @@ - + diff --git a/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.java b/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.java index 8d7326f913..5678b0f7cb 100755 --- a/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.java +++ b/Core/src/org/sleuthkit/autopsy/corecomponents/ViewPreferencesPanel.java @@ -319,7 +319,7 @@ public class ViewPreferencesPanel extends JPanel implements OptionsPanel { .addGap(10, 10, 10) .addComponent(commentsOccurencesColumnsCheckbox)) .addGroup(globalSettingsPanelLayout.createSequentialGroup() - .addGap(30, 30, 30) + .addGap(32, 32, 32) .addComponent(commentsOccurencesColumnWrapAroundText))) .addGap(18, 18, 18) .addGroup(globalSettingsPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) From 315782e574dc7d151f873ebc3d5308496ddb2c45 Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Fri, 16 Nov 2018 11:47:18 -0500 Subject: [PATCH 113/129] 4354 rename cr object_id column to file_obj_id --- .../datamodel/AbstractSqlEamDb.java | 22 +++++++++---------- .../datamodel/PostgresEamDbSettings.java | 8 +++---- .../datamodel/SqliteEamDbSettings.java | 8 +++---- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java index bddfbea7b1..4cee044504 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java @@ -808,7 +808,7 @@ abstract class AbstractSqlEamDb implements EamDb { String sql = "INSERT INTO " + tableName - + "(case_id, data_source_id, value, file_path, known_status, comment, object_id) " + + "(case_id, data_source_id, value, file_path, known_status, comment, file_obj_id) " + "VALUES ((SELECT id FROM cases WHERE case_uid=? LIMIT 1), " + "(SELECT id FROM data_sources WHERE device_id=? AND case_id=? LIMIT 1), ?, ?, ?, ?, ?) " + getConflictClause(); @@ -903,7 +903,7 @@ abstract class AbstractSqlEamDb implements EamDb { + tableName + ".value," + tableName - + ".object_id," + + ".file_obj_id," + " cases.case_name, cases.case_uid, data_sources.id AS data_source_id, data_sources.name, device_id, file_path, known_status, comment, data_sources.case_id FROM " + tableName + " LEFT JOIN cases ON " @@ -968,7 +968,7 @@ abstract class AbstractSqlEamDb implements EamDb { + tableName + ".value," + tableName - + ".object_id," + + ".file_obj_id," + " cases.case_name, cases.case_uid, data_sources.id AS data_source_id, data_sources.name, device_id, file_path, known_status, comment, data_sources.case_id FROM " + tableName + " LEFT JOIN cases ON " @@ -1235,7 +1235,7 @@ abstract class AbstractSqlEamDb implements EamDb { String sql = "INSERT INTO " + tableName - + " (case_id, data_source_id, value, file_path, known_status, comment, object_id) " + + " (case_id, data_source_id, value, file_path, known_status, comment, file_obj_id) " + "VALUES ((SELECT id FROM cases WHERE case_uid=? LIMIT 1), " + "(SELECT id FROM data_sources WHERE device_id=? AND case_id=? LIMIT 1), ?, ?, ?, ?, ?) " + getConflictClause(); @@ -1481,7 +1481,7 @@ abstract class AbstractSqlEamDb implements EamDb { = "SELECT id, value, file_path, known_status, comment FROM " + tableName + " WHERE case_id=?" - + " AND object_id=?"; + + " AND file_obj_id=?"; preparedStatement = conn.prepareStatement(sql); preparedStatement.setInt(1, correlationCase.getID()); @@ -1707,7 +1707,7 @@ abstract class AbstractSqlEamDb implements EamDb { + tableName + ".value, " + tableName - + ".object_id," + + ".file_obj_id," + "cases.case_name, cases.case_uid, data_sources.id AS data_source_id, data_sources.name, device_id, file_path, known_status, comment, data_sources.case_id FROM " + tableName + " LEFT JOIN cases ON " @@ -1765,7 +1765,7 @@ abstract class AbstractSqlEamDb implements EamDb { String tableName = EamDbUtil.correlationTypeToInstanceTableName(aType); String sql - = "SELECT cases.case_name, cases.case_uid, data_sources.name, device_id, file_path, known_status, comment, data_sources.case_id, id, value, object_id FROM " + = "SELECT cases.case_name, cases.case_uid, data_sources.name, device_id, file_path, known_status, comment, data_sources.case_id, id, value, file_obj_id FROM " + tableName + " LEFT JOIN cases ON " + tableName @@ -3092,7 +3092,7 @@ abstract class AbstractSqlEamDb implements EamDb { resultSet.getString("file_path"), resultSet.getString("comment"), TskData.FileKnown.valueOf(resultSet.getByte("known_status")), - resultSet.getLong("object_id")); + resultSet.getLong("file_obj_id")); } private EamOrganization getEamOrganizationFromResultSet(ResultSet resultSet) throws SQLException { @@ -3217,7 +3217,7 @@ abstract class AbstractSqlEamDb implements EamDb { if (dbSchemaVersion.compareTo(new CaseDbSchemaVersionNumber(1, 2)) < 0) { EamDbPlatformEnum selectedPlatform = EamDbPlatformEnum.getSelectedPlatform(); - final String addObjectIdColumnTemplate = "ALTER TABLE %s ADD COLUMN object_id INTEGER;"; //NON-NLS + final String addObjectIdColumnTemplate = "ALTER TABLE %s ADD COLUMN file_obj_id INTEGER;"; //NON-NLS final String addSsidTableTemplate; final String addCaseIdIndexTemplate; final String addDataSourceIdIndexTemplate; @@ -3267,11 +3267,11 @@ abstract class AbstractSqlEamDb implements EamDb { statement.execute(String.format(addValueIndexTemplate, wirelessNetworksTableInstanceName, wirelessNetworksTableInstanceName)); statement.execute(String.format(addKnownStatusIndexTemplate, wirelessNetworksTableInstanceName, wirelessNetworksTableInstanceName)); statement.execute(String.format(addObjectIdIndexTemplate, wirelessNetworksTableInstanceName, wirelessNetworksTableInstanceName)); - //add object_id column to _instances table which do not already have it + //add file_obj_id column to _instances table which do not already have it String instance_type_dbname; for (CorrelationAttributeInstance.Type type : CorrelationAttributeInstance.getDefaultCorrelationTypes()) { instance_type_dbname = EamDbUtil.correlationTypeToInstanceTableName(type); - if (!doesColumnExist(conn, instance_type_dbname, "object_id")) { + if (!doesColumnExist(conn, instance_type_dbname, "file_obj_id")) { statement.execute(String.format(addObjectIdColumnTemplate, instance_type_dbname)); //NON-NLS } statement.execute(String.format(addObjectIdIndexTemplate, instance_type_dbname, instance_type_dbname)); diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDbSettings.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDbSettings.java index a3e3ba196a..8183f445e3 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDbSettings.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/PostgresEamDbSettings.java @@ -487,7 +487,7 @@ public final class PostgresEamDbSettings { createArtifactInstancesTableTemplate.append("file_path text NOT NULL,"); createArtifactInstancesTableTemplate.append("known_status integer NOT NULL,"); createArtifactInstancesTableTemplate.append("comment text,"); - createArtifactInstancesTableTemplate.append("object_id integer,"); + createArtifactInstancesTableTemplate.append("file_obj_id integer,"); createArtifactInstancesTableTemplate.append("CONSTRAINT %s_multi_unique_ UNIQUE (data_source_id, value, file_path),"); createArtifactInstancesTableTemplate.append("foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL,"); createArtifactInstancesTableTemplate.append("foreign key (data_source_id) references data_sources(id) ON UPDATE SET NULL ON DELETE SET NULL"); @@ -548,16 +548,16 @@ public final class PostgresEamDbSettings { } /** - * Get the template for creating an index on the object_id column of an + * Get the template for creating an index on the file_obj_id column of an * instance table. %s will exist in the template where the name of the new * table will be addedd. * - * @return a String which is a template for adding an index to the object_id + * @return a String which is a template for adding an index to the file_obj_id * column of a _instances table */ static String getAddObjectIdIndexTemplate() { // Each "%s" will be replaced with the relevant TYPE_instances table name. - return "CREATE INDEX IF NOT EXISTS %s_object_id ON %s (object_id)"; + return "CREATE INDEX IF NOT EXISTS %s_file_obj_id ON %s (file_obj_id)"; } public boolean insertDefaultDatabaseContent() { diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDbSettings.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDbSettings.java index 2ead0cd4f3..f6c4d29268 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDbSettings.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDbSettings.java @@ -435,7 +435,7 @@ public final class SqliteEamDbSettings { createArtifactInstancesTableTemplate.append("file_path text NOT NULL,"); createArtifactInstancesTableTemplate.append("known_status integer NOT NULL,"); createArtifactInstancesTableTemplate.append("comment text,"); - createArtifactInstancesTableTemplate.append("object_id integer,"); + createArtifactInstancesTableTemplate.append("file_obj_id integer,"); createArtifactInstancesTableTemplate.append("CONSTRAINT %s_multi_unique UNIQUE(data_source_id, value, file_path) ON CONFLICT IGNORE,"); createArtifactInstancesTableTemplate.append("foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL,"); createArtifactInstancesTableTemplate.append("foreign key (data_source_id) references data_sources(id) ON UPDATE SET NULL ON DELETE SET NULL"); @@ -496,16 +496,16 @@ public final class SqliteEamDbSettings { } /** - * Get the template for creating an index on the object_id column of an + * Get the template for creating an index on the file_obj_id column of an * instance table. %s will exist in the template where the name of the new * table will be addedd. * - * @return a String which is a template for adding an index to the object_id + * @return a String which is a template for adding an index to the file_obj_id * column of a _instances table */ static String getAddObjectIdIndexTemplate() { // Each "%s" will be replaced with the relevant TYPE_instances table name. - return "CREATE INDEX IF NOT EXISTS %s_object_id ON %s (object_id)"; + return "CREATE INDEX IF NOT EXISTS %s_file_obj_id ON %s (file_obj_id)"; } public boolean insertDefaultDatabaseContent() { From 7eafe00d1afa9d61b6e30200f4c73b749f7ce35d Mon Sep 17 00:00:00 2001 From: William Schaefer Date: Fri, 16 Nov 2018 12:32:20 -0500 Subject: [PATCH 114/129] 4354 remove redundant create index statement for file_obj_id on wifi attrs --- .../autopsy/centralrepository/datamodel/AbstractSqlEamDb.java | 1 - 1 file changed, 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java index 4cee044504..a916ec9f5e 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/AbstractSqlEamDb.java @@ -3266,7 +3266,6 @@ abstract class AbstractSqlEamDb implements EamDb { statement.execute(String.format(addDataSourceIdIndexTemplate, wirelessNetworksTableInstanceName, wirelessNetworksTableInstanceName)); statement.execute(String.format(addValueIndexTemplate, wirelessNetworksTableInstanceName, wirelessNetworksTableInstanceName)); statement.execute(String.format(addKnownStatusIndexTemplate, wirelessNetworksTableInstanceName, wirelessNetworksTableInstanceName)); - statement.execute(String.format(addObjectIdIndexTemplate, wirelessNetworksTableInstanceName, wirelessNetworksTableInstanceName)); //add file_obj_id column to _instances table which do not already have it String instance_type_dbname; for (CorrelationAttributeInstance.Type type : CorrelationAttributeInstance.getDefaultCorrelationTypes()) { From e3c352ed5e578b81d4b3f1e80ce02f6ff50e486b Mon Sep 17 00:00:00 2001 From: rcordovano Date: Fri, 16 Nov 2018 18:28:48 -0500 Subject: [PATCH 115/129] Update db dump script to ignore pg_catalog SQL --- test/script/tskdbdiff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/script/tskdbdiff.py b/test/script/tskdbdiff.py index baefc55e6a..8380424bb5 100644 --- a/test/script/tskdbdiff.py +++ b/test/script/tskdbdiff.py @@ -333,7 +333,7 @@ class TskDbDiff(object): for line in postgreSQL_db: line = line.strip('\r\n ') # Deal with pg_dump result file - if line.startswith('--') or line.lower().startswith('alter') or not line: # It's comment or alter statement or empty line + if line.startswith('--') or line.lower().startswith('alter') or "pg_catalog" in line or not line: # It's comment or alter statement or catalog entry or empty line continue elif not line.endswith(';'): # Statement not finished dump_line += line From cc6c6aa4727ffc315adf3ae617e16e31cd2598ec Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Mon, 19 Nov 2018 09:21:28 -0500 Subject: [PATCH 116/129] Close IG database, propagate exceptions --- .../imagegallery/ImageGalleryController.java | 120 +- .../imagegallery/datamodel/DrawableDB.java | 1225 +++++++++-------- .../datamodel/HashSetManager.java | 15 +- .../datamodel/grouping/GroupManager.java | 36 +- 4 files changed, 725 insertions(+), 671 deletions(-) diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java index 6881bb2e88..abe89abf40 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java @@ -22,6 +22,7 @@ import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.ThreadFactoryBuilder; import java.beans.PropertyChangeListener; +import java.sql.SQLException; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -206,8 +207,8 @@ public final class ImageGalleryController { // if we just turned on listening and a single-user case is open and that case is not up to date, then rebuild it // For multiuser cases, we defer DB rebuild till the user actually opens Image Gallery if (isEnabled && !wasPreviouslyEnabled - && isDataSourcesTableStale() - && (Case.getCurrentCaseThrows().getCaseType() == CaseType.SINGLE_USER_CASE)) { + && isDataSourcesTableStale() + && (Case.getCurrentCaseThrows().getCaseType() == CaseType.SINGLE_USER_CASE)) { //populate the db this.rebuildDB(); } @@ -326,6 +327,7 @@ public final class ImageGalleryController { groupManager.reset(); shutDownDBExecutor(); + drawableDB.close(); dbExecutor = getNewDBExecutor(); } @@ -386,13 +388,13 @@ public final class ImageGalleryController { } - /** - * Returns a map of all data source object ids, along with - * their DB build status. + /** + * Returns a map of all data source object ids, along with their DB build + * status. * - * This includes any data sources already in the table, - * and any data sources that might have been added to the - * case, but are not in the datasources table. + * This includes any data sources already in the table, and any data sources + * that might have been added to the case, but are not in the datasources + * table. * * @return map of data source object ids and their Db build status. */ @@ -430,7 +432,7 @@ public final class ImageGalleryController { return dataSourceStatusMap; } } - + public boolean hasTooManyFiles(DataSource datasource) throws TskCoreException { String whereClause = (datasource == null) ? "1 = 1" @@ -439,26 +441,28 @@ public final class ImageGalleryController { return sleuthKitCase.countFilesWhere(whereClause) > FILE_LIMIT; } - + /** * Checks if the given data source has any files with no mimetype - * + * * @param datasource + * * @return true if the datasource has any files with no mime type - * @throws TskCoreException + * + * @throws TskCoreException */ public boolean hasFilesWithNoMimetype(Content datasource) throws TskCoreException { - + // There are some special files/attributes in the root folder, like $BadClus:$Bad and $Security:$SDS // The IngestTasksScheduler does not push them down to the ingest modules, // and hence they do not have any assigned mimetype String whereClause = "data_source_obj_id = " + datasource.getId() - + " AND ( meta_type = " + TskData.TSK_FS_META_TYPE_ENUM.TSK_FS_META_TYPE_REG.getValue() + ")" - + " AND ( mime_type IS NULL )" - + " AND ( meta_addr >= 32 ) " - + " AND ( parent_path <> '/' )" - + " AND ( name NOT like '$%:%' )"; - + + " AND ( meta_type = " + TskData.TSK_FS_META_TYPE_ENUM.TSK_FS_META_TYPE_REG.getValue() + ")" + + " AND ( mime_type IS NULL )" + + " AND ( meta_addr >= 32 ) " + + " AND ( parent_path <> '/' )" + + " AND ( name NOT like '$%:%' )"; + return sleuthKitCase.countFilesWhere(whereClause) > 0; } @@ -635,12 +639,8 @@ public final class ImageGalleryController { try { DrawableFile drawableFile = DrawableFile.create(getFile(), true, false); getTaskDB().updateFile(drawableFile); - } catch (NullPointerException ex) { - // This is one of the places where we get many errors if the case is closed during processing. - // We don't want to print out a ton of exceptions if this is the case. - if (Case.isCaseOpen()) { - Logger.getLogger(UpdateFileTask.class.getName()).log(Level.SEVERE, "Error in UpdateFile task"); //NON-NLS - } + } catch (TskCoreException | SQLException ex) { + Logger.getLogger(UpdateFileTask.class.getName()).log(Level.SEVERE, "Error in update file task", ex); //NON-NLS } } } @@ -661,12 +661,8 @@ public final class ImageGalleryController { public void run() { try { getTaskDB().removeFile(getFile().getId()); - } catch (NullPointerException ex) { - // This is one of the places where we get many errors if the case is closed during processing. - // We don't want to print out a ton of exceptions if this is the case. - if (Case.isCaseOpen()) { - Logger.getLogger(RemoveFileTask.class.getName()).log(Level.SEVERE, "Case was closed out from underneath RemoveFile task"); //NON-NLS - } + } catch (TskCoreException | SQLException ex) { + Logger.getLogger(RemoveFileTask.class.getName()).log(Level.SEVERE, "Error in remove file task", ex); //NON-NLS } } } @@ -682,13 +678,13 @@ public final class ImageGalleryController { static private final String FILE_EXTENSION_CLAUSE = "(extension LIKE '" //NON-NLS - + String.join("' OR extension LIKE '", FileTypeUtils.getAllSupportedExtensions()) //NON-NLS - + "') "; + + String.join("' OR extension LIKE '", FileTypeUtils.getAllSupportedExtensions()) //NON-NLS + + "') "; static private final String MIMETYPE_CLAUSE = "(mime_type LIKE '" //NON-NLS - + String.join("' OR mime_type LIKE '", FileTypeUtils.getAllSupportedMimeTypes()) //NON-NLS - + "') "; + + String.join("' OR mime_type LIKE '", FileTypeUtils.getAllSupportedMimeTypes()) //NON-NLS + + "') "; private final String DRAWABLE_QUERY; private final String DATASOURCE_CLAUSE; @@ -711,14 +707,14 @@ public final class ImageGalleryController { DRAWABLE_QUERY = DATASOURCE_CLAUSE - + " AND ( meta_type = " + TskData.TSK_FS_META_TYPE_ENUM.TSK_FS_META_TYPE_REG.getValue() + ")" - + " AND ( " - + //grab files with supported extension + + " AND ( meta_type = " + TskData.TSK_FS_META_TYPE_ENUM.TSK_FS_META_TYPE_REG.getValue() + ")" + + " AND ( " + + //grab files with supported extension FILE_EXTENSION_CLAUSE - //grab files with supported mime-types - + " OR " + MIMETYPE_CLAUSE //NON-NLS - //grab files with image or video mime-types even if we don't officially support them - + " OR mime_type LIKE 'video/%' OR mime_type LIKE 'image/%' )"; //NON-NLS + //grab files with supported mime-types + + " OR " + MIMETYPE_CLAUSE //NON-NLS + //grab files with image or video mime-types even if we don't officially support them + + " OR mime_type LIKE 'video/%' OR mime_type LIKE 'image/%' )"; //NON-NLS } /** @@ -764,7 +760,8 @@ public final class ImageGalleryController { //do in transaction drawableDbTransaction = taskDB.beginTransaction(); - /* We are going to periodically commit the CaseDB transaction + /* + * We are going to periodically commit the CaseDB transaction * and sleep so that the user can have Autopsy do other stuff * while these bulk tasks are ongoing. */ @@ -813,31 +810,34 @@ public final class ImageGalleryController { taskDB.commitTransaction(drawableDbTransaction, true); drawableDbTransaction = null; - } catch (TskCoreException | InterruptedException ex) { - progressHandle.progress(Bundle.BulkTask_stopCopy_status()); - logger.log(Level.WARNING, "Stopping copy to drawable db task. Failed to transfer all database contents", ex); //NON-NLS - MessageNotifyUtil.Notify.warn(Bundle.BulkTask_errPopulating_errMsg(), ex.getMessage()); - cleanup(false); - return; - } finally { - if (null != drawableDbTransaction) { - taskDB.rollbackTransaction(drawableDbTransaction); - } + } catch (TskCoreException | SQLException | InterruptedException ex) { if (null != caseDbTransaction) { try { caseDbTransaction.rollback(); } catch (TskCoreException ex2) { - logger.log(Level.SEVERE, "Error in trying to rollback transaction", ex2); //NON-NLS + logger.log(Level.SEVERE, String.format("Failed to roll back case db transaction after error: %s", ex.getMessage()), ex2); //NON-NLS } } + if (null != drawableDbTransaction) { + try { + taskDB.rollbackTransaction(drawableDbTransaction); + } catch (SQLException ex2) { + logger.log(Level.SEVERE, String.format("Failed to roll back drawables db transaction after error: %s", ex.getMessage()), ex2); //NON-NLS + } + } + progressHandle.progress(Bundle.BulkTask_stopCopy_status()); + logger.log(Level.WARNING, "Stopping copy to drawable db task. Failed to transfer all database contents", ex); //NON-NLS + MessageNotifyUtil.Notify.warn(Bundle.BulkTask_errPopulating_errMsg(), ex.getMessage()); + cleanup(false); + } finally { progressHandle.finish(); - - DrawableDB.DrawableDbBuildStatusEnum datasourceDrawableDBStatus = - (taskCompletionStatus) ? - DrawableDB.DrawableDbBuildStatusEnum.COMPLETE : - DrawableDB.DrawableDbBuildStatusEnum.DEFAULT; + + DrawableDB.DrawableDbBuildStatusEnum datasourceDrawableDBStatus + = (taskCompletionStatus) + ? DrawableDB.DrawableDbBuildStatusEnum.COMPLETE + : DrawableDB.DrawableDbBuildStatusEnum.DEFAULT; taskDB.insertOrUpdateDataSource(dataSourceObjId, datasourceDrawableDBStatus); - + updateMessage(""); updateProgress(-1.0); } diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java index 13eea6d654..3fa918e8b4 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java @@ -80,9 +80,8 @@ import org.sleuthkit.datamodel.TskDataException; import org.sqlite.SQLiteJDBCLoader; /** - * This class is the public interface to the Image Gallery SQLite database. This - * class borrows a lot of ideas and techniques (for good or ill) from - * SleuthkitCase + * Provides access to the drawables database and selected tables in the case + * database. */ public final class DrawableDB { @@ -98,42 +97,42 @@ public final class DrawableDB { private static final String GROUPS_TABLENAME = "image_gallery_groups"; //NON-NLS private static final String GROUPS_SEEN_TABLENAME = "image_gallery_groups_seen"; //NON-NLS - private final PreparedStatement insertHashSetStmt; + private PreparedStatement insertHashSetStmt; - private final List preparedStatements = new ArrayList<>(); + private List preparedStatements = new ArrayList<>(); - private final PreparedStatement removeFileStmt; + private PreparedStatement removeFileStmt; - private final PreparedStatement selectHashSetStmt; + private PreparedStatement selectHashSetStmt; - private final PreparedStatement selectHashSetNamesStmt; + private PreparedStatement selectHashSetNamesStmt; - private final PreparedStatement insertHashHitStmt; + private PreparedStatement insertHashHitStmt; - private final PreparedStatement removeHashHitStmt; + private PreparedStatement removeHashHitStmt; - private final PreparedStatement updateDataSourceStmt; + private PreparedStatement updateDataSourceStmt; - private final PreparedStatement updateFileStmt; - private final PreparedStatement insertFileStmt; + private PreparedStatement updateFileStmt; + private PreparedStatement insertFileStmt; - private final PreparedStatement pathGroupStmt; + private PreparedStatement pathGroupStmt; - private final PreparedStatement nameGroupStmt; + private PreparedStatement nameGroupStmt; - private final PreparedStatement created_timeGroupStmt; + private PreparedStatement created_timeGroupStmt; - private final PreparedStatement modified_timeGroupStmt; + private PreparedStatement modified_timeGroupStmt; - private final PreparedStatement makeGroupStmt; + private PreparedStatement makeGroupStmt; - private final PreparedStatement modelGroupStmt; + private PreparedStatement modelGroupStmt; - private final PreparedStatement analyzedGroupStmt; + private PreparedStatement analyzedGroupStmt; - private final PreparedStatement hashSetGroupStmt; + private PreparedStatement hashSetGroupStmt; - private final PreparedStatement pathGroupFilterByDataSrcStmt; + private PreparedStatement pathGroupFilterByDataSrcStmt; /** * map from {@link DrawableAttribute} to the {@link PreparedStatement} that @@ -146,11 +145,12 @@ public final class DrawableDB { private final Path dbPath; - volatile private Connection con; + @GuardedBy("DBLock") + private Connection con; private final ReentrantReadWriteLock rwLock = new ReentrantReadWriteLock(true); //use fairness policy - private final Lock DBLock = rwLock.writeLock(); //using exclusing lock for all db ops for now + private final Lock DBLock = rwLock.writeLock(); // Currently serializing everything with on database connection // caches to make inserts / updates faster private Cache groupCache = CacheBuilder.newBuilder().expireAfterWrite(5, TimeUnit.MINUTES).build(); @@ -160,8 +160,7 @@ public final class DrawableDB { private Set hasHashCache = new HashSet<>(); // obj id of files with hash set hits private Set hasExifCache = new HashSet<>(); // obj id of files with EXIF (make/model) private int cacheBuildCount = 0; // number of tasks taht requested the caches be built - - + static {//make sure sqlite driver is loaded // possibly redundant try { Class.forName("org.sqlite.JDBC"); @@ -174,13 +173,13 @@ public final class DrawableDB { /** * Enum to track Image gallery db rebuild status for a data source - * + * * DO NOT add in the middle. */ public enum DrawableDbBuildStatusEnum { - UNKNOWN, /// no known status - IN_PROGRESS, /// ingest or db rebuild is in progress - COMPLETE, /// All files in the data source have had file type detected + UNKNOWN, /// no known status + IN_PROGRESS, /// ingest or db rebuild is in progress + COMPLETE, /// All files in the data source have had file type detected DEFAULT; /// Not all files in the data source have had file type detected } @@ -213,44 +212,62 @@ public final class DrawableDB { * call dbReadUnLock() as early as possible, in the same thread where * dbReadLock() was called. */ - void dbReadLock() { - DBLock.lock(); - } - +// void dbReadLock() { +// DBLock.lock(); +// } /** * Release previously acquired read lock acquired in this thread using * dbReadLock(). Call in "finally" block to ensure the lock is always * released. */ - void dbReadUnlock() { - DBLock.unlock(); - } - +// void dbReadUnlock() { +// DBLock.unlock(); +// } /** - * @param dbPath the path to the db file + * Constructs an object that provides access to the drawables database and + * selected tables in the case database. If the specified drawables database + * does not already exist, it is created. * - * @throws SQLException if there is problem creating or configuring the db + * @param dbPath The path to the drawables database file. + * @param controller The controller for the IMage Gallery tool. + * + * @throws IOException The database directory could not be created. + * @throws SQLException The drawables database could not be created or + * opened. + * @throws TskCoreException The drawables database or the case database + * could not be correctly initialized for Image + * Gallery use. */ - private DrawableDB(Path dbPath, ImageGalleryController controller) throws TskCoreException, SQLException, IOException { + private DrawableDB(Path dbPath, ImageGalleryController controller) throws IOException, SQLException, TskCoreException { this.dbPath = dbPath; this.controller = controller; - this.tskCase = controller.getSleuthKitCase(); - this.groupManager = controller.getGroupManager(); - Files.createDirectories(dbPath.getParent()); - if (initializeDBSchema()) { + tskCase = this.controller.getSleuthKitCase(); + groupManager = this.controller.getGroupManager(); + Files.createDirectories(this.dbPath.getParent()); + dbWriteLock(); + try { + con = DriverManager.getConnection("jdbc:sqlite:" + dbPath.toString()); //NON-NLS + if (!initializeDBSchema() || !prepareStatements() || !initializeStandardGroups() || !initializeImageList()) { + close(); + throw new TskCoreException("Failed to create or open drawables database"); //NON-NLS + } + } finally { + dbWriteUnlock(); + } + } + + private boolean prepareStatements() { + try { updateFileStmt = prepareStatement( "INSERT OR REPLACE INTO drawable_files (obj_id, data_source_obj_id, path, name, created_time, modified_time, make, model, analyzed) " //NON-NLS + "VALUES (?,?,?,?,?,?,?,?,?)"); //NON-NLS insertFileStmt = prepareStatement( "INSERT OR IGNORE INTO drawable_files (obj_id, data_source_obj_id, path, name, created_time, modified_time, make, model, analyzed) " //NON-NLS + "VALUES (?,?,?,?,?,?,?,?,?)"); //NON-NLS - updateDataSourceStmt = prepareStatement( "INSERT OR REPLACE INTO datasources (ds_obj_id, drawable_db_build_status) " //NON-NLS + " VALUES (?,?)"); //NON-NLS - removeFileStmt = prepareStatement("DELETE FROM drawable_files WHERE obj_id = ?"); //NON-NLS - pathGroupStmt = prepareStatement("SELECT obj_id , analyzed FROM drawable_files WHERE path = ? ", DrawableAttribute.PATH); //NON-NLS nameGroupStmt = prepareStatement("SELECT obj_id , analyzed FROM drawable_files WHERE name = ? ", DrawableAttribute.NAME); //NON-NLS created_timeGroupStmt = prepareStatement("SELECT obj_id , analyzed FROM drawable_files WHERE created_time = ? ", DrawableAttribute.CREATED_TIME); //NON-NLS @@ -259,39 +276,38 @@ public final class DrawableDB { modelGroupStmt = prepareStatement("SELECT obj_id , analyzed FROM drawable_files WHERE model = ? ", DrawableAttribute.MODEL); //NON-NLS analyzedGroupStmt = prepareStatement("SELECT obj_id , analyzed FROM drawable_files WHERE analyzed = ?", DrawableAttribute.ANALYZED); //NON-NLS hashSetGroupStmt = prepareStatement("SELECT drawable_files.obj_id AS obj_id, analyzed FROM drawable_files , hash_sets , hash_set_hits WHERE drawable_files.obj_id = hash_set_hits.obj_id AND hash_sets.hash_set_id = hash_set_hits.hash_set_id AND hash_sets.hash_set_name = ?", DrawableAttribute.HASHSET); //NON-NLS - - //add other xyzFilterByDataSrc prepared statments as we add support for filtering by DS to other groups pathGroupFilterByDataSrcStmt = prepareFilterByDataSrcStatement("SELECT obj_id , analyzed FROM drawable_files WHERE path = ? AND data_source_obj_id = ?", DrawableAttribute.PATH); - selectHashSetNamesStmt = prepareStatement("SELECT DISTINCT hash_set_name FROM hash_sets"); //NON-NLS insertHashSetStmt = prepareStatement("INSERT OR IGNORE INTO hash_sets (hash_set_name) VALUES (?)"); //NON-NLS selectHashSetStmt = prepareStatement("SELECT hash_set_id FROM hash_sets WHERE hash_set_name = ?"); //NON-NLS - insertHashHitStmt = prepareStatement("INSERT OR IGNORE INTO hash_set_hits (hash_set_id, obj_id) VALUES (?,?)"); //NON-NLS removeHashHitStmt = prepareStatement("DELETE FROM hash_set_hits WHERE obj_id = ?"); //NON-NLS + return true; + } catch (TskCoreException | SQLException ex) { + logger.log(Level.SEVERE, "Failed to prepare all statements", ex); //NON-NLS + return false; + } + } - CaseDbTransaction caseDbTransaction = null; - try { - caseDbTransaction = tskCase.beginTransaction(); - for (DhsImageCategory cat : DhsImageCategory.values()) { - insertGroup(cat.getDisplayName(), DrawableAttribute.CATEGORY, caseDbTransaction); - } - caseDbTransaction.commit(); - caseDbTransaction = null; - } - finally { - if (null != caseDbTransaction) { - try { - caseDbTransaction.rollback(); - } catch (TskCoreException ex2) { - logger.log(Level.SEVERE, "Error in trying to rollback transaction", ex2); - } + private boolean initializeStandardGroups() { + CaseDbTransaction caseDbTransaction = null; + try { + caseDbTransaction = tskCase.beginTransaction(); + for (DhsImageCategory cat : DhsImageCategory.values()) { + insertGroup(cat.getDisplayName(), DrawableAttribute.CATEGORY, caseDbTransaction); + } + caseDbTransaction.commit(); + return true; + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, "Failed to insert standard groups", ex); //NON-NLS + if (null != caseDbTransaction) { + try { + caseDbTransaction.rollback(); + } catch (TskCoreException ex2) { + logger.log(Level.SEVERE, "Failed to roll back case DB transaction", ex2); } } - - initializeImageList(); - } else { - throw new TskCoreException("Failed to initialize Image Gallery db schema"); + return false; } } @@ -306,10 +322,20 @@ public final class DrawableDB { * * @throws SQLException if unable to prepare the statement */ - private PreparedStatement prepareStatement(String stmtString) throws SQLException { - PreparedStatement prepareStatement = con.prepareStatement(stmtString); - preparedStatements.add(prepareStatement); - return prepareStatement; + private PreparedStatement prepareStatement(String stmtString) throws TskCoreException, SQLException { + dbWriteLock(); + try { + if (isClosed()) { + throw new TskCoreException("The drawables database is closed"); + } + PreparedStatement statement = con.prepareStatement(stmtString); + preparedStatements.add(statement); + return statement; + } catch (SQLException ex) { + throw new SQLException(String.format("Error preparing statement %s", stmtString, ex)); + } finally { + dbWriteUnlock(); + } } /** @@ -325,13 +351,12 @@ public final class DrawableDB { * * @throws SQLExceptionif unable to prepare the statement */ - private PreparedStatement prepareStatement(String stmtString, DrawableAttribute attr) throws SQLException { - PreparedStatement prepareStatement = prepareStatement(stmtString); + private PreparedStatement prepareStatement(String stmtString, DrawableAttribute attr) throws TskCoreException, SQLException { + PreparedStatement statement = prepareStatement(stmtString); if (attr != null) { - groupStatementMap.put(attr, prepareStatement); + groupStatementMap.put(attr, statement); } - - return prepareStatement; + return statement; } /** @@ -347,13 +372,12 @@ public final class DrawableDB { * * @throws SQLExceptionif unable to prepare the statement */ - private PreparedStatement prepareFilterByDataSrcStatement(String stmtString, DrawableAttribute attr) throws SQLException { - PreparedStatement prepareStatement = prepareStatement(stmtString); + private PreparedStatement prepareFilterByDataSrcStatement(String stmtString, DrawableAttribute attr) throws TskCoreException, SQLException { + PreparedStatement statement = prepareStatement(stmtString); if (attr != null) { - groupStatementFilterByDataSrcMap.put(attr, prepareStatement); + groupStatementFilterByDataSrcMap.put(attr, statement); } - - return prepareStatement; + return statement; } private void setQueryParams(PreparedStatement statement, GroupKey groupKey) throws SQLException { @@ -361,7 +385,7 @@ public final class DrawableDB { statement.setObject(1, groupKey.getValue()); if (groupKey.getDataSource().isPresent() - && (groupKey.getAttribute() == DrawableAttribute.PATH)) { + && (groupKey.getAttribute() == DrawableAttribute.PATH)) { statement.setObject(2, groupKey.getDataSourceObjId()); } } @@ -436,39 +460,47 @@ public final class DrawableDB { } private void setPragmas() throws SQLException { - - //this should match Sleuthkit db setupt - try (Statement statement = con.createStatement()) { - //reduce i/o operations, we have no OS crash recovery anyway - statement.execute("PRAGMA synchronous = OFF;"); //NON-NLS - //allow to query while in transaction - no need read locks - statement.execute("PRAGMA read_uncommitted = True;"); //NON-NLS - - //TODO: do we need this? - statement.execute("PRAGMA foreign_keys = ON"); //NON-NLS - - //TODO: test this - statement.execute("PRAGMA journal_mode = MEMORY"); //NON-NLS -// - //we don't use this feature, so turn it off for minimal speed up on queries - //this is deprecated and not recomended - statement.execute("PRAGMA count_changes = OFF;"); //NON-NLS - //this made a big difference to query speed - statement.execute("PRAGMA temp_store = MEMORY"); //NON-NLS - //this made a modest improvement in query speeds - statement.execute("PRAGMA cache_size = 50000"); //NON-NLS - //we never delete anything so... - statement.execute("PRAGMA auto_vacuum = 0"); //NON-NLS - } - + dbWriteLock(); try { - logger.log(Level.INFO, String.format("sqlite-jdbc version %s loaded in %s mode", //NON-NLS - SQLiteJDBCLoader.getVersion(), SQLiteJDBCLoader.isNativeMode() - ? "native" : "pure-java")); //NON-NLS - } catch (Exception exception) { - logger.log(Level.WARNING, "exception while checking sqlite-jdbc version and mode", exception); //NON-NLS - } + if (isClosed()) { + throw new SQLException("The drawables database is closed"); + } + //this should match Sleuthkit db setupt + try (Statement statement = con.createStatement()) { + //reduce i/o operations, we have no OS crash recovery anyway + statement.execute("PRAGMA synchronous = OFF;"); //NON-NLS + //allow to query while in transaction - no need read locks + statement.execute("PRAGMA read_uncommitted = True;"); //NON-NLS + + //TODO: do we need this? + statement.execute("PRAGMA foreign_keys = ON"); //NON-NLS + + //TODO: test this + statement.execute("PRAGMA journal_mode = MEMORY"); //NON-NLS + + //we don't use this feature, so turn it off for minimal speed up on queries + //this is deprecated and not recomended + statement.execute("PRAGMA count_changes = OFF;"); //NON-NLS + //this made a big difference to query speed + statement.execute("PRAGMA temp_store = MEMORY"); //NON-NLS + //this made a modest improvement in query speeds + statement.execute("PRAGMA cache_size = 50000"); //NON-NLS + //we never delete anything so... + statement.execute("PRAGMA auto_vacuum = 0"); //NON-NLS + } + + try { + logger.log(Level.INFO, String.format("sqlite-jdbc version %s loaded in %s mode", //NON-NLS + SQLiteJDBCLoader.getVersion(), SQLiteJDBCLoader.isNativeMode() + ? "native" : "pure-java")); //NON-NLS + } catch (Exception exception) { + logger.log(Level.SEVERE, "exception while checking sqlite-jdbc version and mode", exception); //NON-NLS + } + + } finally { + dbWriteUnlock(); + } } /** @@ -478,176 +510,208 @@ public final class DrawableDB { * existing table */ private boolean initializeDBSchema() { + dbWriteLock(); try { if (isClosed()) { - openDBCon(); + logger.log(Level.SEVERE, "The drawables database is closed"); //NON-NLS + return false; } - setPragmas(); - } catch (SQLException ex) { - logger.log(Level.SEVERE, "problem accessing database", ex); //NON-NLS - return false; + try { + setPragmas(); + } catch (SQLException ex) { + logger.log(Level.SEVERE, "Failed to set pragmas", ex); //NON-NLS + return false; + } + + /* + * Create tables in the drawables database. + */ + try (Statement stmt = con.createStatement()) { + try { + String sql = "CREATE TABLE IF NOT EXISTS datasources " //NON-NLS + + "( id INTEGER PRIMARY KEY, " //NON-NLS + + " ds_obj_id integer UNIQUE NOT NULL, " + + " drawable_db_build_status VARCHAR(128) )"; //NON-NLS + stmt.execute(sql); + } catch (SQLException ex) { + logger.log(Level.SEVERE, "Failed to create datasources table", ex); //NON-NLS + return false; + } + + try { + String sql = "CREATE TABLE if not exists drawable_files " //NON-NLS + + "( obj_id INTEGER PRIMARY KEY, " //NON-NLS + + " data_source_obj_id INTEGER NOT NULL, " + + " path VARCHAR(255), " //NON-NLS + + " name VARCHAR(255), " //NON-NLS + + " created_time integer, " //NON-NLS + + " modified_time integer, " //NON-NLS + + " make VARCHAR(255), " //NON-NLS + + " model VARCHAR(255), " //NON-NLS + + " analyzed integer DEFAULT 0)"; //NON-NLS + stmt.execute(sql); + } catch (SQLException ex) { + logger.log(Level.SEVERE, "Failed to create drawable_files table", ex); //NON-NLS + return false; + } + + try { + String sql = "CREATE TABLE if not exists hash_sets " //NON-NLS + + "( hash_set_id INTEGER primary key," //NON-NLS + + " hash_set_name VARCHAR(255) UNIQUE NOT NULL)"; //NON-NLS + stmt.execute(sql); + } catch (SQLException ex) { + logger.log(Level.SEVERE, "Failed to create hash_sets table", ex); //NON-NLS + return false; + } + + try { + String sql = "CREATE TABLE if not exists hash_set_hits " //NON-NLS + + "(hash_set_id INTEGER REFERENCES hash_sets(hash_set_id) not null, " //NON-NLS + + " obj_id INTEGER REFERENCES drawable_files(obj_id) not null, " //NON-NLS + + " PRIMARY KEY (hash_set_id, obj_id))"; //NON-NLS + stmt.execute(sql); + } catch (SQLException ex) { + logger.log(Level.SEVERE, "Failed to create hash_set_hits table", ex); //NON-NLS + return false; + } + + try { + String sql = "CREATE INDEX if not exists path_idx ON drawable_files(path)"; //NON-NLS + stmt.execute(sql); + } catch (SQLException ex) { + logger.log(Level.WARNING, "Failed to create path_idx", ex); //NON-NLS + } + + try { + String sql = "CREATE INDEX if not exists name_idx ON drawable_files(name)"; //NON-NLS + stmt.execute(sql); + } catch (SQLException ex) { + logger.log(Level.WARNING, "Failed to create name_idx", ex); //NON-NLS + } + + try { + String sql = "CREATE INDEX if not exists make_idx ON drawable_files(make)"; //NON-NLS + stmt.execute(sql); + } catch (SQLException ex) { + logger.log(Level.WARNING, "Failed to create make_idx", ex); //NON-NLS + } + + try { + String sql = "CREATE INDEX if not exists model_idx ON drawable_files(model)"; //NON-NLS + stmt.execute(sql); + } catch (SQLException ex) { + logger.log(Level.WARNING, "Failed to create model_idx", ex); //NON-NLS + } + + try { + String sql = "CREATE INDEX if not exists analyzed_idx ON drawable_files(analyzed)"; //NON-NLS + stmt.execute(sql); + } catch (SQLException ex) { + logger.log(Level.WARNING, "Failed to create analyzed_idx", ex); //NON-NLS + } + } catch (SQLException ex) { + logger.log(Level.SEVERE, "Failed to create statement", ex); //NON-NLS + return false; + } + + /* + * Create tables in the case database. + */ + String autogenKeyType = (DbType.POSTGRESQL == tskCase.getDatabaseType()) ? "BIGSERIAL" : "INTEGER"; + try { + String tableSchema + = "( group_id " + autogenKeyType + " PRIMARY KEY, " //NON-NLS + + " data_source_obj_id integer DEFAULT 0, " + + " value VARCHAR(255) not null, " //NON-NLS + + " attribute VARCHAR(255) not null, " //NON-NLS + + " UNIQUE(data_source_obj_id, value, attribute) )"; //NON-NLS + + tskCase.getCaseDbAccessManager().createTable(GROUPS_TABLENAME, tableSchema); + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, String.format("Failed to create %s table in case database", GROUPS_TABLENAME), ex); //NON-NLS + return false; + } + try { + + String tableSchema + = "( id " + autogenKeyType + " PRIMARY KEY, " //NON-NLS + + " group_id integer not null, " //NON-NLS + + " examiner_id integer not null, " //NON-NLS + + " seen integer DEFAULT 0, " //NON-NLS + + " UNIQUE(group_id, examiner_id)," + + " FOREIGN KEY(group_id) REFERENCES " + GROUPS_TABLENAME + "(group_id)," + + " FOREIGN KEY(examiner_id) REFERENCES tsk_examiners(examiner_id)" + + " )"; //NON-NLS + + tskCase.getCaseDbAccessManager().createTable(GROUPS_SEEN_TABLENAME, tableSchema); + } catch (TskCoreException ex) { + logger.log(Level.SEVERE, String.format("Failed to create %s table in case database", GROUPS_SEEN_TABLENAME), ex); //NON-NLS + return false; + } + + return true; + + } finally { + dbWriteUnlock(); } - - try (Statement stmt = con.createStatement()) { - String sql = "CREATE TABLE IF NOT EXISTS datasources " //NON-NLS - + "( id INTEGER PRIMARY KEY, " //NON-NLS - + " ds_obj_id integer UNIQUE NOT NULL, " - + " drawable_db_build_status VARCHAR(128) )"; //NON-NLS - stmt.execute(sql); - } catch (SQLException ex) { - logger.log(Level.SEVERE, "problem creating datasources table", ex); //NON-NLS - return false; - } - - try (Statement stmt = con.createStatement()) { - String sql = "CREATE TABLE if not exists drawable_files " //NON-NLS - + "( obj_id INTEGER PRIMARY KEY, " //NON-NLS - + " data_source_obj_id INTEGER NOT NULL, " - + " path VARCHAR(255), " //NON-NLS - + " name VARCHAR(255), " //NON-NLS - + " created_time integer, " //NON-NLS - + " modified_time integer, " //NON-NLS - + " make VARCHAR(255), " //NON-NLS - + " model VARCHAR(255), " //NON-NLS - + " analyzed integer DEFAULT 0)"; //NON-NLS - stmt.execute(sql); - } catch (SQLException ex) { - logger.log(Level.SEVERE, "problem creating drawable_files table", ex); //NON-NLS - return false; - } - - String autogenKeyType = (DbType.POSTGRESQL == tskCase.getDatabaseType()) ? "BIGSERIAL" : "INTEGER"; - - // The image_gallery_groups table is created in the Case Database - try { - String tableSchema - = "( group_id " + autogenKeyType + " PRIMARY KEY, " //NON-NLS - + " data_source_obj_id integer DEFAULT 0, " - + " value VARCHAR(255) not null, " //NON-NLS - + " attribute VARCHAR(255) not null, " //NON-NLS - + " UNIQUE(data_source_obj_id, value, attribute) )"; //NON-NLS - - tskCase.getCaseDbAccessManager().createTable(GROUPS_TABLENAME, tableSchema); - } catch (TskCoreException ex) { - logger.log(Level.SEVERE, "problem creating groups table", ex); //NON-NLS - return false; - } - - // The image_gallery_groups_seen table is created in the Case Database - try { - - String tableSchema - = "( id " + autogenKeyType + " PRIMARY KEY, " //NON-NLS - + " group_id integer not null, " //NON-NLS - + " examiner_id integer not null, " //NON-NLS - + " seen integer DEFAULT 0, " //NON-NLS - + " UNIQUE(group_id, examiner_id)," - + " FOREIGN KEY(group_id) REFERENCES " + GROUPS_TABLENAME + "(group_id)," - + " FOREIGN KEY(examiner_id) REFERENCES tsk_examiners(examiner_id)" - + " )"; //NON-NLS - - tskCase.getCaseDbAccessManager().createTable(GROUPS_SEEN_TABLENAME, tableSchema); - } catch (TskCoreException ex) { - logger.log(Level.SEVERE, "problem creating image_gallery_groups_seen table", ex); //NON-NLS - return false; - } - - try (Statement stmt = con.createStatement()) { - String sql = "CREATE TABLE if not exists hash_sets " //NON-NLS - + "( hash_set_id INTEGER primary key," //NON-NLS - + " hash_set_name VARCHAR(255) UNIQUE NOT NULL)"; //NON-NLS - stmt.execute(sql); - } catch (SQLException ex) { - logger.log(Level.SEVERE, "problem creating hash_sets table", ex); //NON-NLS - return false; - } - - try (Statement stmt = con.createStatement()) { - String sql = "CREATE TABLE if not exists hash_set_hits " //NON-NLS - + "(hash_set_id INTEGER REFERENCES hash_sets(hash_set_id) not null, " //NON-NLS - + " obj_id INTEGER REFERENCES drawable_files(obj_id) not null, " //NON-NLS - + " PRIMARY KEY (hash_set_id, obj_id))"; //NON-NLS - stmt.execute(sql); - } catch (SQLException ex) { - logger.log(Level.SEVERE, "problem creating hash_set_hits table", ex); //NON-NLS - return false; - } - - try (Statement stmt = con.createStatement()) { - String sql = "CREATE INDEX if not exists path_idx ON drawable_files(path)"; //NON-NLS - stmt.execute(sql); - } catch (SQLException ex) { - logger.log(Level.WARNING, "problem creating path_idx", ex); //NON-NLS - } - - try (Statement stmt = con.createStatement()) { - String sql = "CREATE INDEX if not exists name_idx ON drawable_files(name)"; //NON-NLS - stmt.execute(sql); - } catch (SQLException ex) { - logger.log(Level.WARNING, "problem creating name_idx", ex); //NON-NLS - } - - try (Statement stmt = con.createStatement()) { - String sql = "CREATE INDEX if not exists make_idx ON drawable_files(make)"; //NON-NLS - stmt.execute(sql); - } catch (SQLException ex) { - logger.log(Level.WARNING, "problem creating make_idx", ex); //NON-NLS - } - - try (Statement stmt = con.createStatement()) { - String sql = "CREATE INDEX if not exists model_idx ON drawable_files(model)"; //NON-NLS - stmt.execute(sql); - } catch (SQLException ex) { - logger.log(Level.WARNING, "problem creating model_idx", ex); //NON-NLS - } - - try (Statement stmt = con.createStatement()) { - String sql = "CREATE INDEX if not exists analyzed_idx ON drawable_files(analyzed)"; //NON-NLS - stmt.execute(sql); - } catch (SQLException ex) { - logger.log(Level.WARNING, "problem creating analyzed_idx", ex); //NON-NLS - } - - return true; } @Override - public void finalize() throws Throwable { + protected void finalize() throws Throwable { + /* + * This finalizer is a safety net for freeing this resource. See + * "Effective Java" by Joshua Block, Item #7. + */ + dbWriteLock(); try { - closeDBCon(); + if (!isClosed()) { + logger.log(Level.SEVERE, "Closing drawable.db in finalizer, this should never be necessary"); //NON-NLS + try { + close(); + } finally { + super.finalize(); + } + } } finally { - super.finalize(); + dbWriteUnlock(); } } - public void closeDBCon() { - if (con != null) { - try { - closeStatements(); - con.close(); - } catch (SQLException ex) { - logger.log(Level.WARNING, "Failed to close connection to drawable.db", ex); //NON-NLS - } - } - con = null; - } - - public void openDBCon() { + public void close() { + dbWriteLock(); try { - if (con == null || con.isClosed()) { - con = DriverManager.getConnection("jdbc:sqlite:" + dbPath.toString()); //NON-NLS + if (!isClosed()) { + logger.log(Level.INFO, "Closing the drawable.db"); //NON-NLS + for (PreparedStatement pStmt : preparedStatements) { + try { + pStmt.close(); + } catch (SQLException ex) { + logger.log(Level.SEVERE, String.format("Failed to close prepared statement %s for drawable.db", pStmt.toString()), ex); //NON-NLS + } + } + try { + con.close(); + } catch (SQLException ex) { + logger.log(Level.SEVERE, "Failed to close connection to drawable.db", ex); //NON-NLS + } } - } catch (SQLException ex) { - logger.log(Level.WARNING, "Failed to open connection to drawable.db", ex); //NON-NLS + } finally { + con = null; + dbWriteUnlock(); } } - public boolean isClosed() throws SQLException { - if (con == null) { - return true; + private boolean isClosed() { + dbWriteLock(); + try { + return ((con == null) || (con.isClosed())); + } catch (SQLException unused) { + return false; + } finally { + dbWriteUnlock(); } - return con.isClosed(); } /** @@ -684,7 +748,7 @@ public final class DrawableDB { public Set getHashSetNames() { Set names = new HashSet<>(); // "SELECT DISTINCT hash_set_name FROM hash_sets" - dbReadLock(); + dbWriteLock(); try (ResultSet rs = selectHashSetNamesStmt.executeQuery();) { while (rs.next()) { names.add(rs.getString(HASH_SET_NAME)); @@ -692,7 +756,7 @@ public final class DrawableDB { } catch (SQLException sQLException) { logger.log(Level.WARNING, "failed to get hash set names", sQLException); //NON-NLS } finally { - dbReadUnlock(); + dbWriteUnlock(); } return names; } @@ -700,7 +764,7 @@ public final class DrawableDB { static private String getGroupIdQuery(GroupKey groupKey) { // query to find the group id from attribute/value return String.format(" SELECT group_id FROM " + GROUPS_TABLENAME - + " WHERE attribute = \'%s\' AND value = \'%s\' AND data_source_obj_id = %d", + + " WHERE attribute = \'%s\' AND value = \'%s\' AND data_source_obj_id = %d", SleuthkitCase.escapeSingleQuotes(groupKey.getAttribute().attrName.toString()), SleuthkitCase.escapeSingleQuotes(groupKey.getValueDisplayName()), (groupKey.getAttribute() == DrawableAttribute.PATH) ? groupKey.getDataSourceObjId() : 0); @@ -750,9 +814,9 @@ public final class DrawableDB { try { String groupSeenQueryStmt = "COUNT(*) as count FROM " + GROUPS_SEEN_TABLENAME - + " WHERE seen = 1 " - + " AND group_id in ( " + getGroupIdQuery(groupKey) + ")" - + (examinerId > 0 ? " AND examiner_id = " + examinerId : "");// query to find the group id from attribute/value + + " WHERE seen = 1 " + + " AND group_id in ( " + getGroupIdQuery(groupKey) + ")" + + (examinerId > 0 ? " AND examiner_id = " + examinerId : "");// query to find the group id from attribute/value tskCase.getCaseDbAccessManager().select(groupSeenQueryStmt, queryResultProcessor); return queryResultProcessor.get(); @@ -778,7 +842,7 @@ public final class DrawableDB { // query to find the group id from attribute/value String innerQuery = String.format("( SELECT group_id FROM " + GROUPS_TABLENAME - + " WHERE attribute = \'%s\' AND value = \'%s\' and data_source_obj_id = %d )", + + " WHERE attribute = \'%s\' AND value = \'%s\' and data_source_obj_id = %d )", SleuthkitCase.escapeSingleQuotes(groupKey.getAttribute().attrName.toString()), SleuthkitCase.escapeSingleQuotes(groupKey.getValueDisplayName()), groupKey.getAttribute() == DrawableAttribute.PATH ? groupKey.getDataSourceObjId() : 0); @@ -793,76 +857,105 @@ public final class DrawableDB { } - public boolean removeFile(long id) { - DrawableTransaction trans = beginTransaction(); - boolean removeFile = removeFile(id, trans); - commitTransaction(trans, true); - return removeFile; - } - - public void updateFile(DrawableFile f) { + /** + * Removes a file from the drawables databse. + * + * @param id The object id of the file. + * + * @return True or false. + * + * @throws TskCoreException + * @throws SQLException + */ + public void removeFile(long id) throws TskCoreException, SQLException { DrawableTransaction trans = null; CaseDbTransaction caseDbTransaction = null; + try { + trans = beginTransaction(); + removeFile(id, trans); + commitTransaction(trans, true); + } catch (TskCoreException | SQLException ex) { + if (null != caseDbTransaction) { + try { + caseDbTransaction.rollback(); + } catch (TskCoreException ex2) { + logger.log(Level.SEVERE, String.format("Failed to roll back case db transaction after error: %s", ex.getMessage()), ex2); //NON-NLS + } + } + if (null != trans) { + try { + rollbackTransaction(trans); + } catch (SQLException ex2) { + logger.log(Level.SEVERE, String.format("Failed to roll back drawables db transaction after error: %s", ex.getMessage()), ex2); //NON-NLS + } + } + throw ex; + } + } + public void updateFile(DrawableFile f) throws TskCoreException, SQLException { + DrawableTransaction trans = null; + CaseDbTransaction caseDbTransaction = null; try { trans = beginTransaction(); caseDbTransaction = tskCase.beginTransaction(); updateFile(f, trans, caseDbTransaction); caseDbTransaction.commit(); - caseDbTransaction = null; commitTransaction(trans, true); - trans = null; - - } catch (TskCoreException ex) { - logger.log(Level.SEVERE, "Error updating file", ex); //NON-NLS - } - finally { + } catch (TskCoreException | SQLException ex) { if (null != caseDbTransaction) { try { caseDbTransaction.rollback(); } catch (TskCoreException ex2) { - logger.log(Level.SEVERE, "Error in trying to rollback transaction", ex2); //NON-NLS + logger.log(Level.SEVERE, String.format("Failed to roll back case db transaction after error: %s", ex.getMessage()), ex2); //NON-NLS } } if (null != trans) { - rollbackTransaction(trans); + try { + rollbackTransaction(trans); + } catch (SQLException ex2) { + logger.log(Level.SEVERE, String.format("Failed to roll back drawables db transaction after error: %s", ex.getMessage()), ex2); //NON-NLS + } } + throw ex; } - } /** - * Insert basic file data (no groups) into the DB during pre-population phase + * Insert basic file data (no groups) into the DB during pre-population + * phase + * * @param f * @param tr - * @param caseDbTransaction + * @param caseDbTransaction */ public void insertBasicFileData(DrawableFile f, DrawableTransaction tr, CaseDbTransaction caseDbTransaction) { insertOrUpdateFile(f, tr, caseDbTransaction, false); } /** - * Update an existing entry (or make a new one) into the DB that includes group information. - * Called when a file has been analyzed or during a bulk rebuild - * + * Update an existing entry (or make a new one) into the DB that includes + * group information. Called when a file has been analyzed or during a bulk + * rebuild + * * @param f * @param tr - * @param caseDbTransaction + * @param caseDbTransaction */ public void updateFile(DrawableFile f, DrawableTransaction tr, CaseDbTransaction caseDbTransaction) { insertOrUpdateFile(f, tr, caseDbTransaction, true); } - - + /** * Populate caches based on current state of Case DB */ public void buildFileMetaDataCache() { - - synchronized (cacheLock) { + + synchronized (cacheLock) { cacheBuildCount++; - if (areCachesLoaded == true) + if (areCachesLoaded == true) { return; + } try { // get tags @@ -914,54 +1007,61 @@ public final class DrawableDB { areCachesLoaded = true; } } - + /** * Add a file to cache of files that have EXIF data + * * @param objectID ObjId of file with EXIF */ public void addExifCache(long objectID) { synchronized (cacheLock) { // bail out if we are not maintaining caches - if (cacheBuildCount == 0) + if (cacheBuildCount == 0) { return; + } hasExifCache.add(objectID); } } - + /** * Add a file to cache of files that have hash set hits + * * @param objectID ObjId of file with hash set */ public void addHashSetCache(long objectID) { synchronized (cacheLock) { // bail out if we are not maintaining caches - if (cacheBuildCount == 0) + if (cacheBuildCount == 0) { return; + } hasHashCache.add(objectID); } } - + /** * Add a file to cache of files that have tags + * * @param objectID ObjId of file with tags */ public void addTagCache(long objectID) { - synchronized (cacheLock) { + synchronized (cacheLock) { // bail out if we are not maintaining caches - if (cacheBuildCount == 0) + if (cacheBuildCount == 0) { return; + } hasTagCache.add(objectID); - } + } } - + /** * Free the cached case DB data */ public void freeFileMetaDataCache() { synchronized (cacheLock) { // dont' free these if there is another task still using them - if (--cacheBuildCount > 0) + if (--cacheBuildCount > 0) { return; + } areCachesLoaded = false; hasTagCache.clear(); @@ -979,26 +1079,27 @@ public final class DrawableDB { * //TODO: this is a kinda weird design, is their a better way? //TODO: * implement batch version -jm * - * @param f The file to insert. - * @param tr a transaction to use, must not be null + * @param f The file to insert. + * @param tr a transaction to use, must not be null * @param caseDbTransaction - * @param addGroups True if groups for file should be inserted into db too + * @param addGroups True if groups for file should be inserted into + * db too */ private void insertOrUpdateFile(DrawableFile f, @Nonnull DrawableTransaction tr, @Nonnull CaseDbTransaction caseDbTransaction, boolean addGroups) { PreparedStatement stmt; - - if (tr.isClosed()) { + + if (tr.isCompleted()) { throw new IllegalArgumentException("can't update database with closed transaction"); } - + // assume that we are doing an update if we are adding groups - i.e. not pre-populating if (addGroups) { stmt = updateFileStmt; } else { stmt = insertFileStmt; } - + // get data from caches. Default to true and force the DB lookup if we don't have caches boolean hasExif = true; boolean hasHashSet = true; @@ -1010,7 +1111,7 @@ public final class DrawableDB { hasTag = hasTagCache.contains(f.getId()); } } - + // if we are going to just add basic data, then mark flags that we do not have metadata to prevent lookups if (addGroups == false) { hasExif = false; @@ -1036,13 +1137,13 @@ public final class DrawableDB { } stmt.setBoolean(9, f.isAnalyzed()); stmt.executeUpdate(); - + // Update the list of file IDs in memory addImageFileToList(f.getId()); // update the groups if we are not doing pre-populating if (addGroups) { - + // Update the hash set tables if (hasHashSet) { try { @@ -1076,8 +1177,7 @@ public final class DrawableDB { // skip attributes that we do not have data for if ((attr == DrawableAttribute.TAGS) && (hasTag == false)) { continue; - } - else if ((attr == DrawableAttribute.MAKE || attr == DrawableAttribute.MODEL) && (hasExif == false)) { + } else if ((attr == DrawableAttribute.MAKE || attr == DrawableAttribute.MODEL) && (hasExif == false)) { continue; } Collection> vals = attr.getValue(f); @@ -1085,8 +1185,7 @@ public final class DrawableDB { if ((null != val) && (val.toString().isEmpty() == false)) { if (attr == DrawableAttribute.PATH) { insertGroup(f.getAbstractFile().getDataSource().getId(), val.toString(), attr, caseDbTransaction); - } - else { + } else { insertGroup(val.toString(), attr, caseDbTransaction); } } @@ -1121,25 +1220,20 @@ public final class DrawableDB { */ public Map getDataSourceDbBuildStatus() throws TskCoreException { Statement statement = null; - ResultSet rs = null; Map map = new HashMap<>(); - dbReadLock(); + dbWriteLock(); try { + if (isClosed()) { + throw new TskCoreException("The drawables database is closed"); + } statement = con.createStatement(); - rs = statement.executeQuery("SELECT ds_obj_id, drawable_db_build_status FROM datasources "); //NON-NLS + ResultSet rs = statement.executeQuery("SELECT ds_obj_id, drawable_db_build_status FROM datasources "); //NON-NLS while (rs.next()) { map.put(rs.getLong("ds_obj_id"), DrawableDbBuildStatusEnum.valueOf(rs.getString("drawable_db_build_status"))); } } catch (SQLException e) { throw new TskCoreException("SQLException while getting data source object ids", e); } finally { - if (rs != null) { - try { - rs.close(); - } catch (SQLException ex) { - logger.log(Level.SEVERE, "Error closing resultset", ex); //NON-NLS - } - } if (statement != null) { try { statement.close(); @@ -1147,7 +1241,7 @@ public final class DrawableDB { logger.log(Level.SEVERE, "Error closing statement ", ex); //NON-NLS } } - dbReadUnlock(); + dbWriteUnlock(); } return map; } @@ -1176,94 +1270,71 @@ public final class DrawableDB { } } - public DrawableTransaction beginTransaction() { + public DrawableTransaction beginTransaction() throws TskCoreException, SQLException { return new DrawableTransaction(); } /** - * + * * @param tr * @param notifyGM If true, notify GroupManager about the changes. */ - public void commitTransaction(DrawableTransaction tr, Boolean notifyGM) { - if (tr.isClosed()) { - throw new IllegalArgumentException("can't close already closed transaction"); + public void commitTransaction(DrawableTransaction tr, Boolean notifyGM) throws SQLException { + if (tr.isCompleted()) { + throw new IllegalArgumentException("Attempt to commit completed transaction"); } tr.commit(notifyGM); } - public void rollbackTransaction(DrawableTransaction tr) { - if (tr.isClosed()) { - throw new IllegalArgumentException("can't rollback already closed transaction"); + public void rollbackTransaction(DrawableTransaction tr) throws SQLException { + if (tr.isCompleted()) { + throw new IllegalArgumentException("Attempt to roll back completed transaction"); } tr.rollback(); } - public Boolean isFileAnalyzed(DrawableFile f) { - return isFileAnalyzed(f.getId()); - } - - public Boolean isFileAnalyzed(long fileId) { - dbReadLock(); - try (Statement stmt = con.createStatement(); - ResultSet analyzedQuery = stmt.executeQuery("SELECT analyzed FROM drawable_files WHERE obj_id = " + fileId)) { //NON-NLS - while (analyzedQuery.next()) { - return analyzedQuery.getBoolean(ANALYZED); - } - } catch (SQLException ex) { - String msg = String.format("Failed to determine if file %s is finalized", String.valueOf(fileId)); //NON-NLS - logger.log(Level.WARNING, msg, ex); - } finally { - dbReadUnlock(); - } - - return false; - } - - public Boolean areFilesAnalyzed(Collection fileIds) { - - dbReadLock(); - try (Statement stmt = con.createStatement(); - //Can't make this a preprared statement because of the IN ( ... ) - ResultSet analyzedQuery = stmt.executeQuery("SELECT COUNT(analyzed) AS analyzed FROM drawable_files WHERE analyzed = 1 AND obj_id IN (" + StringUtils.join(fileIds, ", ") + ")")) { //NON-NLS - while (analyzedQuery.next()) { - return analyzedQuery.getInt(ANALYZED) == fileIds.size(); - } - } catch (SQLException ex) { - logger.log(Level.WARNING, "problem counting analyzed files: ", ex); //NON-NLS - } finally { - dbReadUnlock(); - } - - return false; - } - - public Boolean isGroupAnalyzed(GroupKey gk) { - dbReadLock(); + public Boolean areFilesAnalyzed(Collection fileIds) throws SQLException { + dbWriteLock(); try { - Set fileIDsInGroup = getFileIDsInGroup(gk); - try { + if (isClosed()) { + throw new SQLException("The drawables database is closed"); + } + try (Statement stmt = con.createStatement()) { + //Can't make this a preprared statement because of the IN ( ... ) + ResultSet analyzedQuery = stmt.executeQuery("SELECT COUNT(analyzed) AS analyzed FROM drawable_files WHERE analyzed = 1 AND obj_id IN (" + StringUtils.join(fileIds, ", ") + ")"); //NON-NLS + while (analyzedQuery.next()) { + return analyzedQuery.getInt(ANALYZED) == fileIds.size(); + } + return false; + } + } finally { + dbWriteUnlock(); + } + } + + public Boolean isGroupAnalyzed(GroupKey gk) throws SQLException, TskCoreException { + dbWriteLock(); + try { + if (isClosed()) { + throw new SQLException("The drawables database is closed"); + } + try (Statement stmt = con.createStatement()) { // In testing, this method appears to be a lot faster than doing one large select statement + Set fileIDsInGroup = getFileIDsInGroup(gk); for (Long fileID : fileIDsInGroup) { - Statement stmt = con.createStatement(); ResultSet analyzedQuery = stmt.executeQuery("SELECT analyzed FROM drawable_files WHERE obj_id = " + fileID); //NON-NLS while (analyzedQuery.next()) { if (analyzedQuery.getInt(ANALYZED) == 0) { return false; } } - return true; + return true; // THIS APPEARS TO BE A BUG (see JIRA-1130), THE FOR LOOP EXECUTES AT MOST ONCE } - - } catch (SQLException ex) { - logger.log(Level.WARNING, "problem counting analyzed files: ", ex); //NON-NLS } - } catch (TskCoreException tskCoreException) { - logger.log(Level.WARNING, "problem counting analyzed files: ", tskCoreException); //NON-NLS + return false; } finally { - dbReadUnlock(); + dbWriteUnlock(); } - return false; } /** @@ -1279,21 +1350,24 @@ public final class DrawableDB { * @throws TskCoreException */ public Set findAllFileIdsWhere(String sqlWhereClause) throws TskCoreException { - - Set ret = new HashSet<>(); - dbReadLock(); - try (Statement statement = con.createStatement(); - ResultSet rs = statement.executeQuery("SELECT obj_id FROM drawable_files WHERE " + sqlWhereClause);) { - while (rs.next()) { - ret.add(rs.getLong(1)); + dbWriteLock(); + try { + if (isClosed()) { + throw new TskCoreException("The drawables database is closed"); + } + try (Statement statement = con.createStatement()) { + ResultSet rs = statement.executeQuery("SELECT obj_id FROM drawable_files WHERE " + sqlWhereClause); + Set ret = new HashSet<>(); + while (rs.next()) { + ret.add(rs.getLong(1)); + } + return ret; + } catch (SQLException ex) { + throw new TskCoreException(String.format("Failed to query file id for WHERE clause %s", sqlWhereClause), ex); } - } catch (SQLException e) { - throw new TskCoreException("SQLException thrown when calling 'DrawableDB.findAllFileIdsWhere(): " + sqlWhereClause, e); } finally { - - dbReadUnlock(); + dbWriteUnlock(); } - return ret; } /** @@ -1308,14 +1382,19 @@ public final class DrawableDB { * @throws TskCoreException */ public long countFilesWhere(String sqlWhereClause) throws TskCoreException { - dbReadLock(); - try (Statement statement = con.createStatement(); - ResultSet rs = statement.executeQuery("SELECT COUNT(*) AS COUNT FROM drawable_files WHERE " + sqlWhereClause);) { - return rs.getLong("COUNT"); - } catch (SQLException e) { - throw new TskCoreException("SQLException thrown when calling 'DrawableDB.countFilesWhere(): " + sqlWhereClause, e); + dbWriteLock(); + try { + if (isClosed()) { + throw new TskCoreException("The drawables database is closed"); + } + try (Statement statement = con.createStatement()) { + ResultSet rs = statement.executeQuery("SELECT COUNT(*) AS COUNT FROM drawable_files WHERE " + sqlWhereClause); + return rs.getLong("COUNT"); + } catch (SQLException e) { + throw new TskCoreException("SQLException thrown when calling 'DrawableDB.countFilesWhere(): " + sqlWhereClause, e); + } } finally { - dbReadUnlock(); + dbWriteUnlock(); } } @@ -1330,15 +1409,14 @@ public final class DrawableDB { * @param sortOrder Sort ascending or descending. * @param dataSource * - * @return Map of data source (or null of group by attribute ignores data sources) to list of unique group values + * @return Map of data source (or null of group by attribute ignores data + * sources) to list of unique group values * * @throws org.sleuthkit.datamodel.TskCoreException */ @SuppressWarnings("unchecked") public > Multimap findValuesForAttribute(DrawableAttribute groupBy, GroupSortBy sortBy, SortOrder sortOrder, DataSource dataSource) throws TskCoreException { - Multimap values = HashMultimap.create(); - switch (groupBy.attrName) { case ANALYZED: case CATEGORY: @@ -1347,76 +1425,69 @@ public final class DrawableDB { //they should have special handling at a higher level of the stack. throw new UnsupportedOperationException(); default: - dbReadLock(); - //TODO: convert this to prepared statement - - StringBuilder query = new StringBuilder("SELECT data_source_obj_id, " + groupBy.attrName.toString() + ", COUNT(*) FROM drawable_files "); //NON-NLS - - if (dataSource != null) { - query.append(" WHERE data_source_obj_id = ").append(dataSource.getId()); - } - - query.append(" GROUP BY data_source_obj_id, ").append(groupBy.attrName.toString()); - - String orderByClause = ""; - - if (sortBy == GROUP_BY_VALUE) { - orderByClause = " ORDER BY " + groupBy.attrName.toString(); - } else if (sortBy == GroupSortBy.FILE_COUNT) { - orderByClause = " ORDER BY COUNT(*)"; - } - - query.append(orderByClause); - - if (orderByClause.isEmpty() == false) { - String sortOrderClause = ""; - - switch (sortOrder) { - case DESCENDING: - sortOrderClause = " DESC"; //NON-NLS - break; - case ASCENDING: - sortOrderClause = " ASC"; //NON-NLS - break; - default: - orderByClause = ""; + dbWriteLock(); + try { + if (isClosed()) { + throw new TskCoreException("The drawables database is closed"); } - query.append(sortOrderClause); - } + //TODO: convert this to prepared statement + StringBuilder query = new StringBuilder("SELECT data_source_obj_id, " + groupBy.attrName.toString() + ", COUNT(*) FROM drawable_files "); //NON-NLS - try (Statement stmt = con.createStatement(); - ResultSet results = stmt.executeQuery(query.toString())) { - while (results.next()) { - /* - * I don't like that we have to do this cast to A here, - * but can't think of a better alternative at the - * momment unless something has gone seriously wrong, we - * know this should be of type A even if JAVA doesn't - */ - values.put(tskCase.getDataSource(results.getLong("data_source_obj_id")), - (A) results.getObject(groupBy.attrName.toString())); + if (dataSource != null) { + query.append(" WHERE data_source_obj_id = ").append(dataSource.getId()); } - } catch (SQLException ex) { - if (!(ex.getCause() instanceof java.lang.InterruptedException)) { - /* It seems like this originaly comes out of c3p0 when - * its thread is intereupted (cancelled because of - * regroup). It should be safe to just swallow this and - * move on. - * - * see - * https://sourceforge.net/p/c3p0/mailman/c3p0-users/thread/EBB32BB8-6487-43AF-B291-9464C9051869@mchange.com/ - */ + query.append(" GROUP BY data_source_obj_id, ").append(groupBy.attrName.toString()); + + String orderByClause = ""; + + if (sortBy == GROUP_BY_VALUE) { + orderByClause = " ORDER BY " + groupBy.attrName.toString(); + } else if (sortBy == GroupSortBy.FILE_COUNT) { + orderByClause = " ORDER BY COUNT(*)"; + } + + query.append(orderByClause); + + if (orderByClause.isEmpty() == false) { + String sortOrderClause = ""; + + switch (sortOrder) { + case DESCENDING: + sortOrderClause = " DESC"; //NON-NLS + break; + case ASCENDING: + sortOrderClause = " ASC"; //NON-NLS + break; + default: + orderByClause = ""; + } + query.append(sortOrderClause); + } + + try (Statement stmt = con.createStatement()) { + ResultSet results = stmt.executeQuery(query.toString()); + Multimap values = HashMultimap.create(); + while (results.next()) { + /* + * I don't like that we have to do this cast to A + * here, but can't think of a better alternative at + * the momment unless something has gone seriously + * wrong, we know this should be of type A even if + * JAVA doesn't + */ + values.put(tskCase.getDataSource(results.getLong("data_source_obj_id")), + (A) results.getObject(groupBy.attrName.toString())); + } + return values; + } catch (SQLException | TskDataException ex) { throw new TskCoreException("Unable to get values for attribute", ex); //NON-NLS } - } catch (TskDataException ex) { - throw new TskCoreException("Unable to get values for attribute", ex); //NON-NLS + } finally { - dbReadUnlock(); + dbWriteUnlock(); } } - - return values; } /** @@ -1425,8 +1496,10 @@ public final class DrawableDB { * @param value Value of the group (unique to the type) * @param groupBy Type of the grouping (CATEGORY, MAKE, etc.) * @param caseDbTransaction transaction to use for CaseDB insert/updates + * + * @throws TskCoreException */ - private void insertGroup(final String value, DrawableAttribute groupBy, CaseDbTransaction caseDbTransaction) { + private void insertGroup(final String value, DrawableAttribute groupBy, CaseDbTransaction caseDbTransaction) throws TskCoreException { insertGroup(0, value, groupBy, caseDbTransaction); } @@ -1438,27 +1511,23 @@ public final class DrawableDB { * @param groupBy Type of the grouping (CATEGORY, MAKE, etc.) * @param caseDbTransaction transaction to use for CaseDB insert/updates */ - private void insertGroup(long ds_obj_id, final String value, DrawableAttribute groupBy, CaseDbTransaction caseDbTransaction) { - // don't waste DB round trip if we recently added it + private void insertGroup(long ds_obj_id, final String value, DrawableAttribute groupBy, CaseDbTransaction caseDbTransaction) throws TskCoreException { + /* + * Check the groups cache to see if the group has already been added to + * the case database. + */ String cacheKey = Long.toString(ds_obj_id) + "_" + value + "_" + groupBy.getDisplayName(); - if (groupCache.getIfPresent(cacheKey) != null) + if (groupCache.getIfPresent(cacheKey) != null) { return; - - try { - String insertSQL = String.format(" (data_source_obj_id, value, attribute) VALUES (%d, \'%s\', \'%s\')", - ds_obj_id, SleuthkitCase.escapeSingleQuotes(value), SleuthkitCase.escapeSingleQuotes(groupBy.attrName.toString())); - - if (DbType.POSTGRESQL == tskCase.getDatabaseType()) { - insertSQL += " ON CONFLICT DO NOTHING"; - } - tskCase.getCaseDbAccessManager().insert(GROUPS_TABLENAME, insertSQL, caseDbTransaction); - groupCache.put(cacheKey, Boolean.TRUE); - } catch (TskCoreException ex) { - // Don't need to report it if the case was closed - if (Case.isCaseOpen()) { - logger.log(Level.SEVERE, "Unable to insert group", ex); //NON-NLS - } } + + String insertSQL = String.format(" (data_source_obj_id, value, attribute) VALUES (%d, \'%s\', \'%s\')", + ds_obj_id, SleuthkitCase.escapeSingleQuotes(value), SleuthkitCase.escapeSingleQuotes(groupBy.attrName.toString())); + if (DbType.POSTGRESQL == tskCase.getDatabaseType()) { + insertSQL += " ON CONFLICT DO NOTHING"; + } + tskCase.getCaseDbAccessManager().insert(GROUPS_TABLENAME, insertSQL, caseDbTransaction); + groupCache.put(cacheKey, Boolean.TRUE); } /** @@ -1470,13 +1539,11 @@ public final class DrawableDB { * {@link SleuthkitCase} */ public DrawableFile getFileFromID(Long id) throws TskCoreException { + AbstractFile f = tskCase.getAbstractFileById(id); try { - AbstractFile f = tskCase.getAbstractFileById(id); - return DrawableFile.create(f, - areFilesAnalyzed(Collections.singleton(id)), isVideoFile(f)); - } catch (IllegalStateException ex) { - logger.log(Level.SEVERE, "there is no case open; failed to load file with id: {0}", id); //NON-NLS - throw new TskCoreException("there is no case open; failed to load file with id: " + id, ex); + return DrawableFile.create(f, areFilesAnalyzed(Collections.singleton(id)), isVideoFile(f)); + } catch (SQLException ex) { + throw new TskCoreException(String.format("Failed to get file (id=%d)", id), ex); } } @@ -1493,7 +1560,7 @@ public final class DrawableDB { } } Set files = new HashSet<>(); - dbReadLock(); + dbWriteLock(); try { PreparedStatement statement = getGroupStatment(groupKey); setQueryParams(statement, groupKey); @@ -1506,18 +1573,12 @@ public final class DrawableDB { } catch (SQLException ex) { logger.log(Level.WARNING, "failed to get file for group:" + groupKey.getAttribute() + " == " + groupKey.getValue(), ex); //NON-NLS } finally { - dbReadUnlock(); + dbWriteUnlock(); } return files; } - private void closeStatements() throws SQLException { - for (PreparedStatement pStmt : preparedStatements) { - pStmt.close(); - } - } - private PreparedStatement getGroupStatment(GroupKey groupKey) { DrawableAttribute groupBy = groupKey.getAttribute(); if ((groupBy == DrawableAttribute.PATH) && groupKey.getDataSource().isPresent()) { @@ -1544,16 +1605,12 @@ public final class DrawableDB { * delete the row with obj_id = id. * * @param id the obj_id of the row to be deleted - * - * @return true if a row was deleted, 0 if not. */ - public boolean removeFile(long id, DrawableTransaction tr) { - if (tr.isClosed()) { - throw new IllegalArgumentException("can't update database with closed transaction"); + public void removeFile(long id, DrawableTransaction tr) { + if (tr.isCompleted()) { + throw new IllegalArgumentException("Attempt to use a completed transaction"); } - int valsResults = 0; dbWriteLock(); - try { // Update the list of file IDs in memory removeImageFileFromList(id); @@ -1561,7 +1618,7 @@ public final class DrawableDB { //"delete from hash_set_hits where (obj_id = " + id + ")" removeHashHitStmt.setLong(1, id); removeHashHitStmt.executeUpdate(); - + //"delete from drawable_files where (obj_id = " + id + ")" removeFileStmt.setLong(1, id); removeFileStmt.executeUpdate(); @@ -1572,10 +1629,6 @@ public final class DrawableDB { } finally { dbWriteUnlock(); } - - //indicates succesfull removal of 1 file - return valsResults == 1; - } public class MultipleTransactionException extends IllegalStateException { @@ -1617,19 +1670,25 @@ public final class DrawableDB { } } - private void initializeImageList() { - synchronized (fileIDsInDB) { - dbReadLock(); - try (Statement stmt = con.createStatement(); - ResultSet analyzedQuery = stmt.executeQuery("select obj_id from drawable_files");) { + private boolean initializeImageList() { + dbWriteLock(); + try { + if (isClosed()) { + logger.log(Level.SEVERE, "The drawables database is closed"); //NON-NLS + return false; + } + try (Statement stmt = con.createStatement()) { + ResultSet analyzedQuery = stmt.executeQuery("select obj_id from drawable_files"); while (analyzedQuery.next()) { addImageFileToList(analyzedQuery.getLong(OBJ_ID)); } + return true; } catch (SQLException ex) { - logger.log(Level.WARNING, "problem loading file IDs: ", ex); //NON-NLS - } finally { - dbReadUnlock(); + logger.log(Level.SEVERE, "Failed to add image file object ids in drawables database to cache", ex); //NON-NLS + return false; } + } finally { + dbWriteUnlock(); } } @@ -1721,7 +1780,7 @@ public final class DrawableDB { //count the file ids that are in the given list and don't have a non-zero category assigned to them. String name = "SELECT COUNT(obj_id) as obj_count FROM tsk_files where obj_id IN " + fileIdsList //NON-NLS - + " AND obj_id NOT IN (SELECT obj_id FROM content_tags WHERE content_tags.tag_name_id IN " + catTagNameIDs + ")"; //NON-NLS + + " AND obj_id NOT IN (SELECT obj_id FROM content_tags WHERE content_tags.tag_name_id IN " + catTagNameIDs + ")"; //NON-NLS try (SleuthkitCase.CaseDbQuery executeQuery = tskCase.executeQuery(name); ResultSet resultSet = executeQuery.getResultSet();) { while (resultSet.next()) { @@ -1735,63 +1794,63 @@ public final class DrawableDB { } /** - * inner class that can reference access database connection + * Encapsulates a drawables database transaction that uses the enclosing + * DrawableDB object's single JDBC connection. The transaction is begun when + * the DrawableTransaction object is created; clients MUST call either + * commit or rollback. + * + * IMPORTANT: This transaction must be thread-confined. It acquires and + * release a lock specific to a single thread. */ public class DrawableTransaction { - private final Set updatedFiles; + private final Set updatedFiles = new HashSet<>(); + private final Set removedFiles = new HashSet<>(); + private boolean completed; - private final Set removedFiles; - - private boolean closed = false; - - /** - * factory creation method - * - * @param con the {@link ava.sql.Connection} - * - * @return a LogicalFileTransaction for the given connection - * - * @throws SQLException - */ - private DrawableTransaction() { - this.updatedFiles = new HashSet<>(); - this.removedFiles = new HashSet<>(); - //get the write lock, released in close() - dbWriteLock(); + private DrawableTransaction() throws TskCoreException, SQLException { + dbWriteLock(); // Normally released when commit or rollback is called. + if (DrawableDB.this.isClosed()) { + dbWriteUnlock(); + throw new TskCoreException("The drawables database is closed"); + } try { con.setAutoCommit(false); - + completed = false; } catch (SQLException ex) { - logger.log(Level.SEVERE, "failed to set auto-commit to to false", ex); //NON-NLS + completed = true; + dbWriteUnlock(); + throw new SQLException("Failed to begin transaction", ex); } - } - synchronized public void rollback() { - if (!closed) { + synchronized public void rollback() throws SQLException { + if (!completed) { try { - con.rollback(); updatedFiles.clear(); - } catch (SQLException ex1) { - logger.log(Level.SEVERE, "Exception while attempting to rollback!!", ex1); //NON-NLS + con.rollback(); } finally { - close(); + complete(); } } } /** * Commit changes that happened during this transaction - * - * @param notifyGM If true, notify GroupManager about the changes. + * + * @param notifyGM If true, notify GroupManager about the changes. */ - synchronized private void commit(Boolean notifyGM) { - if (!closed) { + synchronized public void commit(Boolean notifyGM) throws SQLException { + if (!completed) { try { + con.commit(); - // make sure we close before we update, bc they'll need locks - close(); + + /* + * Need to close the transaction before notifying the Group + * Manager, so that the lock is released. + */ + complete(); if (notifyGM) { if (groupManager != null) { @@ -1800,35 +1859,27 @@ public final class DrawableDB { } } } catch (SQLException ex) { - if (Case.isCaseOpen()) { - logger.log(Level.SEVERE, "Error commiting drawable.db.", ex); //NON-NLS - } else { - logger.log(Level.WARNING, "Error commiting drawable.db - case is closed."); //NON-NLS - } + logger.log(Level.SEVERE, "Failed to commit transaction, will attempt rollback", ex); //NON-NLS rollback(); } } } - synchronized private void close() { - if (!closed) { + synchronized private void complete() { + if (!completed) { try { con.setAutoCommit(true); } catch (SQLException ex) { - if (Case.isCaseOpen()) { - logger.log(Level.SEVERE, "Error setting auto-commit to true.", ex); //NON-NLS - } else { - logger.log(Level.SEVERE, "Error setting auto-commit to true - case is closed"); //NON-NLS - } + logger.log(Level.SEVERE, "Failed to set auto-commit to false", ex); //NON-NLS } finally { - closed = true; + completed = true; dbWriteUnlock(); } } } - synchronized public Boolean isClosed() { - return closed; + synchronized private Boolean isCompleted() { + return completed; } synchronized private void addUpdatedFile(Long f) { diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/HashSetManager.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/HashSetManager.java index ffcf307f21..c94492c39f 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/HashSetManager.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/HashSetManager.java @@ -33,7 +33,9 @@ import org.sleuthkit.datamodel.TskCoreException; */ public class HashSetManager { - /** The db that initial values are loaded from. */ + /** + * The db that initial values are loaded from. + */ private final DrawableDB drawableDB; public HashSetManager(DrawableDB drawableDB) { @@ -54,14 +56,9 @@ public class HashSetManager { */ private Set getHashSetsForFileHelper(long fileID) { try { - if (drawableDB.isClosed()) { - Logger.getLogger(HashSetManager.class.getName()).log(Level.WARNING, "Failed to get Hash Sets for file. The Db connection was already closed."); //NON-NLS - return Collections.emptySet(); - } else { - return drawableDB.getHashSetsForFile(fileID); - } - } catch (TskCoreException | SQLException ex) { - Logger.getLogger(HashSetManager.class.getName()).log(Level.SEVERE, "Failed to get Hash Sets for file."); //NON-NLS + return drawableDB.getHashSetsForFile(fileID); + } catch (TskCoreException ex) { + Logger.getLogger(HashSetManager.class.getName()).log(Level.SEVERE, String.format("Failed to get hash sets for file (id=%d)", fileID), ex); //NON-NLS return Collections.emptySet(); } } diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupManager.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupManager.java index bfd0fab496..99069b7790 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupManager.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupManager.java @@ -99,18 +99,24 @@ public class GroupManager { private static final Logger logger = Logger.getLogger(GroupManager.class.getName()); - /** An executor to submit async UI related background tasks to. */ + /** + * An executor to submit async UI related background tasks to. + */ private final ListeningExecutorService exec = MoreExecutors.listeningDecorator(Executors.newSingleThreadExecutor( new BasicThreadFactory.Builder().namingPattern("GroupManager BG Thread-%d").build())); //NON-NLS private final ImageGalleryController controller; - /** list of all analyzed groups */ + /** + * list of all analyzed groups + */ @GuardedBy("this") //NOPMD private final ObservableList analyzedGroups = FXCollections.observableArrayList(); private final ObservableList unmodifiableAnalyzedGroups = FXCollections.unmodifiableObservableList(analyzedGroups); - /** list of unseen groups */ + /** + * list of unseen groups + */ @GuardedBy("this") //NOPMD private final ObservableList unSeenGroups = FXCollections.observableArrayList(); private final ObservableList unmodifiableUnSeenGroups = FXCollections.unmodifiableObservableList(unSeenGroups); @@ -487,8 +493,8 @@ public class GroupManager { setSortOrder(sortOrder); //only re-query the db if the data source or group by attribute changed or it is forced if (dataSource != getDataSource() - || groupBy != getGroupBy() - || force) { + || groupBy != getGroupBy() + || force) { setDataSource(dataSource); setGroupBy(groupBy); @@ -645,9 +651,9 @@ public class GroupManager { * analyzed because we don't know all the files that will be a part of * that group. just show them no matter what. */ - if (groupKey.getAttribute() != DrawableAttribute.PATH - || getDrawableDB().isGroupAnalyzed(groupKey)) { - try { + try { + if (groupKey.getAttribute() != DrawableAttribute.PATH + || getDrawableDB().isGroupAnalyzed(groupKey)) { Set fileIDs = getFileIDsInGroup(groupKey); if (Objects.nonNull(fileIDs)) { @@ -673,9 +679,9 @@ public class GroupManager { return group; } - } catch (TskCoreException ex) { - logger.log(Level.SEVERE, "failed to get files for group: " + groupKey.getAttribute().attrName.toString() + " = " + groupKey.getValue(), ex); //NON-NLS } + } catch (SQLException | TskCoreException ex) { + logger.log(Level.SEVERE, "Failed to get files for group: " + groupKey.getAttribute().attrName.toString() + " = " + groupKey.getValue(), ex); //NON-NLS } return null; @@ -735,7 +741,7 @@ public class GroupManager { */ @SuppressWarnings({"unchecked", "rawtypes"}) @NbBundle.Messages({"# {0} - groupBy attribute Name", - "ReGroupTask.displayTitle=regrouping by {0}: " }) + "ReGroupTask.displayTitle=regrouping by {0}: "}) class ReGroupTask> extends LoggedTask { private final DataSource dataSource; @@ -744,13 +750,13 @@ public class GroupManager { private final SortOrder sortOrder; ReGroupTask(DataSource dataSource, DrawableAttribute groupBy, GroupSortBy sortBy, SortOrder sortOrder) { - super(Bundle.ReGroupTask_displayTitle(groupBy.attrName.toString() ), true); + super(Bundle.ReGroupTask_displayTitle(groupBy.attrName.toString()), true); this.dataSource = dataSource; this.groupBy = groupBy; this.sortBy = sortBy; this.sortOrder = sortOrder; - updateTitle(Bundle.ReGroupTask_displayTitle(groupBy.attrName.toString() )); + updateTitle(Bundle.ReGroupTask_displayTitle(groupBy.attrName.toString())); } @Override @@ -791,8 +797,8 @@ public class GroupManager { = viewedKey.map(GroupKey::getAttribute).orElse(null); if (viewedGroup.isPresent() == false //if no group was being viewed, - || (dataSource != null && notEqual(dataSourceOfCurrentGroup, dataSource)) //or the datasource of the viewed group is wrong, - || groupBy != attributeOfCurrentGroup) { // or the groupBy attribute is wrong... + || (dataSource != null && notEqual(dataSourceOfCurrentGroup, dataSource)) //or the datasource of the viewed group is wrong, + || groupBy != attributeOfCurrentGroup) { // or the groupBy attribute is wrong... //the current group should not be visible so ... if (isNotEmpty(unSeenGroups)) { From 2b84fde7e68909a586907e8b1b15bad54a26ce5f Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Mon, 19 Nov 2018 09:25:31 -0500 Subject: [PATCH 117/129] Correct typo in DrawableDB comment --- .../sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java index 3fa918e8b4..8d8cae44b9 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java @@ -150,7 +150,7 @@ public final class DrawableDB { private final ReentrantReadWriteLock rwLock = new ReentrantReadWriteLock(true); //use fairness policy - private final Lock DBLock = rwLock.writeLock(); // Currently serializing everything with on database connection + private final Lock DBLock = rwLock.writeLock(); // Currently serializing everything with one database connection // caches to make inserts / updates faster private Cache groupCache = CacheBuilder.newBuilder().expireAfterWrite(5, TimeUnit.MINUTES).build(); From 80cd3a08e540058daa709a3e7bb5670b11f52efa Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Mon, 19 Nov 2018 17:13:52 -0500 Subject: [PATCH 118/129] Remove commented out code and unused case db trans from DrawableDB --- .../imagegallery/datamodel/DrawableDB.java | 46 ++----------------- 1 file changed, 3 insertions(+), 43 deletions(-) diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java index 8d8cae44b9..985a679014 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java @@ -183,46 +183,14 @@ public final class DrawableDB { DEFAULT; /// Not all files in the data source have had file type detected } - //////////////general database logic , mostly borrowed from sleuthkitcase - /** - * Lock to protect against concurrent write accesses to case database and to - * block readers while database is in write transaction. Should be utilized - * by all db code where underlying storage supports max. 1 concurrent writer - * MUST always call dbWriteUnLock() as early as possible, in the same thread - * where dbWriteLock() was called - */ - public void dbWriteLock() { - //Logger.getLogger("LOCK").log(Level.INFO, "Locking " + rwLock.toString()); + private void dbWriteLock() { DBLock.lock(); } - /** - * Release previously acquired write lock acquired in this thread using - * dbWriteLock(). Call in "finally" block to ensure the lock is always - * released. - */ - public void dbWriteUnlock() { - //Logger.getLogger("LOCK").log(Level.INFO, "UNLocking " + rwLock.toString()); + private void dbWriteUnlock() { DBLock.unlock(); } - - /** - * Lock to protect against read while it is in a write transaction state. - * Supports multiple concurrent readers if there is no writer. MUST always - * call dbReadUnLock() as early as possible, in the same thread where - * dbReadLock() was called. - */ -// void dbReadLock() { -// DBLock.lock(); -// } - /** - * Release previously acquired read lock acquired in this thread using - * dbReadLock(). Call in "finally" block to ensure the lock is always - * released. - */ -// void dbReadUnlock() { -// DBLock.unlock(); -// } + /** * Constructs an object that provides access to the drawables database and * selected tables in the case database. If the specified drawables database @@ -869,19 +837,11 @@ public final class DrawableDB { */ public void removeFile(long id) throws TskCoreException, SQLException { DrawableTransaction trans = null; - CaseDbTransaction caseDbTransaction = null; try { trans = beginTransaction(); removeFile(id, trans); commitTransaction(trans, true); } catch (TskCoreException | SQLException ex) { - if (null != caseDbTransaction) { - try { - caseDbTransaction.rollback(); - } catch (TskCoreException ex2) { - logger.log(Level.SEVERE, String.format("Failed to roll back case db transaction after error: %s", ex.getMessage()), ex2); //NON-NLS - } - } if (null != trans) { try { rollbackTransaction(trans); From c769af555c73ad968540023f83963b67b99a7d2a Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Mon, 19 Nov 2018 19:38:46 -0500 Subject: [PATCH 119/129] Eliminate drawables db creation side effect --- .../imagegallery/datamodel/DrawableDB.java | 79 +++++++++---------- 1 file changed, 37 insertions(+), 42 deletions(-) diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java index 985a679014..06f88aca93 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/DrawableDB.java @@ -190,7 +190,7 @@ public final class DrawableDB { private void dbWriteUnlock() { DBLock.unlock(); } - + /** * Constructs an object that provides access to the drawables database and * selected tables in the case database. If the specified drawables database @@ -217,7 +217,7 @@ public final class DrawableDB { con = DriverManager.getConnection("jdbc:sqlite:" + dbPath.toString()); //NON-NLS if (!initializeDBSchema() || !prepareStatements() || !initializeStandardGroups() || !initializeImageList()) { close(); - throw new TskCoreException("Failed to create or open drawables database"); //NON-NLS + throw new TskCoreException("Failed to initialize drawables database for Image Gallery use"); //NON-NLS } } finally { dbWriteUnlock(); @@ -372,59 +372,52 @@ public final class DrawableDB { */ public static DrawableDB getDrawableDB(ImageGalleryController controller) throws TskCoreException { Path dbPath = ImageGalleryModule.getModuleOutputDir(controller.getAutopsyCase()).resolve("drawable.db"); - boolean hasDataSourceObjIdColumn = hasDataSourceObjIdColumn(dbPath); try { - if (hasDataSourceObjIdColumn == false) { - Files.deleteIfExists(dbPath); - } + deleteDatabaseIfOlderVersion(dbPath); + } catch (SQLException ex) { + throw new TskCoreException("Failed to check for obsolete drawables database schema", ex); //NON-NLS } catch (IOException ex) { - throw new TskCoreException("Error deleting old database", ex); //NON-NLS + throw new TskCoreException("Failed to delete obsolete drawables database", ex); //NON-NLS } try { - return new DrawableDB(dbPath, controller); //NON-NLS - } catch (SQLException ex) { - throw new TskCoreException("SQL error creating database connection", ex); //NON-NLS + return new DrawableDB(dbPath, controller); } catch (IOException ex) { - throw new TskCoreException("Error creating database connection", ex); //NON-NLS + throw new TskCoreException("Failed to create drawables database directory", ex); //NON-NLS + } catch (SQLException ex) { + throw new TskCoreException("Failed to create/open the drawables database", ex); //NON-NLS } } - /** - * Check if the db at the given path has the data_source_obj_id column. If - * the db doesn't exist or doesn't even have the drawable_files table, this - * method returns false. - * - * NOTE: This method makes an ad-hoc connection to db, which has the side - * effect of creating the drawable.db file if it didn't already exist. - */ - private static boolean hasDataSourceObjIdColumn(Path dbPath) throws TskCoreException { - - try (Connection con = DriverManager.getConnection("jdbc:sqlite:" + dbPath.toString()); //NON-NLS - Statement stmt = con.createStatement();) { - boolean tableExists = false; - try (ResultSet results = stmt.executeQuery("SELECT name FROM sqlite_master WHERE type='table'");) {//NON-NLS - while (results.next()) { - if ("drawable_files".equals(results.getString("name"))) { - tableExists = true; - break; + private static void deleteDatabaseIfOlderVersion(Path dbPath) throws SQLException, IOException { + if (Files.exists(dbPath)) { + boolean hasDrawableFilesTable = false; + boolean hasDataSourceIdColumn = false; + try (Connection con = DriverManager.getConnection("jdbc:sqlite:" + dbPath.toString())) { + Statement stmt = con.createStatement(); + try (ResultSet tableQueryResults = stmt.executeQuery("SELECT name FROM sqlite_master WHERE type='table'")) { //NON-NLS + while (tableQueryResults.next()) { + if ("drawable_files".equals(tableQueryResults.getString("name"))) { + hasDrawableFilesTable = true; + break; + } + } + } + if (hasDrawableFilesTable) { + try (ResultSet results = stmt.executeQuery("PRAGMA table_info('drawable_files')")) { + while (results.next()) { + if ("data_source_obj_id".equals(results.getString("name"))) { + hasDataSourceIdColumn = true; + break; + } + } } } } - if (false == tableExists) { - return false; + if (!hasDrawableFilesTable || !hasDataSourceIdColumn) { + Files.delete(dbPath); } - try (ResultSet results = stmt.executeQuery("PRAGMA table_info('drawable_files')");) { //NON-NLS - while (results.next()) { - if ("data_source_obj_id".equals(results.getString("name"))) { - return true; - } - } - } - } catch (SQLException ex) { - throw new TskCoreException("SQL error checking database compatibility", ex); //NON-NLS } - return false; } private void setPragmas() throws SQLException { @@ -777,7 +770,7 @@ public final class DrawableDB { } } } - // Callback to process result of seen query +// Callback to process result of seen query GroupSeenQueryResultProcessor queryResultProcessor = new GroupSeenQueryResultProcessor(); try { @@ -1588,6 +1581,7 @@ public final class DrawableDB { logger.log(Level.WARNING, "failed to delete row for obj_id = " + id, ex); //NON-NLS } finally { dbWriteUnlock(); + } } @@ -1751,6 +1745,7 @@ public final class DrawableDB { } return -1; + } /** From 036da13f8fa8417898b514f09b608233ce23f909 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Tue, 20 Nov 2018 09:22:31 -0500 Subject: [PATCH 120/129] Removed SCO from background task --- .../datamodel/AbstractAbstractFileNode.java | 39 +++-- .../datamodel/SCOAndTranslationTask.java | 141 ------------------ .../autopsy/datamodel/TranslationTask.java | 59 ++++++++ 3 files changed, 78 insertions(+), 161 deletions(-) delete mode 100755 Core/src/org/sleuthkit/autopsy/datamodel/SCOAndTranslationTask.java create mode 100755 Core/src/org/sleuthkit/autopsy/datamodel/TranslationTask.java diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java index 03a733faaa..17c0b2052f 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java @@ -56,7 +56,6 @@ import org.sleuthkit.autopsy.corecomponents.DataResultViewerTable.Score; import org.sleuthkit.autopsy.coreutils.Logger; import static org.sleuthkit.autopsy.datamodel.Bundle.*; import static org.sleuthkit.autopsy.datamodel.AbstractAbstractFileNode.AbstractFilePropertyType.*; -import org.sleuthkit.autopsy.datamodel.SCOAndTranslationTask.SCOResults; import org.sleuthkit.autopsy.ingest.IngestManager; import org.sleuthkit.autopsy.ingest.ModuleContentEvent; import org.sleuthkit.autopsy.texttranslation.NoServiceProviderException; @@ -83,7 +82,7 @@ public abstract class AbstractAbstractFileNode extends A private static final Set CASE_EVENTS_OF_INTEREST = EnumSet.of(Case.Events.CURRENT_CASE, Case.Events.CONTENT_TAG_ADDED, Case.Events.CONTENT_TAG_DELETED, Case.Events.CR_COMMENT_CHANGED); - private static final ExecutorService SCOAndTranslationPool; + private static final ExecutorService translationPool; private static final Integer MAX_POOL_SIZE = 10; /** @@ -108,8 +107,8 @@ public abstract class AbstractAbstractFileNode extends A static { //Initialize this pool only once! This will be used by every instance of AAFN //to do their heavy duty SCO column and translation updates. - SCOAndTranslationPool = Executors.newFixedThreadPool(MAX_POOL_SIZE, - new ThreadFactoryBuilder().setNameFormat("SCOandTranslation-thread-%d").build()); + translationPool = Executors.newFixedThreadPool(MAX_POOL_SIZE, + new ThreadFactoryBuilder().setNameFormat("translation-task-thread-%d").build()); } /** @@ -134,14 +133,12 @@ public abstract class AbstractAbstractFileNode extends A /** * Event signals to indicate the background tasks have completed processing. - * Currently, we have two property tasks in the background: + * Currently, we have one property task in the background: * - * 1) Retreiving the translation of the file name 2) Getting the SCO column - * properties from the databases + * 1) Retreiving the translation of the file name */ enum NodeSpecificEvents { TRANSLATION_AVAILABLE, - SCO_AVAILABLE; } private final PropertyChangeListener pcl = (PropertyChangeEvent evt) -> { @@ -223,12 +220,6 @@ public abstract class AbstractAbstractFileNode extends A */ } else if (eventType.equals(NodeSpecificEvents.TRANSLATION_AVAILABLE.toString())) { updateSheet(new NodeProperty<>(TRANSLATION.toString(),TRANSLATION.toString(),NO_DESCR,evt.getNewValue())); - } else if (eventType.equals(NodeSpecificEvents.SCO_AVAILABLE.toString())) { - SCOResults res = (SCOResults) evt.getNewValue(); - updateSheet(new NodeProperty<>(SCORE.toString(),SCORE.toString(),res.getScoreDescription(),res.getScore()), - new NodeProperty<>(COMMENT.toString(),COMMENT.toString(),NO_DESCR,res.getComment()), - new NodeProperty<>(OCCURRENCES.toString(),OCCURRENCES.toString(),res.getCountDescription(),res.getCount()) - ); } }; /** @@ -296,7 +287,7 @@ public abstract class AbstractAbstractFileNode extends A * blocking the UI. Keep all weak references so * this task doesn't block the ability of this node to be GC'd. */ - SCOAndTranslationPool.submit(new SCOAndTranslationTask(new WeakReference<>(this), weakPcl)); + translationPool.submit(new TranslationTask(new WeakReference<>(this), weakPcl)); return sheet; } @@ -373,17 +364,25 @@ public abstract class AbstractAbstractFileNode extends A List> properties = new ArrayList<>(); properties.add(new NodeProperty<>(NAME.toString(), NAME.toString(), NO_DESCR, getContentDisplayName(content))); /* - * Initialize dummy place holder properties for Translation, - * Score, Comment, and Occurrences). At the bottom, we kick off a + * Initialize an empty place holder value. At the bottom, we kick off a * background task that promises to update these values. */ + if (UserPreferences.displayTranslatedFileNames()) { properties.add(new NodeProperty<>(TRANSLATION.toString(), TRANSLATION.toString(), NO_DESCR, "")); } - properties.add(new NodeProperty<>(SCORE.toString(), SCORE.toString(), NO_DESCR, "")); - properties.add(new NodeProperty<>(COMMENT.toString(), COMMENT.toString(), NO_DESCR, "")); + + //SCO column prereq info.. + List tags = getContentTagsFromDatabase(); + CorrelationAttributeInstance attribute = getCorrelationAttributeInstance(); + + Pair scoreAndDescription = getScorePropertyAndDescription(tags); + properties.add(new NodeProperty<>(SCORE.toString(), SCORE.toString(), scoreAndDescription.getRight(), scoreAndDescription.getLeft())); + DataResultViewerTable.HasCommentStatus comment = getCommentProperty(tags, attribute); + properties.add(new NodeProperty<>(COMMENT.toString(), COMMENT.toString(), NO_DESCR, comment)); if (!UserPreferences.hideCentralRepoCommentsAndOccurrences()) { - properties.add(new NodeProperty<>(OCCURRENCES.toString(), OCCURRENCES.toString(), NO_DESCR, "")); + Pair countAndDescription = getCountPropertyAndDescription(attribute); + properties.add(new NodeProperty<>(OCCURRENCES.toString(), OCCURRENCES.toString(), countAndDescription.getRight(), countAndDescription.getLeft())); } properties.add(new NodeProperty<>(LOCATION.toString(), LOCATION.toString(), NO_DESCR, getContentPath(content))); properties.add(new NodeProperty<>(MOD_TIME.toString(), MOD_TIME.toString(), NO_DESCR, ContentUtils.getStringTime(content.getMtime(), content))); diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/SCOAndTranslationTask.java b/Core/src/org/sleuthkit/autopsy/datamodel/SCOAndTranslationTask.java deleted file mode 100755 index 2dc3f4b9bb..0000000000 --- a/Core/src/org/sleuthkit/autopsy/datamodel/SCOAndTranslationTask.java +++ /dev/null @@ -1,141 +0,0 @@ -/* - * Autopsy Forensic Browser - * - * Copyright 2018-2018 Basis Technology Corp. - * Contact: carrier sleuthkit org - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.sleuthkit.autopsy.datamodel; - -import java.beans.PropertyChangeEvent; -import java.beans.PropertyChangeListener; -import java.lang.ref.WeakReference; -import java.util.List; -import org.apache.commons.lang3.tuple.Pair; -import org.sleuthkit.autopsy.centralrepository.datamodel.CorrelationAttributeInstance; -import org.sleuthkit.autopsy.corecomponents.DataResultViewerTable; -import org.sleuthkit.autopsy.corecomponents.DataResultViewerTable.HasCommentStatus; -import org.sleuthkit.autopsy.corecomponents.DataResultViewerTable.Score; -import org.sleuthkit.autopsy.events.AutopsyEvent; -import org.sleuthkit.datamodel.ContentTag; - - -/** - * Completes the tasks needed to populate the Score, Comment, Occurrences and Translation - * columns in the background so that the UI is not blocked while waiting for responses from the database or - * translation service. Once these events are done, it fires a PropertyChangeEvent - * to let the AbstractAbstractFileNode know it's time to update. - */ -class SCOAndTranslationTask implements Runnable { - - private final WeakReference> weakNodeRef; - private final PropertyChangeListener listener; - - public SCOAndTranslationTask(WeakReference> weakContentRef, PropertyChangeListener listener) { - this.weakNodeRef = weakContentRef; - this.listener = listener; - } - - @Override - public void run() { - try { - AbstractAbstractFileNode fileNode = weakNodeRef.get(); - - //Long DB queries - List tags = fileNode.getContentTagsFromDatabase(); - CorrelationAttributeInstance attribute = fileNode.getCorrelationAttributeInstance(); - Pair scoreAndDescription = fileNode.getScorePropertyAndDescription(tags); - DataResultViewerTable.HasCommentStatus comment = fileNode.getCommentProperty(tags, attribute); - Pair countAndDescription = fileNode.getCountPropertyAndDescription(attribute); - - //Load the results from the SCO column operations into a wrapper object to be passed - //back to the listener so that the node can internally update it's propertySheet. - SCOResults results = new SCOResults( - scoreAndDescription.getLeft(), - scoreAndDescription.getRight(), - comment, - countAndDescription.getLeft(), - countAndDescription.getRight() - ); - - listener.propertyChange(new PropertyChangeEvent( - AutopsyEvent.SourceType.LOCAL.toString(), - AbstractAbstractFileNode.NodeSpecificEvents.SCO_AVAILABLE.toString(), - null, - results)); - - //Once we've got the SCO columns, then lets fire the translation result. - //Updating of this column is significantly lower priority than - //getting results to the SCO columns! - String translatedFileName = fileNode.getTranslatedFileName(); - if(!translatedFileName.isEmpty()) { - //Only fire if the result is meaningful. - listener.propertyChange(new PropertyChangeEvent( - AutopsyEvent.SourceType.LOCAL.toString(), - AbstractAbstractFileNode.NodeSpecificEvents.TRANSLATION_AVAILABLE.toString(), - null, translatedFileName)); - } - } catch (NullPointerException ex) { - //If we are here, that means our weakPcl or content pointer has gone stale (aka - //has been garbage collected). There's no recovery. Netbeans has - //GC'd the node because its not useful to the user anymore. No need - //to log. Fail out fast to keep the thread pool rollin! - } - } - - /** - * Wrapper around data obtained from doing the Score, Comment, and Occurrences - * tasks. This object will be accessed by the AAFN to update it's state. - */ - final class SCOResults { - - private final Score score; - private final String scoreDescription; - - private final HasCommentStatus comment; - - private final Long count; - private final String countDescription; - - public SCOResults(Score score, String scoreDescription, - HasCommentStatus comment, Long count, - String countDescription) { - this.score = score; - this.scoreDescription = scoreDescription; - this.comment = comment; - this.count = count; - this.countDescription = countDescription; - } - - public Score getScore() { - return score; - } - - public String getScoreDescription() { - return scoreDescription; - } - - public HasCommentStatus getComment() { - return comment; - } - - public Long getCount() { - return count; - } - - public String getCountDescription() { - return countDescription; - } - } -} diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/TranslationTask.java b/Core/src/org/sleuthkit/autopsy/datamodel/TranslationTask.java new file mode 100755 index 0000000000..7589fbbf1e --- /dev/null +++ b/Core/src/org/sleuthkit/autopsy/datamodel/TranslationTask.java @@ -0,0 +1,59 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2018-2018 Basis Technology Corp. + * Contact: carrier sleuthkit org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.datamodel; + +import java.beans.PropertyChangeEvent; +import java.beans.PropertyChangeListener; +import java.lang.ref.WeakReference; +import org.sleuthkit.autopsy.events.AutopsyEvent; + +/** + * Completes the tasks needed to populate the Translation columns in the + * background so that the UI is not blocked while waiting for responses from the + * translation service. Once the event is done, it fires a PropertyChangeEvent + * to let the AbstractAbstractFileNode know it's time to update. + */ +class TranslationTask implements Runnable { + + private final WeakReference> weakNodeRef; + private final PropertyChangeListener listener; + + public TranslationTask(WeakReference> weakContentRef, PropertyChangeListener listener) { + this.weakNodeRef = weakContentRef; + this.listener = listener; + } + + @Override + public void run() { + AbstractAbstractFileNode fileNode = weakNodeRef.get(); + //Check for stale reference + if (fileNode == null) { + return; + } + + String translatedFileName = fileNode.getTranslatedFileName(); + if (!translatedFileName.isEmpty() && listener != null) { + //Only fire if the result is meaningful and the listener is not a stale reference + listener.propertyChange(new PropertyChangeEvent( + AutopsyEvent.SourceType.LOCAL.toString(), + AbstractAbstractFileNode.NodeSpecificEvents.TRANSLATION_AVAILABLE.toString(), + null, translatedFileName)); + } + } +} From fdd2038a3d106677a85e916fe75773836a5b46ad Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Tue, 20 Nov 2018 09:30:27 -0500 Subject: [PATCH 121/129] Fixed a comment to reflect changes --- .../sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java index 17c0b2052f..9a9c52ab39 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/AbstractAbstractFileNode.java @@ -283,8 +283,7 @@ public abstract class AbstractAbstractFileNode extends A }); /* - * Submit the database queries ASAP. We want updated SCO columns without - * blocking the UI. Keep all weak references so + * Submit the translation task ASAP. Keep all weak references so * this task doesn't block the ability of this node to be GC'd. */ translationPool.submit(new TranslationTask(new WeakReference<>(this), weakPcl)); From d36546074ce00b4901238d9201ec6d0ec008b313 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Tue, 20 Nov 2018 10:07:57 -0500 Subject: [PATCH 122/129] Fix Codacy issues in SQLiteTableReader --- .../autopsy/coreutils/SQLiteTableReader.java | 45 ++++++++++--------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReader.java b/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReader.java index 44a9696eae..3bcbfe4f3a 100755 --- a/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReader.java +++ b/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReader.java @@ -18,6 +18,7 @@ */ package org.sleuthkit.autopsy.coreutils; +import com.mchange.v2.cfg.DelayedLogItem; import java.io.File; import java.io.IOException; import java.sql.Connection; @@ -31,6 +32,7 @@ import java.util.List; import java.util.Objects; import java.util.function.BooleanSupplier; import java.util.function.Consumer; +import java.util.logging.Level; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.NoCurrentCaseException; import org.sleuthkit.autopsy.casemodule.services.FileManager; @@ -45,22 +47,19 @@ import org.sleuthkit.datamodel.TskCoreException; * the row values. Table values are processed by data type. Users configure * these actions for certain data types in the Builder. Example usage: * - * SQLiteTableReader reader = new SQLiteTableReader.Builder(file) - * .onInteger((i) - * -> { System.out.println(i); }) - * .build(); - * + * SQLiteTableReader reader = new SQLiteTableReader.Builder(file) .onInteger((i) + * -> { System.out.println(i); }) .build(); + * * reader.read(tableName); * * or * - * SQLiteTableReader reader = new SQLiteTableReader.Builder(file) - * .onInteger(new Consumer() { - * @Override public void accept(Integer i) { - * System.out.println(i); - * } - * }).build(); - * + * SQLiteTableReader reader = new SQLiteTableReader.Builder(file) .onInteger(new + * Consumer() { + * + * @Override public void accept(Integer i) { System.out.println(i); } + * }).build(); + * * reader.reader(tableName); * * Invocation of read(String tableName) reads row by row. When an Integer is @@ -74,7 +73,7 @@ public class SQLiteTableReader implements AutoCloseable { public static class Builder { private final AbstractFile file; - + private Consumer onColumnNameAction; private Consumer onStringAction; private Consumer onLongAction; @@ -82,9 +81,10 @@ public class SQLiteTableReader implements AutoCloseable { private Consumer onFloatAction; private Consumer onBlobAction; private Consumer forAllAction; - + static Consumer doNothing() { - return NOOP -> {}; + return NOOP -> { + }; } /** @@ -209,8 +209,9 @@ public class SQLiteTableReader implements AutoCloseable { private final AbstractFile file; private final Builder builder; - - private final String SELECT_ALL_QUERY = "SELECT * FROM \"%s\""; + + private static final String SELECT_ALL_QUERY = "SELECT * FROM \"%s\""; + private static final Logger logger = Logger.getLogger(SQLiteTableReader.class.getName()); private Connection conn; private PreparedStatement statement; @@ -226,7 +227,7 @@ public class SQLiteTableReader implements AutoCloseable { private String prevTableName; /** - * Holds reference to the builder instance so that we can use its actions + * Holds reference to the builder instance so that we can use its actions * during iteration. */ private SQLiteTableReader(Builder builder) { @@ -324,7 +325,7 @@ public class SQLiteTableReader implements AutoCloseable { * */ public void read(String tableName, int limit, int offset) throws SQLiteTableReaderException { - readHelper(String.format(SELECT_ALL_QUERY, tableName)+ " LIMIT " + limit + readHelper(String.format(SELECT_ALL_QUERY, tableName) + " LIMIT " + limit + " OFFSET " + offset, () -> false); } @@ -341,7 +342,7 @@ public class SQLiteTableReader implements AutoCloseable { public void read(String tableName, BooleanSupplier condition) throws SQLiteTableReaderException { if (Objects.isNull(prevTableName) || !prevTableName.equals(tableName)) { prevTableName = tableName; - closeTableResources(); + closeTableResources(); } readHelper(String.format(SELECT_ALL_QUERY, tableName), condition); } @@ -513,7 +514,7 @@ public class SQLiteTableReader implements AutoCloseable { } liveResultSet = false; } catch (SQLException ex) { - //Do nothing, can't close.. tried our best. + logger.log(Level.SEVERE, "Failed to close table resources", ex); } } @@ -553,7 +554,7 @@ public class SQLiteTableReader implements AutoCloseable { try { close(); } catch (SQLiteTableReaderException ex) { - //Do nothing, we tried out best to close the connection. + logger.log(Level.SEVERE, "Failed to close reader in finalizer", ex); } super.finalize(); } From f9a0844cdb3348955a8c53af017e6a8283f94ec1 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Tue, 20 Nov 2018 10:09:24 -0500 Subject: [PATCH 123/129] Remove unused import in SQLiteTableReader --- Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReader.java | 1 - 1 file changed, 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReader.java b/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReader.java index 3bcbfe4f3a..a6766a20f5 100755 --- a/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReader.java +++ b/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReader.java @@ -18,7 +18,6 @@ */ package org.sleuthkit.autopsy.coreutils; -import com.mchange.v2.cfg.DelayedLogItem; import java.io.File; import java.io.IOException; import java.sql.Connection; From 58024fa29af4045426833cdd8f57df3b4442a2c7 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Tue, 20 Nov 2018 10:12:30 -0500 Subject: [PATCH 124/129] Reformat class docs in SQLiteTableReader --- .../sleuthkit/autopsy/coreutils/SQLiteTableReader.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReader.java b/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReader.java index a6766a20f5..b9d3f7d837 100755 --- a/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReader.java +++ b/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReader.java @@ -46,8 +46,10 @@ import org.sleuthkit.datamodel.TskCoreException; * the row values. Table values are processed by data type. Users configure * these actions for certain data types in the Builder. Example usage: * - * SQLiteTableReader reader = new SQLiteTableReader.Builder(file) .onInteger((i) - * -> { System.out.println(i); }) .build(); + * SQLiteTableReader reader = new SQLiteTableReader.Builder(file) + * .onInteger((i) + * -> { System.out.println(i); }) + * .build(); * * reader.read(tableName); * @@ -55,8 +57,7 @@ import org.sleuthkit.datamodel.TskCoreException; * * SQLiteTableReader reader = new SQLiteTableReader.Builder(file) .onInteger(new * Consumer() { - * - * @Override public void accept(Integer i) { System.out.println(i); } + * @Override public void accept(Integer i) { System.out.println(i); } * }).build(); * * reader.reader(tableName); From e691425a5d0783d021109ceeccb954a10b378a4c Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Tue, 20 Nov 2018 10:14:01 -0500 Subject: [PATCH 125/129] Reformat class docs in SQLiteTableReader --- .../org/sleuthkit/autopsy/coreutils/SQLiteTableReader.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReader.java b/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReader.java index b9d3f7d837..286d6c672a 100755 --- a/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReader.java +++ b/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReader.java @@ -57,7 +57,9 @@ import org.sleuthkit.datamodel.TskCoreException; * * SQLiteTableReader reader = new SQLiteTableReader.Builder(file) .onInteger(new * Consumer() { - * @Override public void accept(Integer i) { System.out.println(i); } + * @Override public void accept(Integer i) { + * System.out.println(i); + * } * }).build(); * * reader.reader(tableName); From a3633370118920cdf7d92c86dfa569f3398ddeaf Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Tue, 20 Nov 2018 10:50:47 -0500 Subject: [PATCH 126/129] Added logging statement and changed variable names --- .../keywordsearch/SqliteTextExtractor.java | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SqliteTextExtractor.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SqliteTextExtractor.java index cf252111d9..f7fff3c134 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SqliteTextExtractor.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SqliteTextExtractor.java @@ -127,7 +127,7 @@ class SqliteTextExtractor extends ContentTextExtractor { this.file = file; reader = new SQLiteTableReader.Builder(file) .onColumnNames(getColumnNameStrategy()) - .forAll(getForAllStrategy()).build(); + .forAll(getForAllTableValuesStrategy()).build(); } /** @@ -141,28 +141,28 @@ class SqliteTextExtractor extends ContentTextExtractor { * * @return Our consumer class defined to do the steps above. */ - private Consumer getForAllStrategy() { + private Consumer getForAllTableValuesStrategy() { return new Consumer() { - private int rowIndex = 0; + private int columnIndex = 0; @Override public void accept(Object value) { - rowIndex++; + columnIndex++; //Ignore blobs String objectStr = (value instanceof byte[]) ? "" : Objects.toString(value, ""); - if (rowIndex > 1 && rowIndex < totalColumns) { + if (columnIndex > 1 && columnIndex < totalColumns) { objectStr += " "; } - if (rowIndex == 1) { + if (columnIndex == 1) { objectStr = "\t" + objectStr + " "; } - if (rowIndex == totalColumns) { + if (columnIndex == totalColumns) { objectStr += "\n"; } fillBuffer(objectStr); - rowIndex = rowIndex % totalColumns; + columnIndex = columnIndex % totalColumns; } }; } @@ -285,8 +285,7 @@ class SqliteTextExtractor extends ContentTextExtractor { try { reader.close(); } catch (SQLiteTableReaderException ex) { - //Done reading, but couldn't close the resources. Nothing we can - //do as a recovery. + logger.log(Level.WARNING, "Could not close SQliteTableReader.", ex.getMessage()); } } From ccc03329877d0879610c48ae8729c7d762354876 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Tue, 20 Nov 2018 12:27:22 -0500 Subject: [PATCH 127/129] Removed old functions that caused build error --- .../datamodel/SqliteEamDbSettings.java | 77 ------------------- 1 file changed, 77 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDbSettings.java b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDbSettings.java index 2dcef5930d..8b034149fc 100644 --- a/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDbSettings.java +++ b/Core/src/org/sleuthkit/autopsy/centralrepository/datamodel/SqliteEamDbSettings.java @@ -417,83 +417,6 @@ public final class SqliteEamDbSettings { return true; } - /** - * Get the template String for creating a new _instances table in a Sqlite - * central repository. %s will exist in the template where the name of the - * new table will be addedd. - * - * @return a String which is a template for cretating a new _instances table - */ - static String getCreateArtifactInstancesTableTemplate() { - // Each "%s" will be replaced with the relevant TYPE_instances table name. - StringBuilder createArtifactInstancesTableTemplate = new StringBuilder(); - createArtifactInstancesTableTemplate.append("CREATE TABLE IF NOT EXISTS %s ("); - createArtifactInstancesTableTemplate.append("id integer primary key autoincrement NOT NULL,"); - createArtifactInstancesTableTemplate.append("case_id integer NOT NULL,"); - createArtifactInstancesTableTemplate.append("data_source_id integer NOT NULL,"); - createArtifactInstancesTableTemplate.append("value text NOT NULL,"); - createArtifactInstancesTableTemplate.append("file_path text NOT NULL,"); - createArtifactInstancesTableTemplate.append("known_status integer NOT NULL,"); - createArtifactInstancesTableTemplate.append("comment text,"); - createArtifactInstancesTableTemplate.append("CONSTRAINT %s_multi_unique UNIQUE(data_source_id, value, file_path) ON CONFLICT IGNORE,"); - createArtifactInstancesTableTemplate.append("foreign key (case_id) references cases(id) ON UPDATE SET NULL ON DELETE SET NULL,"); - createArtifactInstancesTableTemplate.append("foreign key (data_source_id) references data_sources(id) ON UPDATE SET NULL ON DELETE SET NULL"); - createArtifactInstancesTableTemplate.append(")"); - return createArtifactInstancesTableTemplate.toString(); - } - - /** - * Get the template for creating an index on the case_id column of an - * instance table. %s will exist in the template where the name of the new - * table will be addedd. - * - * @return a String which is a template for adding an index to the case_id - * column of a _instances table - */ - static String getAddCaseIdIndexTemplate() { - // Each "%s" will be replaced with the relevant TYPE_instances table name. - return "CREATE INDEX IF NOT EXISTS %s_case_id ON %s (case_id)"; - } - - /** - * Get the template for creating an index on the data_source_id column of an - * instance table. %s will exist in the template where the name of the new - * table will be addedd. - * - * @return a String which is a template for adding an index to the - * data_source_id column of a _instances table - */ - static String getAddDataSourceIdIndexTemplate() { - // Each "%s" will be replaced with the relevant TYPE_instances table name. - return "CREATE INDEX IF NOT EXISTS %s_data_source_id ON %s (data_source_id)"; - } - - /** - * Get the template for creating an index on the value column of an instance - * table. %s will exist in the template where the name of the new table will - * be addedd. - * - * @return a String which is a template for adding an index to the value - * column of a _instances table - */ - static String getAddValueIndexTemplate() { - // Each "%s" will be replaced with the relevant TYPE_instances table name. - return "CREATE INDEX IF NOT EXISTS %s_value ON %s (value)"; - } - - /** - * Get the template for creating an index on the known_status column of an - * instance table. %s will exist in the template where the name of the new - * table will be addedd. - * - * @return a String which is a template for adding an index to the - * known_status column of a _instances table - */ - static String getAddKnownStatusIndexTemplate() { - // Each "%s" will be replaced with the relevant TYPE_instances table name. - return "CREATE INDEX IF NOT EXISTS %s_value_known_status ON %s (value, known_status)"; - } - /** * Get the template String for creating a new _instances table in a Sqlite * central repository. %s will exist in the template where the name of the From d3cc7db9951b32df7d673272bce2371d06ee02e0 Mon Sep 17 00:00:00 2001 From: "U-BASIS\\dsmyda" Date: Tue, 20 Nov 2018 13:05:35 -0500 Subject: [PATCH 128/129] Changed close() to only close connection and not redundantly close table resources --- Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReader.java | 1 - 1 file changed, 1 deletion(-) diff --git a/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReader.java b/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReader.java index 286d6c672a..fdfcffd589 100755 --- a/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReader.java +++ b/Core/src/org/sleuthkit/autopsy/coreutils/SQLiteTableReader.java @@ -528,7 +528,6 @@ public class SQLiteTableReader implements AutoCloseable { @Override public void close() throws SQLiteTableReaderException { try { - closeTableResources(); if (Objects.nonNull(conn)) { conn.close(); } From 3acf779e95b4f30d86c8b222b23621e7d41ab309 Mon Sep 17 00:00:00 2001 From: Raman Date: Wed, 21 Nov 2018 09:22:51 -0500 Subject: [PATCH 129/129] 1136: Intermittent error: "Error marking group as seen" --- .../imagegallery/ImageGalleryController.java | 40 +++---------------- .../imagegallery/ImageGalleryModule.java | 9 +---- .../datamodel/grouping/GroupManager.java | 2 +- 3 files changed, 7 insertions(+), 44 deletions(-) diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java index abe89abf40..862713a807 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryController.java @@ -599,10 +599,11 @@ public final class ImageGalleryController { } } + /** - * Abstract base class for tasks associated with a file in the database + * task that updates one file in database with results from ingest */ - static abstract class FileTask extends BackgroundTask { + static class UpdateFileTask extends BackgroundTask { private final AbstractFile file; private final DrawableDB taskDB; @@ -614,22 +615,12 @@ public final class ImageGalleryController { public AbstractFile getFile() { return file; } - - FileTask(AbstractFile f, DrawableDB taskDB) { + + UpdateFileTask(AbstractFile f, DrawableDB taskDB) { super(); this.file = f; this.taskDB = taskDB; } - } - - /** - * task that updates one file in database with results from ingest - */ - static class UpdateFileTask extends FileTask { - - UpdateFileTask(AbstractFile f, DrawableDB taskDB) { - super(f, taskDB); - } /** * Update a file in the database @@ -645,27 +636,6 @@ public final class ImageGalleryController { } } - /** - * task that updates one file in database with results from ingest - */ - static class RemoveFileTask extends FileTask { - - RemoveFileTask(AbstractFile f, DrawableDB taskDB) { - super(f, taskDB); - } - - /** - * Update a file in the database - */ - @Override - public void run() { - try { - getTaskDB().removeFile(getFile().getId()); - } catch (TskCoreException | SQLException ex) { - Logger.getLogger(RemoveFileTask.class.getName()).log(Level.SEVERE, "Error in remove file task", ex); //NON-NLS - } - } - } /** * Base abstract class for various methods of copying image files data, for diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryModule.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryModule.java index 530e3eb7d6..5341e0be27 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryModule.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/ImageGalleryModule.java @@ -189,14 +189,7 @@ public class ImageGalleryModule { if (isDrawableAndNotKnown(file)) { con.queueDBTask(new ImageGalleryController.UpdateFileTask(file, controller.getDatabase())); } - // Remove it from the DB if it is no longer relevant, but had the correct extension - else if (FileTypeUtils.getAllSupportedExtensions().contains(file.getNameExtension())) { - /* Doing this check results in fewer tasks queued - * up, and faster completion of db update. This file - * would have gotten scooped up in initial grab, but - * actually we don't need it */ - con.queueDBTask(new ImageGalleryController.RemoveFileTask(file, controller.getDatabase())); - } + } catch (FileTypeDetector.FileTypeDetectorInitException ex) { logger.log(Level.SEVERE, "Unable to determine if file is drawable and not known. Not making any changes to DB", ex); //NON-NLS MessageNotifyUtil.Notify.error("Image Gallery Error", diff --git a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupManager.java b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupManager.java index 99069b7790..0b52e0ff0e 100644 --- a/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupManager.java +++ b/ImageGallery/src/org/sleuthkit/autopsy/imagegallery/datamodel/grouping/GroupManager.java @@ -279,7 +279,7 @@ public class GroupManager { updateUnSeenGroups(group); } } catch (TskCoreException ex) { - logger.log(Level.SEVERE, "Error marking group as seen", ex); //NON-NLS + logger.log(Level.SEVERE, String.format("Error setting seen status for group: %s", group.getGroupKey().getValue().toString()), ex); //NON-NLS } }); }