Merge remote-tracking branch 'upstream/develop' into 7453-populate-osaccount-realm-name

This commit is contained in:
Mark McKinnon 2021-04-08 14:20:26 -04:00
commit 426f3ffe2a
16 changed files with 261 additions and 112 deletions

View File

@ -87,10 +87,10 @@ import org.sleuthkit.autopsy.casemodule.events.HostsChangedEvent;
import org.sleuthkit.autopsy.casemodule.events.HostsRemovedEvent;
import org.sleuthkit.autopsy.casemodule.events.OsAccountAddedEvent;
import org.sleuthkit.autopsy.casemodule.events.OsAccountChangedEvent;
import org.sleuthkit.autopsy.casemodule.events.OsAccountRemovedEvent;
import org.sleuthkit.autopsy.casemodule.events.OsAccountDeletedEvent;
import org.sleuthkit.autopsy.casemodule.events.PersonsAddedEvent;
import org.sleuthkit.autopsy.casemodule.events.PersonsChangedEvent;
import org.sleuthkit.autopsy.casemodule.events.PersonsRemovedEvent;
import org.sleuthkit.autopsy.casemodule.events.PersonsDeletedEvent;
import org.sleuthkit.autopsy.casemodule.events.ReportAddedEvent;
import org.sleuthkit.autopsy.casemodule.multiusercases.CaseNodeData.CaseNodeDataException;
import org.sleuthkit.autopsy.casemodule.multiusercases.CoordinationServiceUtils;
@ -140,24 +140,16 @@ import org.sleuthkit.datamodel.ContentTag;
import org.sleuthkit.datamodel.DataSource;
import org.sleuthkit.datamodel.FileSystem;
import org.sleuthkit.datamodel.Host;
import org.sleuthkit.datamodel.HostManager.HostsCreationEvent;
import org.sleuthkit.datamodel.HostManager.HostsUpdateEvent;
import org.sleuthkit.datamodel.HostManager.HostsDeletionEvent;
import org.sleuthkit.datamodel.Image;
import org.sleuthkit.datamodel.OsAccount;
import org.sleuthkit.datamodel.OsAccountManager.OsAccountsCreationEvent;
import org.sleuthkit.datamodel.OsAccountManager.OsAccountsDeleteEvent;
import org.sleuthkit.datamodel.OsAccountManager.OsAccountsUpdateEvent;
import org.sleuthkit.datamodel.Person;
import org.sleuthkit.datamodel.PersonManager.PersonsCreationEvent;
import org.sleuthkit.datamodel.PersonManager.PersonsUpdateEvent;
import org.sleuthkit.datamodel.PersonManager.PersonsDeletionEvent;
import org.sleuthkit.datamodel.Report;
import org.sleuthkit.datamodel.SleuthkitCase;
import org.sleuthkit.datamodel.TimelineManager;
import org.sleuthkit.datamodel.SleuthkitCaseAdminUtil;
import org.sleuthkit.datamodel.TskCoreException;
import org.sleuthkit.datamodel.TskDataException;
import org.sleuthkit.datamodel.TskEvent;
import org.sleuthkit.datamodel.TskUnsupportedSchemaVersionException;
/**
@ -504,36 +496,36 @@ public class Case {
event.getArtifacts(artifactType)));
}
}
@Subscribe
public void publishOsAccountAddedEvent(OsAccountsCreationEvent event) {
for (OsAccount account : event.getOsAcounts()) {
@Subscribe
public void publishOsAccountAddedEvent(TskEvent.OsAccountsAddedTskEvent event) {
for(OsAccount account: event.getOsAcounts()) {
eventPublisher.publish(new OsAccountAddedEvent(account));
}
}
@Subscribe
public void publishOsAccountChangedEvent(OsAccountsUpdateEvent event) {
for (OsAccount account : event.getOsAcounts()) {
@Subscribe
public void publishOsAccountChangedEvent(TskEvent.OsAccountsChangedTskEvent event) {
for(OsAccount account: event.getOsAcounts()) {
eventPublisher.publish(new OsAccountChangedEvent(account));
}
}
@Subscribe
public void publishOsAccountDeletedEvent(OsAccountsDeleteEvent event) {
for (Long accountId : event.getOsAcountObjectIds()) {
eventPublisher.publish(new OsAccountRemovedEvent(accountId));
@Subscribe
public void publishOsAccountDeletedEvent(TskEvent.OsAccountsDeletedTskEvent event) {
for(Long accountId: event.getOsAcountObjectIds()) {
eventPublisher.publish(new OsAccountDeletedEvent(accountId));
}
}
/**
* Publishes an autopsy event from the sleuthkit HostCreationEvent
* Publishes an autopsy event from the sleuthkit HostAddedEvent
* indicating that hosts have been created.
*
* @param event The sleuthkit event for the creation of hosts.
*/
@Subscribe
public void publishHostsAddedEvent(HostsCreationEvent event) {
@Subscribe
public void publishHostsAddedEvent(TskEvent.HostsAddedTskEvent event) {
eventPublisher.publish(new HostsAddedEvent(
event == null ? Collections.emptyList() : event.getHosts()));
}
@ -543,9 +535,9 @@ public class Case {
* indicating that hosts have been updated.
*
* @param event The sleuthkit event for the updating of hosts.
*/
@Subscribe
public void publishHostsChangedEvent(HostsUpdateEvent event) {
*/
@Subscribe
public void publishHostsChangedEvent(TskEvent.HostsChangedTskEvent event) {
eventPublisher.publish(new HostsChangedEvent(
event == null ? Collections.emptyList() : event.getHosts()));
}
@ -555,33 +547,33 @@ public class Case {
* indicating that hosts have been deleted.
*
* @param event The sleuthkit event for the deleting of hosts.
*/
@Subscribe
public void publishHostsDeletedEvent(HostsDeletionEvent event) {
*/
@Subscribe
public void publishHostsDeletedEvent(TskEvent.HostsDeletedTskEvent event) {
eventPublisher.publish(new HostsRemovedEvent(
event == null ? Collections.emptyList() : event.getHosts()));
}
/**
* Publishes an autopsy event from the sleuthkit PersonCreationEvent
* Publishes an autopsy event from the sleuthkit PersonAddedEvent
* indicating that persons have been created.
*
* @param event The sleuthkit event for the creation of persons.
*/
@Subscribe
public void publishPersonsAddedEvent(PersonsCreationEvent event) {
@Subscribe
public void publishPersonsAddedEvent(TskEvent.PersonsAddedTskEvent event) {
eventPublisher.publish(new PersonsAddedEvent(
event == null ? Collections.emptyList() : event.getPersons()));
}
/**
* Publishes an autopsy event from the sleuthkit PersonUpdateEvent
* Publishes an autopsy event from the sleuthkit PersonChangedEvent
* indicating that persons have been updated.
*
* @param event The sleuthkit event for the updating of persons.
*/
@Subscribe
public void publishPersonsChangedEvent(PersonsUpdateEvent event) {
*/
@Subscribe
public void publishPersonsChangedEvent(TskEvent.PersonsChangedTskEvent event) {
eventPublisher.publish(new PersonsChangedEvent(
event == null ? Collections.emptyList() : event.getPersons()));
}
@ -591,10 +583,10 @@ public class Case {
* indicating that persons have been deleted.
*
* @param event The sleuthkit event for the deleting of persons.
*/
@Subscribe
public void publishPersonsDeletedEvent(PersonsDeletionEvent event) {
eventPublisher.publish(new PersonsRemovedEvent(
*/
@Subscribe
public void publishPersonsDeletedEvent(TskEvent.PersonsDeletedTskEvent event) {
eventPublisher.publish(new PersonsDeletedEvent(
event == null ? Collections.emptyList() : event.getPersons()));
}
}
@ -1796,7 +1788,7 @@ public class Case {
}
public void notifyOsAccountRemoved(Long osAccountObjectId) {
eventPublisher.publish(new OsAccountRemovedEvent(osAccountObjectId));
eventPublisher.publish(new OsAccountDeletedEvent(osAccountObjectId));
}
/**
@ -1850,7 +1842,7 @@ public class Case {
* @param person The person that has been deleted.
*/
public void notifyPersonDeleted(Person person) {
eventPublisher.publish(new PersonsRemovedEvent(Collections.singletonList(person)));
eventPublisher.publish(new PersonsDeletedEvent(Collections.singletonList(person)));
}
/**

View File

@ -33,6 +33,8 @@ import org.sleuthkit.datamodel.TskCoreException;
*/
public class HostsEvent extends TskDataModelChangeEvent<Host> {
private static final long serialVersionUID = 1L;
/**
* Retrieves a list of ids from a list of hosts.
*

View File

@ -27,11 +27,11 @@ import org.sleuthkit.autopsy.events.AutopsyEvent;
* oldValue will contain the objectId of the account that was removed. newValue
* will be null.
*/
public final class OsAccountRemovedEvent extends AutopsyEvent {
public final class OsAccountDeletedEvent extends AutopsyEvent {
private static final long serialVersionUID = 1L;
public OsAccountRemovedEvent(Long osAccountObjectId) {
public OsAccountDeletedEvent(Long osAccountObjectId) {
super(Case.Events.OS_ACCOUNT_REMOVED.toString(), osAccountObjectId, null);
}
}

View File

@ -25,7 +25,7 @@ import org.sleuthkit.datamodel.Person;
/**
* Event fired when persons are removed.
*/
public class PersonsRemovedEvent extends PersonsEvent {
public class PersonsDeletedEvent extends PersonsEvent {
private static final long serialVersionUID = 1L;
@ -33,7 +33,7 @@ public class PersonsRemovedEvent extends PersonsEvent {
* Main constructor.
* @param dataModelObjects The list of persons that have been deleted.
*/
public PersonsRemovedEvent(List<Person> dataModelObjects) {
public PersonsDeletedEvent(List<Person> dataModelObjects) {
super(Case.Events.PERSONS_DELETED.name(), dataModelObjects);
}
}

View File

@ -21,6 +21,7 @@ package org.sleuthkit.autopsy.centralrepository.datamodel;
import java.sql.SQLException;
import java.util.Collection;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import org.sleuthkit.datamodel.TskData;
import org.sleuthkit.autopsy.casemodule.Case;
@ -860,10 +861,10 @@ public interface CentralRepository {
* Get account type by type name.
*
* @param accountTypeName account type name to look for
* @return CR account type
* @return CR account type (if found)
* @throws CentralRepoException
*/
CentralRepoAccountType getAccountTypeByName(String accountTypeName) throws CentralRepoException;
Optional<CentralRepoAccountType> getAccountTypeByName(String accountTypeName) throws CentralRepoException;
/**
* Gets all account types.

View File

@ -22,6 +22,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.logging.Level;
import org.openide.util.NbBundle.Messages;
@ -308,8 +309,12 @@ public class CorrelationAttributeUtil {
if (Account.Type.DEVICE.getTypeName().equalsIgnoreCase(accountTypeStr) == false && predefinedAccountType != null) {
// Get the corresponding CentralRepoAccountType from the database.
CentralRepoAccountType crAccountType = CentralRepository.getInstance().getAccountTypeByName(accountTypeStr);
Optional<CentralRepoAccountType> optCrAccountType = CentralRepository.getInstance().getAccountTypeByName(accountTypeStr);
if (!optCrAccountType.isPresent()) {
return;
}
CentralRepoAccountType crAccountType = optCrAccountType.get();
int corrTypeId = crAccountType.getCorrelationTypeId();
CorrelationAttributeInstance.Type corrType = CentralRepository.getInstance().getCorrelationTypeById(corrTypeId);

View File

@ -26,8 +26,10 @@ import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import org.apache.commons.lang3.StringUtils;
import org.sleuthkit.datamodel.Account;
import org.sleuthkit.datamodel.TskCoreException;
/**
* This class represents an association between a Persona and an Account.
@ -206,10 +208,15 @@ public class PersonaAccount {
);
// create account
CentralRepoAccount.CentralRepoAccountType crAccountType = getCRInstance().getAccountTypeByName(rs.getString("type_name"));
String accountTypeName = rs.getString("type_name");
Optional<CentralRepoAccount.CentralRepoAccountType> optCrAccountType = getCRInstance().getAccountTypeByName(accountTypeName);
if (! optCrAccountType.isPresent()) {
// The CR account can not be null, so throw an exception
throw new CentralRepoException("Account type with name '" + accountTypeName + "' not found in Central Repository");
}
CentralRepoAccount account = new CentralRepoAccount(
rs.getInt("account_id"),
crAccountType,
optCrAccountType.get(),
rs.getString("account_unique_identifier"));
// create persona account
@ -389,10 +396,15 @@ public class PersonaAccount {
while (rs.next()) {
// create account
CentralRepoAccount.CentralRepoAccountType crAccountType = getCRInstance().getAccountTypeByName(rs.getString("type_name"));
String accountTypeName = rs.getString("type_name");
Optional<CentralRepoAccount.CentralRepoAccountType> optCrAccountType = getCRInstance().getAccountTypeByName(accountTypeName);
if (! optCrAccountType.isPresent()) {
// The CR account can not be null, so throw an exception
throw new CentralRepoException("Account type with name '" + accountTypeName + "' not found in Central Repository");
}
CentralRepoAccount account = new CentralRepoAccount(
rs.getInt("account_id"),
crAccountType,
optCrAccountType.get(),
rs.getString("account_unique_identifier"));
accountsList.add(account);

View File

@ -37,6 +37,7 @@ import java.time.LocalDate;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
@ -78,7 +79,7 @@ abstract class RdbmsCentralRepo implements CentralRepository {
private static final int CASE_CACHE_TIMEOUT = 5;
private static final int DATA_SOURCE_CACHE_TIMEOUT = 5;
private static final int ACCOUNTS_CACHE_TIMEOUT = 5;
private static final Cache<String, CentralRepoAccountType> accountTypesCache = CacheBuilder.newBuilder().build();
private static final Cache<String, Optional<CentralRepoAccountType>> accountTypesCache = CacheBuilder.newBuilder().build();
private static final Cache<Pair<CentralRepoAccountType, String>, CentralRepoAccount> accountsCache = CacheBuilder.newBuilder()
.expireAfterWrite(ACCOUNTS_CACHE_TIMEOUT, TimeUnit.MINUTES).
build();
@ -1115,7 +1116,7 @@ abstract class RdbmsCentralRepo implements CentralRepository {
}
@Override
public CentralRepoAccountType getAccountTypeByName(String accountTypeName) throws CentralRepoException {
public Optional<CentralRepoAccountType> getAccountTypeByName(String accountTypeName) throws CentralRepoException {
try {
return accountTypesCache.get(accountTypeName, () -> getCRAccountTypeFromDb(accountTypeName));
} catch (CacheLoader.InvalidCacheLoadException | ExecutionException ex) {
@ -1155,7 +1156,7 @@ abstract class RdbmsCentralRepo implements CentralRepository {
*
* @throws CentralRepoException
*/
private CentralRepoAccountType getCRAccountTypeFromDb(String accountTypeName) throws CentralRepoException {
private Optional<CentralRepoAccountType> getCRAccountTypeFromDb(String accountTypeName) throws CentralRepoException {
String sql = "SELECT * FROM account_types WHERE type_name = ?";
try (Connection conn = connect();
@ -1166,10 +1167,11 @@ abstract class RdbmsCentralRepo implements CentralRepository {
if (resultSet.next()) {
Account.Type acctType = new Account.Type(accountTypeName, resultSet.getString("display_name"));
CentralRepoAccountType crAccountType = new CentralRepoAccountType(resultSet.getInt("id"), acctType, resultSet.getInt("correlation_type_id"));
accountTypesCache.put(accountTypeName, crAccountType);
return crAccountType;
accountTypesCache.put(accountTypeName, Optional.of(crAccountType));
return Optional.of(crAccountType);
} else {
throw new CentralRepoException("Failed to find entry for account type = " + accountTypeName);
accountTypesCache.put(accountTypeName, Optional.empty());
return Optional.empty();
}
}
} catch (SQLException ex) {

View File

@ -21,6 +21,7 @@ package org.sleuthkit.autopsy.communications.relationships;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.logging.Level;
import org.openide.nodes.AbstractNode;
@ -75,9 +76,9 @@ final class CorrelationCaseChildNodeFactory extends ChildFactory<CorrelationCase
accounts.forEach((account) -> {
try {
CorrelationAttributeInstance.Type correlationType = getCorrelationType(account.getAccountType());
if (correlationType != null) {
List<CorrelationAttributeInstance> correlationInstances = dbInstance.getArtifactInstancesByTypeValue(correlationType, account.getTypeSpecificID());
Optional<CorrelationAttributeInstance.Type> optCorrelationType = getCorrelationType(account.getAccountType());
if (optCorrelationType.isPresent()) {
List<CorrelationAttributeInstance> correlationInstances = dbInstance.getArtifactInstancesByTypeValue(optCorrelationType.get(), account.getTypeSpecificID());
correlationInstances.forEach((correlationInstance) -> {
CorrelationCase correlationCase = correlationInstance.getCorrelationCase();
uniqueCaseMap.put(correlationCase.getCaseUUID(), correlationCase);
@ -103,20 +104,22 @@ final class CorrelationCaseChildNodeFactory extends ChildFactory<CorrelationCase
*
* @param accountType Account type
*
* @return CorrelationAttributeInstance.Type for given account or null if
* @return CorrelationAttributeInstance.Type for given account or empty if
* there is no match
*
* @throws CentralRepoException
*/
private CorrelationAttributeInstance.Type getCorrelationType(Account.Type accountType) throws CentralRepoException {
private Optional<CorrelationAttributeInstance.Type> getCorrelationType(Account.Type accountType) throws CentralRepoException {
String accountTypeStr = accountType.getTypeName();
if (Account.Type.DEVICE.getTypeName().equalsIgnoreCase(accountTypeStr) == false) {
CentralRepoAccount.CentralRepoAccountType crAccountType = CentralRepository.getInstance().getAccountTypeByName(accountTypeStr);
int corrTypeId = crAccountType.getCorrelationTypeId();
return CentralRepository.getInstance().getCorrelationTypeById(corrTypeId);
Optional<CentralRepoAccount.CentralRepoAccountType> optCrAccountType = CentralRepository.getInstance().getAccountTypeByName(accountTypeStr);
if (optCrAccountType.isPresent()) {
int corrTypeId = optCrAccountType.get().getCorrelationTypeId();
return Optional.of(CentralRepository.getInstance().getCorrelationTypeById(corrTypeId));
}
}
return null;
return Optional.empty();
}
/**

View File

@ -21,10 +21,12 @@ package org.sleuthkit.autopsy.communications.relationships;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Optional;
import java.util.logging.Level;
import javax.swing.SwingWorker;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.centralrepository.datamodel.CentralRepoAccount;
import org.sleuthkit.autopsy.centralrepository.datamodel.CentralRepoException;
import org.sleuthkit.autopsy.centralrepository.datamodel.CentralRepository;
import org.sleuthkit.autopsy.centralrepository.datamodel.Persona;
import org.sleuthkit.autopsy.centralrepository.datamodel.PersonaAccount;
@ -76,12 +78,15 @@ class SummaryPanelWorker extends SwingWorker<SummaryPanelWorker.SummaryWorkerRes
personaList.add(pAccount.getPersona());
}
try {
crAccount = CentralRepository.getInstance().getAccount(CentralRepository.getInstance().getAccountTypeByName(account.getAccountType().getTypeName()), account.getTypeSpecificID());
} catch (InvalidAccountIDException unused) {
// This was probably caused to a phone number not making
// threw the normalization.
logger.log(Level.WARNING, String.format("Exception thrown from CR getAccount for account %s (%d)", account.getTypeSpecificID(), account.getAccountID()));
Optional<CentralRepoAccount.CentralRepoAccountType> optCrAccountType = CentralRepository.getInstance().getAccountTypeByName(account.getAccountType().getTypeName());
if (optCrAccountType.isPresent()) {
try {
crAccount = CentralRepository.getInstance().getAccount(optCrAccountType.get(), account.getTypeSpecificID());
} catch (InvalidAccountIDException unused) {
// This was probably caused to a phone number not making
// threw the normalization.
logger.log(Level.WARNING, String.format("Exception thrown from CR getAccount for account %s (%d)", account.getTypeSpecificID(), account.getAccountID()));
}
}
}

View File

@ -32,6 +32,7 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.CancellationException;
import java.util.concurrent.ExecutionException;
import java.util.logging.Level;
@ -630,11 +631,13 @@ public class ContactArtifactViewer extends javax.swing.JPanel implements Artifac
// make a list of all unique accounts for this contact
if (!account.getAccountType().equals(Account.Type.DEVICE)) {
CentralRepoAccount.CentralRepoAccountType crAccountType = CentralRepository.getInstance().getAccountTypeByName(account.getAccountType().getTypeName());
CentralRepoAccount crAccount = CentralRepository.getInstance().getAccount(crAccountType, account.getTypeSpecificID());
Optional<CentralRepoAccount.CentralRepoAccountType> optCrAccountType = CentralRepository.getInstance().getAccountTypeByName(account.getAccountType().getTypeName());
if (optCrAccountType.isPresent()) {
CentralRepoAccount crAccount = CentralRepository.getInstance().getAccount(optCrAccountType.get(), account.getTypeSpecificID());
if (crAccount != null && uniqueAccountsList.contains(crAccount) == false) {
uniqueAccountsList.add(crAccount);
if (crAccount != null && uniqueAccountsList.contains(crAccount) == false) {
uniqueAccountsList.add(crAccount);
}
}
}

View File

@ -26,6 +26,7 @@ import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import java.io.File;
import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.sql.Connection;
import java.sql.DriverManager;
@ -44,7 +45,7 @@ import java.util.stream.IntStream;
import java.util.stream.Stream;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.openide.modules.InstalledFileLocator;
import org.sleuthkit.autopsy.coreutils.PlatformUtil;
import org.sleuthkit.autopsy.url.analytics.DomainCategory;
/**
@ -136,13 +137,19 @@ class WebCategoriesDataModel implements AutoCloseable {
* @return The path or null if the path cannot be reconciled.
*/
private static File getDefaultPath() {
File dir = InstalledFileLocator.getDefault().locate(ROOT_FOLDER, WebCategoriesDataModel.class.getPackage().getName(), false);
if (dir == null || !dir.exists()) {
logger.log(Level.WARNING, String.format("Unable to find file %s with InstalledFileLocator", ROOT_FOLDER));
String configDir = PlatformUtil.getUserConfigDirectory();
if (configDir == null || !new File(configDir).exists()) {
logger.log(Level.WARNING, "Unable to find UserConfigDirectory");
return null;
}
return Paths.get(dir.getAbsolutePath(), FILE_REL_PATH).toFile();
Path subDirPath = Paths.get(configDir, ROOT_FOLDER);
File subDir = subDirPath.toFile();
if (!subDir.exists() && !subDir.mkdirs()) {
logger.log(Level.WARNING, "There was an issue creating custom domain config at: {0}", subDirPath.toString());
}
return Paths.get(configDir, ROOT_FOLDER, FILE_REL_PATH).toFile();
}
/**
@ -530,7 +537,7 @@ class WebCategoriesDataModel implements AutoCloseable {
public synchronized void close() throws SQLException {
if (dbConn != null) {
dbConn.close();
dbConn = null;
dbConn = null;
}
}
}

View File

@ -15,6 +15,7 @@ DataSourceUsage_DJU_Drone_DAT=DJI Internal SD Card
DataSourceUsage_FlashDrive=Flash Drive
DataSourceUsageAnalyzer.customVolume.label=OS Drive ({0})
DataSourceUsageAnalyzer.parentModuleName=Recent Activity
DefaultPriorityDomainCategorizer_searchEngineCategory=Search Engine
DomainCategoryRunner_moduleName_text=DomainCategoryRunner
DomainCategoryRunner_parentModuleName=Recent Activity
DomainCategoryRunner_Progress_Message_Domain_Types=Finding Domain Types

View File

@ -0,0 +1,104 @@
/*
* Autopsy Forensic Browser
*
* Copyright 2021 Basis Technology Corp.
* Contact: carrier <at> sleuthkit <dot> org
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sleuthkit.autopsy.recentactivity;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.commons.lang.StringUtils;
import org.openide.util.NbBundle.Messages;
import org.sleuthkit.autopsy.url.analytics.DomainCategorizer;
import org.sleuthkit.autopsy.url.analytics.DomainCategorizerException;
import org.sleuthkit.autopsy.url.analytics.DomainCategory;
/**
* The autopsy provided domain category provider that overrides all domain
* category providers except the custom web domain categorizations.
*/
@Messages({
"DefaultPriorityDomainCategorizer_searchEngineCategory=Search Engine"
})
public class DefaultPriorityDomainCategorizer implements DomainCategorizer {
// taken from https://www.google.com/supported_domains
private static final List<String> GOOGLE_DOMAINS = Arrays.asList("google.com", "google.ad", "google.ae", "google.com.af", "google.com.ag", "google.com.ai", "google.al", "google.am", "google.co.ao", "google.com.ar", "google.as", "google.at", "google.com.au", "google.az", "google.ba", "google.com.bd", "google.be", "google.bf", "google.bg", "google.com.bh", "google.bi", "google.bj", "google.com.bn", "google.com.bo", "google.com.br", "google.bs", "google.bt", "google.co.bw", "google.by", "google.com.bz", "google.ca", "google.cd", "google.cf", "google.cg", "google.ch", "google.ci", "google.co.ck", "google.cl", "google.cm", "google.cn", "google.com.co", "google.co.cr", "google.com.cu", "google.cv", "google.com.cy", "google.cz", "google.de", "google.dj", "google.dk", "google.dm", "google.com.do", "google.dz", "google.com.ec", "google.ee", "google.com.eg", "google.es", "google.com.et", "google.fi", "google.com.fj", "google.fm", "google.fr", "google.ga", "google.ge", "google.gg", "google.com.gh", "google.com.gi", "google.gl", "google.gm", "google.gr", "google.com.gt", "google.gy", "google.com.hk", "google.hn", "google.hr", "google.ht", "google.hu", "google.co.id", "google.ie", "google.co.il", "google.im", "google.co.in", "google.iq", "google.is", "google.it", "google.je", "google.com.jm", "google.jo", "google.co.jp", "google.co.ke", "google.com.kh", "google.ki", "google.kg", "google.co.kr", "google.com.kw", "google.kz", "google.la", "google.com.lb", "google.li", "google.lk", "google.co.ls", "google.lt", "google.lu", "google.lv", "google.com.ly", "google.co.ma", "google.md", "google.me", "google.mg", "google.mk", "google.ml", "google.com.mm", "google.mn", "google.ms", "google.com.mt", "google.mu", "google.mv", "google.mw", "google.com.mx", "google.com.my", "google.co.mz", "google.com.na", "google.com.ng", "google.com.ni", "google.ne", "google.nl", "google.no", "google.com.np", "google.nr", "google.nu", "google.co.nz", "google.com.om", "google.com.pa", "google.com.pe", "google.com.pg", "google.com.ph", "google.com.pk", "google.pl", "google.pn", "google.com.pr", "google.ps", "google.pt", "google.com.py", "google.com.qa", "google.ro", "google.ru", "google.rw", "google.com.sa", "google.com.sb", "google.sc", "google.se", "google.com.sg", "google.sh", "google.si", "google.sk", "google.com.sl", "google.sn", "google.so", "google.sm", "google.sr", "google.st", "google.com.sv", "google.td", "google.tg", "google.co.th", "google.com.tj", "google.tl", "google.tm", "google.tn", "google.to", "google.com.tr", "google.tt", "google.com.tw", "google.co.tz", "google.com.ua", "google.co.ug", "google.co.uk", "google.com.uy", "google.co.uz", "google.com.vc", "google.co.ve", "google.vg", "google.co.vi", "google.com.vn", "google.vu", "google.ws", "google.rs", "google.co.za", "google.co.zm", "google.co.zw", "google.cat");
// taken from https://www.yahoo.com/everything/world
private static final List<String> YAHOO_DOMAINS = Arrays.asList("espanol.yahoo.com", "au.yahoo.com", "be.yahoo.com", "fr-be.yahoo.com", "br.yahoo.com", "ca.yahoo.com", "espanol.yahoo.com", "espanol.yahoo.com", "de.yahoo.com", "es.yahoo.com", "espanol.yahoo.com", "fr.yahoo.com", "in.yahoo.com", "id.yahoo.com", "ie.yahoo.com", "it.yahoo.com", "en-maktoob.yahoo.com", "malaysia.yahoo.com", "espanol.yahoo.com", "nz.yahoo.com", "espanol.yahoo.com", "ph.yahoo.com", "qc.yahoo.com", "ro.yahoo.com", "sg.yahoo.com", "za.yahoo.com", "se.yahoo.com", "uk.yahoo.com", "yahoo.com", "espanol.yahoo.com", "vn.yahoo.com", "gr.yahoo.com", "maktoob.yahoo.com", "yahoo.com", "hk.yahoo.com", "tw.yahoo.com", "yahoo.co.jp");
private static final List<String> OTHER_SEARCH_ENGINES = Arrays.asList(
"bing.com",
"baidu.com",
"sogou.com",
"soso.com",
"duckduckgo.com",
"swisscows.com",
"gibiru.com",
"cutestat.com",
"youdao.com",
"biglobe.ne.jp",
"givewater.com",
"ekoru.org",
"ecosia.org",
// according to https://en.wikipedia.org/wiki/Yandex
"yandex.ru",
"yandex.com"
);
private static final String WWW_PREFIX = "www";
private static final Map<String, String> DOMAIN_LOOKUP
= Stream.of(GOOGLE_DOMAINS, YAHOO_DOMAINS, OTHER_SEARCH_ENGINES)
.flatMap((lst) -> lst.stream())
.collect(Collectors.toMap((k) -> k, (k) -> Bundle.DefaultPriorityDomainCategorizer_searchEngineCategory(), (v1, v2) -> v1));
@Override
public void initialize() throws DomainCategorizerException {
}
@Override
public DomainCategory getCategory(String domain, String host) throws DomainCategorizerException {
String hostToUse = StringUtils.isBlank(host) ? domain : host;
if (StringUtils.isBlank(hostToUse)) {
return null;
}
List<String> domainWords = Stream.of(hostToUse.toLowerCase().split("\\."))
.filter(StringUtils::isNotBlank)
.map(String::trim)
.collect(Collectors.toList());
String sanitizedDomain = domainWords.stream()
// skip first word segment if 'www'
.skip(domainWords.size() > 0 && WWW_PREFIX.equals(domainWords.get(0)) ? 1 : 0)
.collect(Collectors.joining("."));
String category = DOMAIN_LOOKUP.get(sanitizedDomain);
return category == null ? null : new DomainCategory(sanitizedDomain, category);
}
@Override
public void close() throws IOException {
}
}

View File

@ -20,6 +20,7 @@ package org.sleuthkit.autopsy.recentactivity;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
@ -453,33 +454,45 @@ class DomainCategoryRunner extends Extract {
@Override
void configExtractor() throws IngestModule.IngestModuleException {
// lookup all providers, filter null providers, and sort providers
Collection<? extends DomainCategorizer> lookupList = Lookup.getDefault().lookupAll(DomainCategorizer.class);
if (lookupList == null) {
lookupList = Collections.emptyList();
}
List<DomainCategorizer> foundProviders = lookupList.stream()
.filter(provider -> provider != null)
.sorted((a, b) -> {
boolean aIsCustom = a.getClass().getName().contains(CUSTOM_CATEGORIZER_PATH);
boolean bIsCustom = b.getClass().getName().contains(CUSTOM_CATEGORIZER_PATH);
if (aIsCustom != bIsCustom) {
// push custom categorizer to top
return -Boolean.compare(aIsCustom, bIsCustom);
}
return a.getClass().getName().compareToIgnoreCase(b.getClass().getName());
Collection<? extends DomainCategorizer> lookupCollection = Lookup.getDefault().lookupAll(DomainCategorizer.class);
Collection<? extends DomainCategorizer> lookupList = (lookupCollection == null) ?
Collections.emptyList() :
lookupCollection;
// this will be the class instance of the foundProviders
List<DomainCategorizer> foundProviders = new ArrayList<>();
// find the custom domain categories provider if present and add it first to the list
lookupList.stream()
.filter(categorizer -> categorizer.getClass().getName().contains(CUSTOM_CATEGORIZER_PATH))
.findFirst()
.ifPresent((provider) -> foundProviders.add(provider));
// add the default priority categorizer
foundProviders.add(new DefaultPriorityDomainCategorizer());
// add all others except for the custom web domain categorizer, the default priority
// categorizer and the default categorizer
lookupList.stream()
.filter(categorizer -> categorizer != null)
.filter(categorizer -> {
String className = categorizer.getClass().getName();
return !className.contains(CUSTOM_CATEGORIZER_PATH) &&
!className.equals(DefaultPriorityDomainCategorizer.class.getName()) &&
!className.equals(DefaultDomainCategorizer.class.getName());
})
.collect(Collectors.toList());
// add the default categorizer last as a last resort
.sorted((a, b) -> a.getClass().getName().compareToIgnoreCase(b.getClass().getName()))
.forEach(foundProviders::add);
// add the default categorizer last
foundProviders.add(new DefaultDomainCategorizer());
for (DomainCategorizer provider : foundProviders) {
try {
provider.initialize();
} catch (DomainCategorizerException ex) {
throw new IngestModule.IngestModuleException("There was an error instantiating the provider: " + provider.getClass().getSimpleName(), ex);
throw new IngestModule.IngestModuleException("There was an error instantiating the provider: " +
provider.getClass().getSimpleName(), ex);
}
}

View File

@ -733,7 +733,6 @@ class Firefox extends Extract {
}
j++;
dbFile.delete();
break;
}
if(!context.dataSourceIngestIsCancelled()) {