19 package org.sleuthkit.autopsy.centralrepository.eventlisteners;
21 import com.google.common.util.concurrent.ThreadFactoryBuilder;
22 import java.beans.PropertyChangeEvent;
23 import java.beans.PropertyChangeListener;
24 import static java.lang.Boolean.FALSE;
25 import java.util.ArrayList;
26 import java.util.Arrays;
27 import java.util.Collection;
28 import java.util.EnumSet;
29 import java.util.Iterator;
30 import java.util.LinkedHashSet;
31 import java.util.List;
33 import java.util.concurrent.ExecutorService;
34 import java.util.concurrent.Executors;
35 import java.util.logging.Level;
36 import java.util.stream.Collectors;
37 import org.apache.commons.lang3.StringUtils;
38 import org.openide.util.NbBundle;
55 import static org.
sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME;
56 import static org.
sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_CORRELATION_TYPE;
57 import static org.
sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_CORRELATION_VALUE;
58 import static org.
sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_OTHER_CASES;
73 @NbBundle.Messages({
"IngestEventsListener.ingestmodule.name=Central Repository"})
79 private static final String MODULE_NAME = Bundle.IngestEventsListener_ingestmodule_name();
85 private static final String INGEST_EVENT_THREAD_NAME =
"Ingest-Event-Listener-%d";
89 final Collection<String> recentlyAddedCeArtifacts =
new LinkedHashSet<>();
91 static final int MAX_NUM_PREVIOUS_CASES_FOR_LIKELY_NOTABLE_SCORE = 10;
92 static final int MAX_NUM_PREVIOUS_CASES_FOR_PREV_SEEN_ARTIFACT_CREATION = 20;
95 jobProcessingExecutor = Executors.newSingleThreadExecutor(
new ThreadFactoryBuilder().setNameFormat(INGEST_EVENT_THREAD_NAME).build());
123 correlationModuleInstanceCount++;
131 if (getCeModuleInstanceCount() > 0) {
132 correlationModuleInstanceCount--;
140 synchronized static void resetCeModuleInstanceCount() {
141 correlationModuleInstanceCount = 0;
151 return correlationModuleInstanceCount;
160 return flagNotableItems;
169 return flagSeenDevices;
178 return createCrProperties;
187 flagNotableItems = value;
196 flagSeenDevices = value;
205 flagUniqueArtifacts = value;
214 return flagUniqueArtifacts;
223 createCrProperties = value;
236 @NbBundle.Messages({
"IngestEventsListener.prevTaggedSet.text=Previously Tagged As Notable (Central Repository)",
237 "IngestEventsListener.prevCaseComment.text=Previous Case: "})
240 String prevCases = caseDisplayNames.stream().distinct().collect(Collectors.joining(
","));
241 String justification =
"Previously marked as notable in cases " + prevCases;
242 Collection<BlackboardAttribute> attributesForNewArtifact = Arrays.asList(
new BlackboardAttribute(
243 TSK_SET_NAME, MODULE_NAME,
244 Bundle.IngestEventsListener_prevTaggedSet_text()),
245 new BlackboardAttribute(
246 TSK_CORRELATION_TYPE, MODULE_NAME,
247 aType.getDisplayName()),
248 new BlackboardAttribute(
249 TSK_CORRELATION_VALUE, MODULE_NAME,
251 new BlackboardAttribute(
252 TSK_OTHER_CASES, MODULE_NAME,
254 makeAndPostArtifact(BlackboardArtifact.Type.TSK_PREVIOUSLY_NOTABLE, originalArtifact, attributesForNewArtifact, Bundle.IngestEventsListener_prevTaggedSet_text(),
255 Score.SCORE_NOTABLE, justification);
268 @NbBundle.Messages({
"IngestEventsListener.prevExists.text=Previously Seen Devices (Central Repository)",
271 "IngestEventsListener.prevCount.text=Number of previous {0}: {1}"})
277 int numCases = caseDisplayNames.size();
278 if (numCases <= MAX_NUM_PREVIOUS_CASES_FOR_LIKELY_NOTABLE_SCORE) {
279 score = Score.SCORE_LIKELY_NOTABLE;
280 }
else if (numCases > MAX_NUM_PREVIOUS_CASES_FOR_LIKELY_NOTABLE_SCORE && numCases <= MAX_NUM_PREVIOUS_CASES_FOR_PREV_SEEN_ARTIFACT_CREATION) {
281 score = Score.SCORE_NONE;
287 String prevCases = caseDisplayNames.stream().distinct().collect(Collectors.joining(
","));
288 String justification =
"Previously seen in cases " + prevCases;
289 Collection<BlackboardAttribute> attributesForNewArtifact = Arrays.asList(
new BlackboardAttribute(
290 TSK_SET_NAME, MODULE_NAME,
291 Bundle.IngestEventsListener_prevExists_text()),
292 new BlackboardAttribute(
293 TSK_CORRELATION_TYPE, MODULE_NAME,
294 aType.getDisplayName()),
295 new BlackboardAttribute(
296 TSK_CORRELATION_VALUE, MODULE_NAME,
298 new BlackboardAttribute(
299 TSK_OTHER_CASES, MODULE_NAME,
301 makeAndPostArtifact(BlackboardArtifact.Type.TSK_PREVIOUSLY_SEEN, originalArtifact, attributesForNewArtifact, Bundle.IngestEventsListener_prevExists_text(),
302 score, justification);
315 Collection<BlackboardAttribute> attributesForNewArtifact = Arrays.asList(
316 new BlackboardAttribute(
317 TSK_CORRELATION_TYPE, MODULE_NAME,
318 aType.getDisplayName()),
319 new BlackboardAttribute(
320 TSK_CORRELATION_VALUE, MODULE_NAME,
322 makeAndPostArtifact(BlackboardArtifact.Type.TSK_PREVIOUSLY_UNSEEN, originalArtifact, attributesForNewArtifact,
"",
323 Score.SCORE_LIKELY_NOTABLE,
"This application has not been previously seen before");
336 private static void makeAndPostArtifact(BlackboardArtifact.Type newArtifactType, BlackboardArtifact originalArtifact, Collection<BlackboardAttribute> attributesForNewArtifact, String configuration,
337 Score score, String justification) {
339 SleuthkitCase tskCase = originalArtifact.getSleuthkitCase();
340 Blackboard blackboard = tskCase.getBlackboard();
342 BlackboardArtifact.ARTIFACT_TYPE type = BlackboardArtifact.ARTIFACT_TYPE.fromID(newArtifactType.getTypeID());
343 if (!blackboard.artifactExists(originalArtifact, type, attributesForNewArtifact)) {
344 BlackboardArtifact newArtifact = originalArtifact.newAnalysisResult(
345 newArtifactType, score,
346 null, configuration, justification, attributesForNewArtifact)
347 .getAnalysisResult();
351 blackboard.postArtifact(newArtifact, MODULE_NAME);
352 }
catch (Blackboard.BlackboardException ex) {
353 LOGGER.log(Level.SEVERE,
"Unable to index blackboard artifact " + newArtifact.getArtifactID(), ex);
356 }
catch (TskCoreException ex) {
357 LOGGER.log(Level.SEVERE,
"Failed to create BlackboardArtifact.", ex);
358 }
catch (IllegalStateException ex) {
359 LOGGER.log(Level.SEVERE,
"Failed to create BlackboardAttribute.", ex);
375 LOGGER.log(Level.SEVERE,
"Failed to connect to Central Repository database.", ex);
386 jobProcessingExecutor.submit(
new DataAddedTask(dbManager, evt, flagNotable, flagPrevious, createAttributes, flagUnique));
404 LOGGER.log(Level.SEVERE,
"Failed to connect to Central Repository database.", ex);
409 case DATA_SOURCE_ANALYSIS_COMPLETED: {
423 private final PropertyChangeEvent
event;
433 if (getCeModuleInstanceCount() == 0) {
434 recentlyAddedCeArtifacts.clear();
446 String dataSourceName =
"";
447 long dataSourceObjectId = -1;
454 if (!(dataSource instanceof Image)) {
458 dataSourceName = dataSource.getName();
459 dataSourceObjectId = dataSource.getId();
464 if (null == correlationCase) {
465 correlationCase = dbManager.
newCase(openCase);
469 if (correlationDataSource == null) {
474 if (dataSource instanceof Image) {
475 Image image = (Image) dataSource;
477 String imageMd5Hash = image.
getMd5();
478 if (imageMd5Hash == null) {
481 String crMd5Hash = correlationDataSource.
getMd5();
482 if (StringUtils.equals(imageMd5Hash, crMd5Hash) ==
false) {
483 correlationDataSource.
setMd5(imageMd5Hash);
486 String imageSha1Hash = image.getSha1();
487 if (imageSha1Hash == null) {
490 String crSha1Hash = correlationDataSource.
getSha1();
491 if (StringUtils.equals(imageSha1Hash, crSha1Hash) ==
false) {
492 correlationDataSource.
setSha1(imageSha1Hash);
495 String imageSha256Hash = image.getSha256();
496 if (imageSha256Hash == null) {
497 imageSha256Hash =
"";
499 String crSha256Hash = correlationDataSource.
getSha256();
500 if (StringUtils.equals(imageSha256Hash, crSha256Hash) ==
false) {
501 correlationDataSource.
setSha256(imageSha256Hash);
506 LOGGER.log(Level.SEVERE, String.format(
507 "Unable to fetch data from the Central Repository for data source '%s' (obj_id=%d)",
508 dataSourceName, dataSourceObjectId), ex);
510 LOGGER.log(Level.SEVERE,
"No current case opened.", ex);
511 }
catch (TskCoreException ex) {
512 LOGGER.log(Level.SEVERE, String.format(
513 "Unable to fetch data from the case database for data source '%s' (obj_id=%d)",
514 dataSourceName, dataSourceObjectId), ex);
522 private final PropertyChangeEvent
event;
528 private DataAddedTask(
CentralRepository db, PropertyChangeEvent evt,
boolean flagNotableItemsEnabled,
boolean flagPreviousItemsEnabled,
boolean createCorrelationAttributes,
boolean flagUnique) {
531 this.flagNotableItemsEnabled = flagNotableItemsEnabled;
532 this.flagPreviousItemsEnabled = flagPreviousItemsEnabled;
533 this.createCorrelationAttributes = createCorrelationAttributes;
534 this.flagUniqueItemsEnabled = flagUnique;
543 Collection<BlackboardArtifact> bbArtifacts = mde.
getArtifacts();
544 if (null == bbArtifacts) {
547 List<CorrelationAttributeInstance> eamArtifacts =
new ArrayList<>();
549 for (BlackboardArtifact bbArtifact : bbArtifacts) {
551 List<CorrelationAttributeInstance> convertedArtifacts =
new ArrayList<>();
552 if (bbArtifact instanceof DataArtifact){
558 if (recentlyAddedCeArtifacts.add(eamArtifact.toString())) {
561 List<CorrelationAttributeInstance> previousOccurrences =
new ArrayList<>();
563 if (flagNotableItemsEnabled || flagPreviousItemsEnabled || flagUniqueItemsEnabled) {
568 for (Iterator<CorrelationAttributeInstance> iterator = previousOccurrences.iterator(); iterator.hasNext();) {
576 LOGGER.log(Level.INFO, String.format(
"Unable to flag previously seen device: %s.", eamArtifact.toString()), ex);
584 if (flagNotableItemsEnabled) {
585 List<String> caseDisplayNames = getCaseDisplayNamesForNotable(previousOccurrences);
586 if (!caseDisplayNames.isEmpty()) {
587 makeAndPostPreviousNotableArtifact(bbArtifact,
588 caseDisplayNames, eamArtifact.getCorrelationType(), eamArtifact.getCorrelationValue());
596 if (flagPreviousItemsEnabled && !previousOccurrences.isEmpty()
605 List<String> caseDisplayNames = getCaseDisplayNames(previousOccurrences);
606 makeAndPostPreviousSeenArtifact(bbArtifact, caseDisplayNames, eamArtifact.getCorrelationType(), eamArtifact.getCorrelationValue());
610 if (flagUniqueItemsEnabled
614 if (previousOccurrences.isEmpty()) {
615 makeAndPostPreviouslyUnseenArtifact(bbArtifact, eamArtifact.getCorrelationType(), eamArtifact.getCorrelationValue());
618 if (createCorrelationAttributes) {
619 eamArtifacts.add(eamArtifact);
623 LOGGER.log(Level.SEVERE,
"Error counting notable artifacts.", ex);
627 if (FALSE == eamArtifacts.isEmpty()) {
632 LOGGER.log(Level.SEVERE,
"Error adding artifact to database.", ex);
647 List<String> caseNames =
new ArrayList<>();
649 caseNames.add(occurrence.getCorrelationCase().getDisplayName());
662 List<String> caseNames =
new ArrayList<>();
664 if (occurrence.getKnownStatus() == TskData.FileKnown.BAD) {
665 caseNames.add(occurrence.getCorrelationCase().getDisplayName());
Collection< BlackboardArtifact > getArtifacts()
static final int EMAIL_TYPE_ID
void removeIngestModuleEventListener(final PropertyChangeListener listener)
static boolean flagSeenDevices
final ExecutorService jobProcessingExecutor
static final int USBID_TYPE_ID
final boolean flagUniqueItemsEnabled
void setMd5(String md5Hash)
static void makeAndPostArtifact(BlackboardArtifact.Type newArtifactType, BlackboardArtifact originalArtifact, Collection< BlackboardAttribute > attributesForNewArtifact, String configuration, Score score, String justification)
static synchronized IngestManager getInstance()
final CentralRepository dbManager
static synchronized int getCeModuleInstanceCount()
static final int ICCID_TYPE_ID
static synchronized boolean isFlagSeenDevices()
DataAddedTask(CentralRepository db, PropertyChangeEvent evt, boolean flagNotableItemsEnabled, boolean flagPreviousItemsEnabled, boolean createCorrelationAttributes, boolean flagUnique)
static synchronized void setCreateCrProperties(boolean value)
List< CorrelationAttributeInstance > getArtifactInstancesByTypeValue(CorrelationAttributeInstance.Type aType, String value)
static CorrelationDataSource fromTSKDataSource(CorrelationCase correlationCase, Content dataSource)
boolean isIngestRunning()
CorrelationCase getCase(Case autopsyCase)
static synchronized boolean isFlagUniqueArtifacts()
DATA_SOURCE_ANALYSIS_COMPLETED
static final int IMEI_TYPE_ID
static boolean createCrProperties
static synchronized boolean isFlagNotableItems()
void removeIngestJobEventListener(final PropertyChangeListener listener)
static void makeAndPostPreviousNotableArtifact(BlackboardArtifact originalArtifact, List< String > caseDisplayNames, CorrelationAttributeInstance.Type aType, String value)
void addArtifactInstance(CorrelationAttributeInstance eamArtifact)
final boolean createCorrelationAttributes
static void shutDownTaskExecutor(ExecutorService executor)
void uninstallListeners()
final PropertyChangeEvent event
void addIngestJobEventListener(final PropertyChangeListener listener)
final boolean flagNotableItemsEnabled
static final int DOMAIN_TYPE_ID
void setSha256(String sha256Hash)
static boolean flagUniqueArtifacts
static final int PHONE_TYPE_ID
List< String > getCaseDisplayNames(List< CorrelationAttributeInstance > occurrences)
CorrelationCase newCase(CorrelationCase eamCase)
static synchronized void setFlagSeenDevices(boolean value)
AnalysisCompleteTask(CentralRepository db, PropertyChangeEvent evt)
static final int INSTALLED_PROGS_TYPE_ID
static synchronized void setFlagUniqueArtifacts(boolean value)
void propertyChange(PropertyChangeEvent evt)
void setSha1(String sha1Hash)
static void makeAndPostPreviousSeenArtifact(BlackboardArtifact originalArtifact, List< String > caseDisplayNames, CorrelationAttributeInstance.Type aType, String value)
void propertyChange(PropertyChangeEvent evt)
static int correlationModuleInstanceCount
static synchronized void setFlagNotableItems(boolean value)
static final int MAC_TYPE_ID
final CentralRepository dbManager
static final int IMSI_TYPE_ID
static void makeAndPostPreviouslyUnseenArtifact(BlackboardArtifact originalArtifact, CorrelationAttributeInstance.Type aType, String value)
void addIngestModuleEventListener(final PropertyChangeListener listener)
static List< CorrelationAttributeInstance > makeCorrAttrsToSave(DataArtifact artifact)
CorrelationDataSource getDataSource(CorrelationCase correlationCase, Long caseDbDataSourceId)
synchronized static Logger getLogger(String name)
static Case getCurrentCaseThrows()
static boolean flagNotableItems
CorrelationCase getCorrelationCase()
static synchronized boolean shouldCreateCrProperties()
List< String > getCaseDisplayNamesForNotable(List< CorrelationAttributeInstance > occurrences)
final boolean flagPreviousItemsEnabled
final PropertyChangeEvent event
static synchronized void incrementCorrelationEngineModuleCount()
static synchronized void decrementCorrelationEngineModuleCount()
static CentralRepository getInstance()
static boolean isEnabled()