Autopsy  4.15.0
Graphical digital forensics platform for The Sleuth Kit and other tools.
IngestEventsListener.java
Go to the documentation of this file.
1 /*
2  * Central Repository
3  *
4  * Copyright 2017-2020 Basis Technology Corp.
5  * Contact: carrier <at> sleuthkit <dot> org
6  *
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  * http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  */
19 package org.sleuthkit.autopsy.centralrepository.eventlisteners;
20 
21 import com.google.common.util.concurrent.ThreadFactoryBuilder;
22 import java.beans.PropertyChangeEvent;
23 import java.beans.PropertyChangeListener;
24 import static java.lang.Boolean.FALSE;
25 import java.util.ArrayList;
26 import java.util.Arrays;
27 import java.util.Collection;
28 import java.util.EnumSet;
29 import java.util.LinkedHashSet;
30 import java.util.List;
31 import java.util.Set;
32 import java.util.concurrent.ExecutorService;
33 import java.util.concurrent.Executors;
34 import java.util.logging.Level;
35 import java.util.stream.Collectors;
36 import org.apache.commons.lang3.StringUtils;
37 import org.openide.util.NbBundle;
50 import org.sleuthkit.datamodel.AbstractFile;
51 import org.sleuthkit.datamodel.Blackboard;
52 import org.sleuthkit.datamodel.BlackboardArtifact;
53 import static org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_ARTIFACT_HIT;
54 import org.sleuthkit.datamodel.BlackboardAttribute;
57 import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_ASSOCIATED_ARTIFACT;
58 import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_COMMENT;
59 import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME;
61 import org.sleuthkit.datamodel.Content;
62 import org.sleuthkit.datamodel.Image;
63 import org.sleuthkit.datamodel.SleuthkitCase;
64 import org.sleuthkit.datamodel.TskCoreException;
68 import org.sleuthkit.datamodel.Account;
69 import static org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE.TSK_CONTACT;
70 import org.sleuthkit.datamodel.CommunicationsUtils;
71 
76 @NbBundle.Messages({"IngestEventsListener.ingestmodule.name=Central Repository"})
77 public class IngestEventsListener {
78 
79  private static final Logger LOGGER = Logger.getLogger(CorrelationAttributeInstance.class.getName());
80  private static final Set<IngestManager.IngestJobEvent> INGEST_JOB_EVENTS_OF_INTEREST = EnumSet.of(IngestManager.IngestJobEvent.DATA_SOURCE_ANALYSIS_COMPLETED);
81  private static final Set<IngestManager.IngestModuleEvent> INGEST_MODULE_EVENTS_OF_INTEREST = EnumSet.of(DATA_ADDED);
82  private static final String MODULE_NAME = Bundle.IngestEventsListener_ingestmodule_name();
83  private static int correlationModuleInstanceCount;
84  private static boolean flagNotableItems;
85  private static boolean flagSeenDevices;
86  private static boolean createCrProperties;
87  private static final String INGEST_EVENT_THREAD_NAME = "Ingest-Event-Listener-%d";
88  private final ExecutorService jobProcessingExecutor;
89  private final PropertyChangeListener pcl1 = new IngestModuleEventListener();
90  private final PropertyChangeListener pcl2 = new IngestJobEventListener();
91  final Collection<String> recentlyAddedCeArtifacts = new LinkedHashSet<>();
92 
94  jobProcessingExecutor = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setNameFormat(INGEST_EVENT_THREAD_NAME).build());
95  }
96 
97  void shutdown() {
98  ThreadUtils.shutDownTaskExecutor(jobProcessingExecutor);
99  }
100 
101  /*
102  * Add all of our Ingest Event Listeners to the IngestManager Instance.
103  */
104  public void installListeners() {
105  IngestManager.getInstance().addIngestModuleEventListener(INGEST_MODULE_EVENTS_OF_INTEREST, pcl1);
106  IngestManager.getInstance().addIngestJobEventListener(INGEST_JOB_EVENTS_OF_INTEREST, pcl2);
107  }
108 
109  /*
110  * Remove all of our Ingest Event Listeners from the IngestManager Instance.
111  */
112  public void uninstallListeners() {
115  }
116 
121  public synchronized static void incrementCorrelationEngineModuleCount() {
122  correlationModuleInstanceCount++; //Should be called once in the Central Repository module's startup method.
123  }
124 
129  public synchronized static void decrementCorrelationEngineModuleCount() {
130  if (getCeModuleInstanceCount() > 0) { //prevent it ingestJobCounter from going negative
131  correlationModuleInstanceCount--; //Should be called once in the Central Repository module's shutdown method.
132  }
133  }
134 
139  synchronized static void resetCeModuleInstanceCount() {
140  correlationModuleInstanceCount = 0; //called when a case is opened in case for some reason counter was not reset
141  }
142 
149  public synchronized static int getCeModuleInstanceCount() {
150  return correlationModuleInstanceCount;
151  }
152 
158  public synchronized static boolean isFlagNotableItems() {
159  return flagNotableItems;
160  }
161 
167  public synchronized static boolean isFlagSeenDevices() {
168  return flagSeenDevices;
169  }
170 
176  public synchronized static boolean shouldCreateCrProperties() {
177  return createCrProperties;
178  }
179 
185  public synchronized static void setFlagNotableItems(boolean value) {
186  flagNotableItems = value;
187  }
188 
194  public synchronized static void setFlagSeenDevices(boolean value) {
195  flagSeenDevices = value;
196  }
197 
203  public synchronized static void setCreateCrProperties(boolean value) {
204  createCrProperties = value;
205  }
206 
212  @NbBundle.Messages({"IngestEventsListener.prevTaggedSet.text=Previously Tagged As Notable (Central Repository)",
213  "IngestEventsListener.prevCaseComment.text=Previous Case: "})
214  static private void makeAndPostPreviousNotableArtifact(BlackboardArtifact originalArtifact, List<String> caseDisplayNames) {
215 
216  Collection<BlackboardAttribute> attributesForNewArtifact = Arrays.asList(new BlackboardAttribute(
217  TSK_SET_NAME, MODULE_NAME,
218  Bundle.IngestEventsListener_prevTaggedSet_text()),
219  new BlackboardAttribute(
220  TSK_COMMENT, MODULE_NAME,
221  Bundle.IngestEventsListener_prevCaseComment_text() + caseDisplayNames.stream().distinct().collect(Collectors.joining(","))),
222  new BlackboardAttribute(
223  TSK_ASSOCIATED_ARTIFACT, MODULE_NAME,
224  originalArtifact.getArtifactID()));
225  makeAndPostInterestingArtifact(originalArtifact, attributesForNewArtifact);
226  }
227 
235  @NbBundle.Messages({"IngestEventsListener.prevExists.text=Previously Seen Devices (Central Repository)",
236  "# {0} - typeName",
237  "# {1} - count",
238  "IngestEventsListener.prevCount.text=Number of previous {0}: {1}"})
239  static private void makeAndPostPreviousSeenArtifact(BlackboardArtifact originalArtifact, List<String> caseDisplayNames) {
240  Collection<BlackboardAttribute> attributesForNewArtifact = Arrays.asList(new BlackboardAttribute(
241  TSK_SET_NAME, MODULE_NAME,
242  Bundle.IngestEventsListener_prevExists_text()),
243  new BlackboardAttribute(
244  TSK_COMMENT, MODULE_NAME,
245  Bundle.IngestEventsListener_prevCaseComment_text() + caseDisplayNames.stream().distinct().collect(Collectors.joining(","))),
246  new BlackboardAttribute(
247  TSK_ASSOCIATED_ARTIFACT, MODULE_NAME,
248  originalArtifact.getArtifactID()));
249  makeAndPostInterestingArtifact(originalArtifact, attributesForNewArtifact);
250  }
251 
257  private static void makeAndPostInterestingArtifact(BlackboardArtifact originalArtifact, Collection<BlackboardAttribute> attributesForNewArtifact) {
258  try {
259  SleuthkitCase tskCase = originalArtifact.getSleuthkitCase();
260  AbstractFile abstractFile = tskCase.getAbstractFileById(originalArtifact.getObjectID());
261  Blackboard blackboard = tskCase.getBlackboard();
262  // Create artifact if it doesn't already exist.
263  if (!blackboard.artifactExists(abstractFile, TSK_INTERESTING_ARTIFACT_HIT, attributesForNewArtifact)) {
264  BlackboardArtifact newInterestingArtifact = abstractFile.newArtifact(TSK_INTERESTING_ARTIFACT_HIT);
265  newInterestingArtifact.addAttributes(attributesForNewArtifact);
266 
267  try {
268  // index the artifact for keyword search
269  blackboard.postArtifact(newInterestingArtifact, MODULE_NAME);
270  } catch (Blackboard.BlackboardException ex) {
271  LOGGER.log(Level.SEVERE, "Unable to index blackboard artifact " + newInterestingArtifact.getArtifactID(), ex); //NON-NLS
272  }
273  }
274  } catch (TskCoreException ex) {
275  LOGGER.log(Level.SEVERE, "Failed to create BlackboardArtifact.", ex); // NON-NLS
276  } catch (IllegalStateException ex) {
277  LOGGER.log(Level.SEVERE, "Failed to create BlackboardAttribute.", ex); // NON-NLS
278  }
279  }
280 
281  private class IngestModuleEventListener implements PropertyChangeListener {
282 
283  @Override
284  public void propertyChange(PropertyChangeEvent evt) {
285  //if ingest is running we want there to check if there is a Central Repository module running
286  //sometimes artifacts are generated by DSPs or other sources while ingest is not running
287  //in these cases we still want to create correlation attributesForNewArtifact for those artifacts when appropriate
288  if (!IngestManager.getInstance().isIngestRunning() || getCeModuleInstanceCount() > 0) {
289  CentralRepository dbManager;
290  try {
291  dbManager = CentralRepository.getInstance();
292  } catch (CentralRepoException ex) {
293  LOGGER.log(Level.SEVERE, "Failed to connect to Central Repository database.", ex);
294  return;
295  }
296  switch (IngestManager.IngestModuleEvent.valueOf(evt.getPropertyName())) {
297  case DATA_ADDED: {
298  //if ingest isn't running create the interesting items otherwise use the ingest module setting to determine if we create interesting items
299  boolean flagNotable = !IngestManager.getInstance().isIngestRunning() || isFlagNotableItems();
300  boolean flagPrevious = !IngestManager.getInstance().isIngestRunning() || isFlagSeenDevices();
301  boolean createAttributes = !IngestManager.getInstance().isIngestRunning() || shouldCreateCrProperties();
302  jobProcessingExecutor.submit(new DataAddedTask(dbManager, evt, flagNotable, flagPrevious, createAttributes));
303  break;
304  }
305  default:
306  break;
307  }
308  }
309  }
310  }
311 
312  private class IngestJobEventListener implements PropertyChangeListener {
313 
314  @Override
315  public void propertyChange(PropertyChangeEvent evt) {
316  CentralRepository dbManager;
317  try {
318  dbManager = CentralRepository.getInstance();
319  } catch (CentralRepoException ex) {
320  LOGGER.log(Level.SEVERE, "Failed to connect to Central Repository database.", ex);
321  return;
322  }
323 
324  switch (IngestManager.IngestJobEvent.valueOf(evt.getPropertyName())) {
325  case DATA_SOURCE_ANALYSIS_COMPLETED: {
326  jobProcessingExecutor.submit(new AnalysisCompleteTask(dbManager, evt));
327  break;
328  }
329  default:
330  break;
331  }
332  }
333 
334  }
335 
336  private final class AnalysisCompleteTask implements Runnable {
337 
339  private final PropertyChangeEvent event;
340 
341  private AnalysisCompleteTask(CentralRepository db, PropertyChangeEvent evt) {
342  dbManager = db;
343  event = evt;
344  }
345 
346  @Override
347  public void run() {
348  // clear the tracker to reduce memory usage
349  if (getCeModuleInstanceCount() == 0) {
350  recentlyAddedCeArtifacts.clear();
351  }
352  //else another instance of the Central Repository Module is still being run.
353 
354  /*
355  * Ensure the data source in the Central Repository has hash values
356  * that match those in the case database.
357  */
358  if (!CentralRepository.isEnabled()) {
359  return;
360  }
361  Content dataSource;
362  String dataSourceName = "";
363  long dataSourceObjectId = -1;
364  try {
365  dataSource = ((DataSourceAnalysisEvent) event).getDataSource();
366  /*
367  * We only care about Images for the purpose of
368  * updating hash values.
369  */
370  if (!(dataSource instanceof Image)) {
371  return;
372  }
373 
374  dataSourceName = dataSource.getName();
375  dataSourceObjectId = dataSource.getId();
376 
377  Case openCase = Case.getCurrentCaseThrows();
378 
379  CorrelationCase correlationCase = dbManager.getCase(openCase);
380  if (null == correlationCase) {
381  correlationCase = dbManager.newCase(openCase);
382  }
383 
384  CorrelationDataSource correlationDataSource = dbManager.getDataSource(correlationCase, dataSource.getId());
385  if (correlationDataSource == null) {
386  // Add the data source.
387  CorrelationDataSource.fromTSKDataSource(correlationCase, dataSource);
388  } else {
389  // Sync the data source hash values if necessary.
390  if (dataSource instanceof Image) {
391  Image image = (Image) dataSource;
392 
393  String imageMd5Hash = image.getMd5();
394  if (imageMd5Hash == null) {
395  imageMd5Hash = "";
396  }
397  String crMd5Hash = correlationDataSource.getMd5();
398  if (StringUtils.equals(imageMd5Hash, crMd5Hash) == false) {
399  correlationDataSource.setMd5(imageMd5Hash);
400  }
401 
402  String imageSha1Hash = image.getSha1();
403  if (imageSha1Hash == null) {
404  imageSha1Hash = "";
405  }
406  String crSha1Hash = correlationDataSource.getSha1();
407  if (StringUtils.equals(imageSha1Hash, crSha1Hash) == false) {
408  correlationDataSource.setSha1(imageSha1Hash);
409  }
410 
411  String imageSha256Hash = image.getSha256();
412  if (imageSha256Hash == null) {
413  imageSha256Hash = "";
414  }
415  String crSha256Hash = correlationDataSource.getSha256();
416  if (StringUtils.equals(imageSha256Hash, crSha256Hash) == false) {
417  correlationDataSource.setSha256(imageSha256Hash);
418  }
419  }
420  }
421  } catch (CentralRepoException ex) {
422  LOGGER.log(Level.SEVERE, String.format(
423  "Unable to fetch data from the Central Repository for data source '%s' (obj_id=%d)",
424  dataSourceName, dataSourceObjectId), ex);
425  } catch (NoCurrentCaseException ex) {
426  LOGGER.log(Level.SEVERE, "No current case opened.", ex);
427  } catch (TskCoreException ex) {
428  LOGGER.log(Level.SEVERE, String.format(
429  "Unable to fetch data from the case database for data source '%s' (obj_id=%d)",
430  dataSourceName, dataSourceObjectId), ex);
431  }
432  } // DATA_SOURCE_ANALYSIS_COMPLETED
433  }
434 
435  private final class DataAddedTask implements Runnable {
436 
438  private final PropertyChangeEvent event;
439  private final boolean flagNotableItemsEnabled;
440  private final boolean flagPreviousItemsEnabled;
441  private final boolean createCorrelationAttributes;
442 
443  private DataAddedTask(CentralRepository db, PropertyChangeEvent evt, boolean flagNotableItemsEnabled, boolean flagPreviousItemsEnabled, boolean createCorrelationAttributes) {
444  this.dbManager = db;
445  this.event = evt;
446  this.flagNotableItemsEnabled = flagNotableItemsEnabled;
447  this.flagPreviousItemsEnabled = flagPreviousItemsEnabled;
448  this.createCorrelationAttributes = createCorrelationAttributes;
449  }
450 
451  @Override
452  public void run() {
453  if (!CentralRepository.isEnabled()) {
454  return;
455  }
456  final ModuleDataEvent mde = (ModuleDataEvent) event.getOldValue();
457  Collection<BlackboardArtifact> bbArtifacts = mde.getArtifacts();
458  if (null == bbArtifacts) { //the ModuleDataEvents don't always have a collection of artifacts set
459  return;
460  }
461  List<CorrelationAttributeInstance> eamArtifacts = new ArrayList<>();
462 
463  for (BlackboardArtifact bbArtifact : bbArtifacts) {
464  // makeCorrAttrToSave will filter out artifacts which should not be sources of CR data.
465  List<CorrelationAttributeInstance> convertedArtifacts = CorrelationAttributeUtil.makeCorrAttrsToSave(bbArtifact);
466  for (CorrelationAttributeInstance eamArtifact : convertedArtifacts) {
467  try {
468  // Only do something with this artifact if it's unique within the job
469  if (recentlyAddedCeArtifacts.add(eamArtifact.toString())) {
470  // Was it previously marked as bad?
471  // query db for artifact instances having this TYPE/VALUE and knownStatus = "Bad".
472  // if getKnownStatus() is "Unknown" and this artifact instance was marked bad in a previous case,
473  // create TSK_INTERESTING_ARTIFACT_HIT artifact on BB.
474  if (flagNotableItemsEnabled) {
475  List<String> caseDisplayNames;
476  try {
477  caseDisplayNames = dbManager.getListCasesHavingArtifactInstancesKnownBad(eamArtifact.getCorrelationType(), eamArtifact.getCorrelationValue());
478  if (!caseDisplayNames.isEmpty()) {
479  makeAndPostPreviousNotableArtifact(bbArtifact,
480  caseDisplayNames);
481  }
483  LOGGER.log(Level.INFO, String.format("Unable to flag notable item: %s.", eamArtifact.toString()), ex);
484  }
485  }
486  if (flagPreviousItemsEnabled
487  && (eamArtifact.getCorrelationType().getId() == CorrelationAttributeInstance.USBID_TYPE_ID
488  || eamArtifact.getCorrelationType().getId() == CorrelationAttributeInstance.ICCID_TYPE_ID
489  || eamArtifact.getCorrelationType().getId() == CorrelationAttributeInstance.IMEI_TYPE_ID
490  || eamArtifact.getCorrelationType().getId() == CorrelationAttributeInstance.IMSI_TYPE_ID
491  || eamArtifact.getCorrelationType().getId() == CorrelationAttributeInstance.MAC_TYPE_ID)) {
492  try {
493  //only alert to previous instances when they were in another case
494  List<CorrelationAttributeInstance> previousOccurences = dbManager.getArtifactInstancesByTypeValue(eamArtifact.getCorrelationType(), eamArtifact.getCorrelationValue());
495  List<String> caseDisplayNames;
496  for (CorrelationAttributeInstance instance : previousOccurences) {
497  if (!instance.getCorrelationCase().getCaseUUID().equals(eamArtifact.getCorrelationCase().getCaseUUID())) {
498  caseDisplayNames = dbManager.getListCasesHavingArtifactInstances(eamArtifact.getCorrelationType(), eamArtifact.getCorrelationValue());
499  makeAndPostPreviousSeenArtifact(bbArtifact, caseDisplayNames);
500  break;
501  }
502  }
504  LOGGER.log(Level.INFO, String.format("Unable to flag notable item: %s.", eamArtifact.toString()), ex);
505  }
506  }
507  if (createCorrelationAttributes) {
508  eamArtifacts.add(eamArtifact);
509  }
510  }
511  } catch (CentralRepoException ex) {
512  LOGGER.log(Level.SEVERE, "Error counting notable artifacts.", ex);
513  }
514  }
515  }
516  if (FALSE == eamArtifacts.isEmpty()) {
517  for (CorrelationAttributeInstance eamArtifact : eamArtifacts) {
518  try {
519  dbManager.addArtifactInstance(eamArtifact);
520  } catch (CentralRepoException ex) {
521  LOGGER.log(Level.SEVERE, "Error adding artifact to database.", ex); //NON-NLS
522  }
523  }
524  } // DATA_ADDED
525  }
526  }
527 }
Collection< BlackboardArtifact > getArtifacts()
void removeIngestModuleEventListener(final PropertyChangeListener listener)
static synchronized IngestManager getInstance()
List< String > getListCasesHavingArtifactInstances(CorrelationAttributeInstance.Type aType, String value)
static void makeAndPostPreviousSeenArtifact(BlackboardArtifact originalArtifact, List< String > caseDisplayNames)
List< CorrelationAttributeInstance > getArtifactInstancesByTypeValue(CorrelationAttributeInstance.Type aType, String value)
static CorrelationDataSource fromTSKDataSource(CorrelationCase correlationCase, Content dataSource)
static void makeAndPostInterestingArtifact(BlackboardArtifact originalArtifact, Collection< BlackboardAttribute > attributesForNewArtifact)
List< String > getListCasesHavingArtifactInstancesKnownBad(CorrelationAttributeInstance.Type aType, String value)
void removeIngestJobEventListener(final PropertyChangeListener listener)
void addArtifactInstance(CorrelationAttributeInstance eamArtifact)
static void shutDownTaskExecutor(ExecutorService executor)
void addIngestJobEventListener(final PropertyChangeListener listener)
static void makeAndPostPreviousNotableArtifact(BlackboardArtifact originalArtifact, List< String > caseDisplayNames)
static List< CorrelationAttributeInstance > makeCorrAttrsToSave(BlackboardArtifact artifact)
void addIngestModuleEventListener(final PropertyChangeListener listener)
CorrelationDataSource getDataSource(CorrelationCase correlationCase, Long caseDbDataSourceId)
synchronized static Logger getLogger(String name)
Definition: Logger.java:124
DataAddedTask(CentralRepository db, PropertyChangeEvent evt, boolean flagNotableItemsEnabled, boolean flagPreviousItemsEnabled, boolean createCorrelationAttributes)

Copyright © 2012-2020 Basis Technology. Generated on: Mon Jul 6 2020
This work is licensed under a Creative Commons Attribution-Share Alike 3.0 United States License.