Autopsy  4.14.0
Graphical digital forensics platform for The Sleuth Kit and other tools.
IngestEventsListener.java
Go to the documentation of this file.
1 /*
2  * Central Repository
3  *
4  * Copyright 2017-2020 Basis Technology Corp.
5  * Contact: carrier <at> sleuthkit <dot> org
6  *
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  * http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  */
19 package org.sleuthkit.autopsy.centralrepository.eventlisteners;
20 
21 import com.google.common.util.concurrent.ThreadFactoryBuilder;
22 import java.beans.PropertyChangeEvent;
23 import java.beans.PropertyChangeListener;
24 import static java.lang.Boolean.FALSE;
25 import java.util.ArrayList;
26 import java.util.Arrays;
27 import java.util.Collection;
28 import java.util.EnumSet;
29 import java.util.LinkedHashSet;
30 import java.util.List;
31 import java.util.Set;
32 import java.util.concurrent.ExecutorService;
33 import java.util.concurrent.Executors;
34 import java.util.logging.Level;
35 import java.util.stream.Collectors;
36 import org.apache.commons.lang3.StringUtils;
37 import org.openide.util.NbBundle;
49 import org.sleuthkit.datamodel.AbstractFile;
50 import org.sleuthkit.datamodel.Blackboard;
51 import org.sleuthkit.datamodel.BlackboardArtifact;
52 import static org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_ARTIFACT_HIT;
53 import org.sleuthkit.datamodel.BlackboardAttribute;
56 import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_ASSOCIATED_ARTIFACT;
57 import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_COMMENT;
58 import static org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME;
60 import org.sleuthkit.datamodel.Content;
61 import org.sleuthkit.datamodel.Image;
62 import org.sleuthkit.datamodel.SleuthkitCase;
63 import org.sleuthkit.datamodel.TskCoreException;
65 
70 @NbBundle.Messages({"IngestEventsListener.ingestmodule.name=Correlation Engine"})
71 public class IngestEventsListener {
72 
73  private static final Logger LOGGER = Logger.getLogger(CorrelationAttributeInstance.class.getName());
74  private static final Set<IngestManager.IngestJobEvent> INGEST_JOB_EVENTS_OF_INTEREST = EnumSet.of(IngestManager.IngestJobEvent.DATA_SOURCE_ANALYSIS_COMPLETED);
75  private static final Set<IngestManager.IngestModuleEvent> INGEST_MODULE_EVENTS_OF_INTEREST = EnumSet.of(DATA_ADDED);
76  private static final String MODULE_NAME = Bundle.IngestEventsListener_ingestmodule_name();
77  private static int correlationModuleInstanceCount;
78  private static boolean flagNotableItems;
79  private static boolean flagSeenDevices;
80  private static boolean createCrProperties;
81  private static final String INGEST_EVENT_THREAD_NAME = "Ingest-Event-Listener-%d";
82  private final ExecutorService jobProcessingExecutor;
83  private final PropertyChangeListener pcl1 = new IngestModuleEventListener();
84  private final PropertyChangeListener pcl2 = new IngestJobEventListener();
85  final Collection<String> recentlyAddedCeArtifacts = new LinkedHashSet<>();
86 
88  jobProcessingExecutor = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setNameFormat(INGEST_EVENT_THREAD_NAME).build());
89  }
90 
91  void shutdown() {
92  ThreadUtils.shutDownTaskExecutor(jobProcessingExecutor);
93  }
94 
95  /*
96  * Add all of our Ingest Event Listeners to the IngestManager Instance.
97  */
98  public void installListeners() {
99  IngestManager.getInstance().addIngestModuleEventListener(INGEST_MODULE_EVENTS_OF_INTEREST, pcl1);
100  IngestManager.getInstance().addIngestJobEventListener(INGEST_JOB_EVENTS_OF_INTEREST, pcl2);
101  }
102 
103  /*
104  * Remove all of our Ingest Event Listeners from the IngestManager Instance.
105  */
106  public void uninstallListeners() {
109  }
110 
115  public synchronized static void incrementCorrelationEngineModuleCount() {
116  correlationModuleInstanceCount++; //Should be called once in the Correlation Engine module's startup method.
117  }
118 
123  public synchronized static void decrementCorrelationEngineModuleCount() {
124  if (getCeModuleInstanceCount() > 0) { //prevent it ingestJobCounter from going negative
125  correlationModuleInstanceCount--; //Should be called once in the Correlation Engine module's shutdown method.
126  }
127  }
128 
133  synchronized static void resetCeModuleInstanceCount() {
134  correlationModuleInstanceCount = 0; //called when a case is opened in case for some reason counter was not reset
135  }
136 
143  public synchronized static int getCeModuleInstanceCount() {
144  return correlationModuleInstanceCount;
145  }
146 
152  public synchronized static boolean isFlagNotableItems() {
153  return flagNotableItems;
154  }
155 
161  public synchronized static boolean isFlagSeenDevices() {
162  return flagSeenDevices;
163  }
164 
170  public synchronized static boolean shouldCreateCrProperties() {
171  return createCrProperties;
172  }
173 
179  public synchronized static void setFlagNotableItems(boolean value) {
180  flagNotableItems = value;
181  }
182 
188  public synchronized static void setFlagSeenDevices(boolean value) {
189  flagSeenDevices = value;
190  }
191 
197  public synchronized static void setCreateCrProperties(boolean value) {
198  createCrProperties = value;
199  }
200 
206  @NbBundle.Messages({"IngestEventsListener.prevTaggedSet.text=Previously Tagged As Notable (Central Repository)",
207  "IngestEventsListener.prevCaseComment.text=Previous Case: "})
208  static private void makeAndPostPreviousNotableArtifact(BlackboardArtifact originalArtifact, List<String> caseDisplayNames) {
209 
210  Collection<BlackboardAttribute> attributesForNewArtifact = Arrays.asList(new BlackboardAttribute(
211  TSK_SET_NAME, MODULE_NAME,
212  Bundle.IngestEventsListener_prevTaggedSet_text()),
213  new BlackboardAttribute(
214  TSK_COMMENT, MODULE_NAME,
215  Bundle.IngestEventsListener_prevCaseComment_text() + caseDisplayNames.stream().distinct().collect(Collectors.joining(","))),
216  new BlackboardAttribute(
217  TSK_ASSOCIATED_ARTIFACT, MODULE_NAME,
218  originalArtifact.getArtifactID()));
219  makeAndPostInterestingArtifact(originalArtifact, attributesForNewArtifact);
220  }
221 
229  @NbBundle.Messages({"IngestEventsListener.prevExists.text=Previously Seen Devices (Central Repository)",
230  "# {0} - typeName",
231  "# {1} - count",
232  "IngestEventsListener.prevCount.text=Number of previous {0}: {1}"})
233  static private void makeAndPostPreviousSeenArtifact(BlackboardArtifact originalArtifact, List<String> caseDisplayNames) {
234  Collection<BlackboardAttribute> attributesForNewArtifact = Arrays.asList(new BlackboardAttribute(
235  TSK_SET_NAME, MODULE_NAME,
236  Bundle.IngestEventsListener_prevExists_text()),
237  new BlackboardAttribute(
238  TSK_COMMENT, MODULE_NAME,
239  Bundle.IngestEventsListener_prevCaseComment_text() + caseDisplayNames.stream().distinct().collect(Collectors.joining(","))),
240  new BlackboardAttribute(
241  TSK_ASSOCIATED_ARTIFACT, MODULE_NAME,
242  originalArtifact.getArtifactID()));
243  makeAndPostInterestingArtifact(originalArtifact, attributesForNewArtifact);
244  }
245 
251  private static void makeAndPostInterestingArtifact(BlackboardArtifact originalArtifact, Collection<BlackboardAttribute> attributesForNewArtifact) {
252  try {
253  SleuthkitCase tskCase = originalArtifact.getSleuthkitCase();
254  AbstractFile abstractFile = tskCase.getAbstractFileById(originalArtifact.getObjectID());
255  Blackboard blackboard = tskCase.getBlackboard();
256  // Create artifact if it doesn't already exist.
257  if (!blackboard.artifactExists(abstractFile, TSK_INTERESTING_ARTIFACT_HIT, attributesForNewArtifact)) {
258  BlackboardArtifact newInterestingArtifact = abstractFile.newArtifact(TSK_INTERESTING_ARTIFACT_HIT);
259  newInterestingArtifact.addAttributes(attributesForNewArtifact);
260 
261  try {
262  // index the artifact for keyword search
263  blackboard.postArtifact(newInterestingArtifact, MODULE_NAME);
264  } catch (Blackboard.BlackboardException ex) {
265  LOGGER.log(Level.SEVERE, "Unable to index blackboard artifact " + newInterestingArtifact.getArtifactID(), ex); //NON-NLS
266  }
267  }
268  } catch (TskCoreException ex) {
269  LOGGER.log(Level.SEVERE, "Failed to create BlackboardArtifact.", ex); // NON-NLS
270  } catch (IllegalStateException ex) {
271  LOGGER.log(Level.SEVERE, "Failed to create BlackboardAttribute.", ex); // NON-NLS
272  }
273  }
274 
275  private class IngestModuleEventListener implements PropertyChangeListener {
276 
277  @Override
278  public void propertyChange(PropertyChangeEvent evt) {
279  //if ingest is running we want there to check if there is a Correlation Engine module running
280  //sometimes artifacts are generated by DSPs or other sources while ingest is not running
281  //in these cases we still want to create correlation attributesForNewArtifact for those artifacts when appropriate
282  if (!IngestManager.getInstance().isIngestRunning() || getCeModuleInstanceCount() > 0) {
283  CentralRepository dbManager;
284  try {
285  dbManager = CentralRepository.getInstance();
286  } catch (CentralRepoException ex) {
287  LOGGER.log(Level.SEVERE, "Failed to connect to Central Repository database.", ex);
288  return;
289  }
290  switch (IngestManager.IngestModuleEvent.valueOf(evt.getPropertyName())) {
291  case DATA_ADDED: {
292  //if ingest isn't running create the interesting items otherwise use the ingest module setting to determine if we create interesting items
293  boolean flagNotable = !IngestManager.getInstance().isIngestRunning() || isFlagNotableItems();
294  boolean flagPrevious = !IngestManager.getInstance().isIngestRunning() || isFlagSeenDevices();
295  boolean createAttributes = !IngestManager.getInstance().isIngestRunning() || shouldCreateCrProperties();
296  jobProcessingExecutor.submit(new DataAddedTask(dbManager, evt, flagNotable, flagPrevious, createAttributes));
297  break;
298  }
299  default:
300  break;
301  }
302  }
303  }
304  }
305 
306  private class IngestJobEventListener implements PropertyChangeListener {
307 
308  @Override
309  public void propertyChange(PropertyChangeEvent evt) {
310  CentralRepository dbManager;
311  try {
312  dbManager = CentralRepository.getInstance();
313  } catch (CentralRepoException ex) {
314  LOGGER.log(Level.SEVERE, "Failed to connect to Central Repository database.", ex);
315  return;
316  }
317 
318  switch (IngestManager.IngestJobEvent.valueOf(evt.getPropertyName())) {
319  case DATA_SOURCE_ANALYSIS_COMPLETED: {
320  jobProcessingExecutor.submit(new AnalysisCompleteTask(dbManager, evt));
321  break;
322  }
323  default:
324  break;
325  }
326  }
327 
328  }
329 
330  private final class AnalysisCompleteTask implements Runnable {
331 
333  private final PropertyChangeEvent event;
334 
335  private AnalysisCompleteTask(CentralRepository db, PropertyChangeEvent evt) {
336  dbManager = db;
337  event = evt;
338  }
339 
340  @Override
341  public void run() {
342  // clear the tracker to reduce memory usage
343  if (getCeModuleInstanceCount() == 0) {
344  recentlyAddedCeArtifacts.clear();
345  }
346  //else another instance of the Correlation Engine Module is still being run.
347 
348  /*
349  * Ensure the data source in the Central Repository has hash values
350  * that match those in the case database.
351  */
352  if (!CentralRepository.isEnabled()) {
353  return;
354  }
355  Content dataSource;
356  String dataSourceName = "";
357  long dataSourceObjectId = -1;
358  try {
359  dataSource = ((DataSourceAnalysisEvent) event).getDataSource();
360  /*
361  * We only care about Images for the purpose of
362  * updating hash values.
363  */
364  if (!(dataSource instanceof Image)) {
365  return;
366  }
367 
368  dataSourceName = dataSource.getName();
369  dataSourceObjectId = dataSource.getId();
370 
371  Case openCase = Case.getCurrentCaseThrows();
372 
373  CorrelationCase correlationCase = dbManager.getCase(openCase);
374  if (null == correlationCase) {
375  correlationCase = dbManager.newCase(openCase);
376  }
377 
378  CorrelationDataSource correlationDataSource = dbManager.getDataSource(correlationCase, dataSource.getId());
379  if (correlationDataSource == null) {
380  // Add the data source.
381  CorrelationDataSource.fromTSKDataSource(correlationCase, dataSource);
382  } else {
383  // Sync the data source hash values if necessary.
384  if (dataSource instanceof Image) {
385  Image image = (Image) dataSource;
386 
387  String imageMd5Hash = image.getMd5();
388  if (imageMd5Hash == null) {
389  imageMd5Hash = "";
390  }
391  String crMd5Hash = correlationDataSource.getMd5();
392  if (StringUtils.equals(imageMd5Hash, crMd5Hash) == false) {
393  correlationDataSource.setMd5(imageMd5Hash);
394  }
395 
396  String imageSha1Hash = image.getSha1();
397  if (imageSha1Hash == null) {
398  imageSha1Hash = "";
399  }
400  String crSha1Hash = correlationDataSource.getSha1();
401  if (StringUtils.equals(imageSha1Hash, crSha1Hash) == false) {
402  correlationDataSource.setSha1(imageSha1Hash);
403  }
404 
405  String imageSha256Hash = image.getSha256();
406  if (imageSha256Hash == null) {
407  imageSha256Hash = "";
408  }
409  String crSha256Hash = correlationDataSource.getSha256();
410  if (StringUtils.equals(imageSha256Hash, crSha256Hash) == false) {
411  correlationDataSource.setSha256(imageSha256Hash);
412  }
413  }
414  }
415  } catch (CentralRepoException ex) {
416  LOGGER.log(Level.SEVERE, String.format(
417  "Unable to fetch data from the Central Repository for data source '%s' (obj_id=%d)",
418  dataSourceName, dataSourceObjectId), ex);
419  } catch (NoCurrentCaseException ex) {
420  LOGGER.log(Level.SEVERE, "No current case opened.", ex);
421  } catch (TskCoreException ex) {
422  LOGGER.log(Level.SEVERE, String.format(
423  "Unable to fetch data from the case database for data source '%s' (obj_id=%d)",
424  dataSourceName, dataSourceObjectId), ex);
425  }
426  } // DATA_SOURCE_ANALYSIS_COMPLETED
427  }
428 
429  private final class DataAddedTask implements Runnable {
430 
432  private final PropertyChangeEvent event;
433  private final boolean flagNotableItemsEnabled;
434  private final boolean flagPreviousItemsEnabled;
435  private final boolean createCorrelationAttributes;
436 
437  private DataAddedTask(CentralRepository db, PropertyChangeEvent evt, boolean flagNotableItemsEnabled, boolean flagPreviousItemsEnabled, boolean createCorrelationAttributes) {
438  this.dbManager = db;
439  this.event = evt;
440  this.flagNotableItemsEnabled = flagNotableItemsEnabled;
441  this.flagPreviousItemsEnabled = flagPreviousItemsEnabled;
442  this.createCorrelationAttributes = createCorrelationAttributes;
443  }
444 
445  @Override
446  public void run() {
447  if (!CentralRepository.isEnabled()) {
448  return;
449  }
450  final ModuleDataEvent mde = (ModuleDataEvent) event.getOldValue();
451  Collection<BlackboardArtifact> bbArtifacts = mde.getArtifacts();
452  if (null == bbArtifacts) { //the ModuleDataEvents don't always have a collection of artifacts set
453  return;
454  }
455  List<CorrelationAttributeInstance> eamArtifacts = new ArrayList<>();
456 
457  for (BlackboardArtifact bbArtifact : bbArtifacts) {
458  // makeCorrAttrToSave will filter out artifacts which should not be sources of CR data.
459  List<CorrelationAttributeInstance> convertedArtifacts = CorrelationAttributeUtil.makeCorrAttrsToSave(bbArtifact);
460  for (CorrelationAttributeInstance eamArtifact : convertedArtifacts) {
461  try {
462  // Only do something with this artifact if it's unique within the job
463  if (recentlyAddedCeArtifacts.add(eamArtifact.toString())) {
464  // Was it previously marked as bad?
465  // query db for artifact instances having this TYPE/VALUE and knownStatus = "Bad".
466  // if getKnownStatus() is "Unknown" and this artifact instance was marked bad in a previous case,
467  // create TSK_INTERESTING_ARTIFACT_HIT artifact on BB.
468  if (flagNotableItemsEnabled) {
469  List<String> caseDisplayNames;
470  try {
471  caseDisplayNames = dbManager.getListCasesHavingArtifactInstancesKnownBad(eamArtifact.getCorrelationType(), eamArtifact.getCorrelationValue());
472  if (!caseDisplayNames.isEmpty()) {
473  makeAndPostPreviousNotableArtifact(bbArtifact,
474  caseDisplayNames);
475  }
477  LOGGER.log(Level.INFO, String.format("Unable to flag notable item: %s.", eamArtifact.toString()), ex);
478  }
479  }
480  if (flagPreviousItemsEnabled
481  && (eamArtifact.getCorrelationType().getId() == CorrelationAttributeInstance.USBID_TYPE_ID
482  || eamArtifact.getCorrelationType().getId() == CorrelationAttributeInstance.ICCID_TYPE_ID
483  || eamArtifact.getCorrelationType().getId() == CorrelationAttributeInstance.IMEI_TYPE_ID
484  || eamArtifact.getCorrelationType().getId() == CorrelationAttributeInstance.IMSI_TYPE_ID
485  || eamArtifact.getCorrelationType().getId() == CorrelationAttributeInstance.MAC_TYPE_ID)) {
486  try {
487  //only alert to previous instances when they were in another case
488  List<CorrelationAttributeInstance> previousOccurences = dbManager.getArtifactInstancesByTypeValue(eamArtifact.getCorrelationType(), eamArtifact.getCorrelationValue());
489  List<String> caseDisplayNames;
490  for (CorrelationAttributeInstance instance : previousOccurences) {
491  if (!instance.getCorrelationCase().getCaseUUID().equals(eamArtifact.getCorrelationCase().getCaseUUID())) {
492  caseDisplayNames = dbManager.getListCasesHavingArtifactInstances(eamArtifact.getCorrelationType(), eamArtifact.getCorrelationValue());
493  makeAndPostPreviousSeenArtifact(bbArtifact, caseDisplayNames);
494  break;
495  }
496  }
498  LOGGER.log(Level.INFO, String.format("Unable to flag notable item: %s.", eamArtifact.toString()), ex);
499  }
500  }
501  if (createCorrelationAttributes) {
502  eamArtifacts.add(eamArtifact);
503  }
504  }
505  } catch (CentralRepoException ex) {
506  LOGGER.log(Level.SEVERE, "Error counting notable artifacts.", ex);
507  }
508  }
509  }
510  if (FALSE == eamArtifacts.isEmpty()) {
511  for (CorrelationAttributeInstance eamArtifact : eamArtifacts) {
512  try {
513  dbManager.addArtifactInstance(eamArtifact);
514  } catch (CentralRepoException ex) {
515  LOGGER.log(Level.SEVERE, "Error adding artifact to database.", ex); //NON-NLS
516  }
517  }
518  } // DATA_ADDED
519  }
520  }
521 }
Collection< BlackboardArtifact > getArtifacts()
void removeIngestModuleEventListener(final PropertyChangeListener listener)
static synchronized IngestManager getInstance()
List< String > getListCasesHavingArtifactInstances(CorrelationAttributeInstance.Type aType, String value)
static void makeAndPostPreviousSeenArtifact(BlackboardArtifact originalArtifact, List< String > caseDisplayNames)
List< CorrelationAttributeInstance > getArtifactInstancesByTypeValue(CorrelationAttributeInstance.Type aType, String value)
static CorrelationDataSource fromTSKDataSource(CorrelationCase correlationCase, Content dataSource)
static void makeAndPostInterestingArtifact(BlackboardArtifact originalArtifact, Collection< BlackboardAttribute > attributesForNewArtifact)
List< String > getListCasesHavingArtifactInstancesKnownBad(CorrelationAttributeInstance.Type aType, String value)
void removeIngestJobEventListener(final PropertyChangeListener listener)
void addArtifactInstance(CorrelationAttributeInstance eamArtifact)
static void shutDownTaskExecutor(ExecutorService executor)
void addIngestJobEventListener(final PropertyChangeListener listener)
static void makeAndPostPreviousNotableArtifact(BlackboardArtifact originalArtifact, List< String > caseDisplayNames)
static List< CorrelationAttributeInstance > makeCorrAttrsToSave(BlackboardArtifact artifact)
void addIngestModuleEventListener(final PropertyChangeListener listener)
CorrelationDataSource getDataSource(CorrelationCase correlationCase, Long caseDbDataSourceId)
synchronized static Logger getLogger(String name)
Definition: Logger.java:124
DataAddedTask(CentralRepository db, PropertyChangeEvent evt, boolean flagNotableItemsEnabled, boolean flagPreviousItemsEnabled, boolean createCorrelationAttributes)

Copyright © 2012-2020 Basis Technology. Generated on: Wed Apr 8 2020
This work is licensed under a Creative Commons Attribution-Share Alike 3.0 United States License.